From d7c7454a19ec6fe46492a71a360625d88b5ce9b8 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 26 Feb 2026 21:31:28 +0000 Subject: [PATCH 01/30] feat: Making big steps in Makefile understanding, but it's still messy --- .claude/settings.local.json | 23 ++ .devcontainer/Dockerfile | 1 + .github/workflows/ci.yml | 2 +- .gitignore | 6 +- Makefile | 215 +++++++++++++------ PROD_DEBUG_REPORT.md | 234 +++++++++++++++++++++ SIMPLE_FIX_PLAN.md | 77 +++++++ charts/gitops-reverser/templates/rbac.yaml | 4 +- config/rbac/cluster-role.yaml | 53 ----- config/rbac/kustomization.yaml | 2 +- dist/kustomization.yaml | 8 + docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md | 6 +- docs/e2e-shared-infra-parallel-plan.md | 4 +- docs/e2e/kubectl-context.md | 76 +++++++ docs/make-e2e-deps.md | 4 +- docs/makefile-e2e-overview-and-plan.md | 172 +++++++++++++++ test/e2e/e2e_suite_test.go | 83 ++++++++ test/e2e/gitea-values.yaml | 6 +- test/e2e/quickstart_framework_e2e_test.go | 229 ++++++++++++++++++++ test/e2e/scripts/setup-gitea.sh | 51 +++-- test/e2e/scripts/setup-port-forwards.sh | 36 +++- 21 files changed, 1134 insertions(+), 158 deletions(-) create mode 100644 .claude/settings.local.json create mode 100644 PROD_DEBUG_REPORT.md create mode 100644 SIMPLE_FIX_PLAN.md delete mode 100644 config/rbac/cluster-role.yaml create mode 100644 dist/kustomization.yaml create mode 100644 docs/e2e/kubectl-context.md create mode 100644 docs/makefile-e2e-overview-and-plan.md create mode 100644 test/e2e/quickstart_framework_e2e_test.go diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 000000000..a15d65dea --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,23 @@ +{ + "permissions": { + "allow": [ + "Bash(make fmt)", + "Bash(make vet)", + "Bash(make lint)", + "Bash(make test)", + "Bash(make test-e2e)", + "Bash(make --dry-run test-e2e-quickstart-manifest)", + "Bash(make test-e2e-quickstart-manifest)", + "Bash(make test-e2e-quickstart-helm)", + "Bash(kubectl --context kind-gitops-reverser-test-e2e-quickstart-helm get clusterrole,clusterrolebinding,crd)", + "Bash(make --dry-run test-e2e-quickstart-helm)", + "Bash(make --dry-run test-e2e)", + "Bash(grep -n \"setupSOPSAgeSecret\\\\|generateAgeKey\\\\|e2eAgeKeyPath\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(grep -n \"readSOPSAgeKeyFromFile\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(grep -r \"make\\\\|Make\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(make .stamps/image/controller.id)", + "Bash(docker version)", + "Bash(docker buildx version)" + ] + } +} diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b3606c288..d80dd56c8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update \ vim \ less \ jq \ + yq \ sudo \ acl \ rsync \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7916f427..e72aaa4c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,7 +161,7 @@ jobs: - name: Generate install.yaml from Helm chart run: | - make build-installer + make dist/install.yaml - name: Package Helm chart run: | diff --git a/.gitignore b/.gitignore index 0d9190218..4f01e8feb 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,9 @@ __debug* charts/gitops-reverser/crds/*.yaml charts/gitops-reverser/config/*.yaml -# Keep the install.yaml for distribution -dist/install.yaml +# Generated single-file installer artifact (not tracked). +/dist/install.yaml *.o.yaml *.ignore.* -.stamps/* \ No newline at end of file +.stamps/* diff --git a/Makefile b/Makefile index 1824919d8..07daa1f37 100644 --- a/Makefile +++ b/Makefile @@ -41,20 +41,41 @@ help: ## Display this help. ##@ Development +CRD_BASE_OUTPUTS := \ + config/crd/bases/configbutler.ai_clusterwatchrules.yaml \ + config/crd/bases/configbutler.ai_gitproviders.yaml \ + config/crd/bases/configbutler.ai_gittargets.yaml \ + config/crd/bases/configbutler.ai_watchrules.yaml +MANIFEST_OUTPUTS := $(CRD_BASE_OUTPUTS) \ + config/rbac/role.yaml \ + config/webhook/manifests.yaml +MANIFEST_INPUTS := $(shell find api internal cmd -type f -name '*.go' \ + ! -name '*_test.go' ! -name 'zz_generated.deepcopy.go') .PHONY: manifests -manifests: ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - @rm -f config/crd/bases/*.yaml +manifests: $(MANIFEST_OUTPUTS) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +$(MANIFEST_OUTPUTS) &: $(MANIFEST_INPUTS) + @rm -f config/crd/bases/*.yaml \ + config/rbac/role.yaml \ + config/webhook/manifests.yaml $(CONTROLLER_GEN) rbac:roleName=gitops-reverser crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases +HELM_CRD_OUTPUTS := $(patsubst config/crd/bases/%,charts/gitops-reverser/crds/%,$(CRD_BASE_OUTPUTS)) +HELM_SYNC_OUTPUTS := $(HELM_CRD_OUTPUTS) \ + charts/gitops-reverser/config/role.yaml .PHONY: helm-sync -helm-sync: ## Sync CRDs and roles from config/crd/bases to Helm chart crds directory (for packaging) - @rm -f charts/gitops-reverser/crds/*.yaml +helm-sync: $(HELM_SYNC_OUTPUTS) ## Sync CRDs and roles from config/crd/bases to Helm chart crds directory (for packaging) +$(HELM_SYNC_OUTPUTS) &: $(MANIFEST_OUTPUTS) + @mkdir -p charts/gitops-reverser/crds charts/gitops-reverser/config + @rm -f charts/gitops-reverser/crds/*.yaml charts/gitops-reverser/config/*.yaml @cp config/crd/bases/*.yaml charts/gitops-reverser/crds/ - @rm -f charts/gitops-reverser/config/*.yaml - @cp config/rbac/cluster-role.yaml charts/gitops-reverser/config/ + @cp config/rbac/role.yaml charts/gitops-reverser/config/role.yaml +GENERATE_INPUTS := $(shell find api -type f -name '*.go' ! -name '*_test.go' \ + ! -name 'zz_generated.deepcopy.go') hack/boilerplate.go.txt +GENERATE_OUTPUT := api/v1alpha1/zz_generated.deepcopy.go -.PHONY: generate -generate: ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. +generate: $(GENERATE_OUTPUT) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + +$(GENERATE_OUTPUT): $(GENERATE_INPUTS) $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt @@ -114,8 +135,22 @@ cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps rm -rf .stamps/cluster/kind-$$cluster; \ done +cleanup-namespace-sut: $(CS)/sut.namespace.ready + kubectl --context $(CTX) delete validatingwebhookconfiguration \ + gitops-reverser-validating-webhook-configuration --ignore-not-found=true + +$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml + mkdir -p $(CS) + kubectl --context $(CTX) apply -f config/namespace.yaml + kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted + touch $@ + +// This is called by the full e2e suite -> it should for now clean the sut namespace, recreate it and run the installer and build and install new image. Let's use dependecies as much as possible -> this code can't call the e2e golang stuff: this is what we will call from golang +$(CS)/$(NAMESPACE)/e2e/prepare: + +// This is still what I would like to call! make test-e2e is the classic command that I'm used to .PHONY: test-e2e -test-e2e: $(CS)/e2e.passed ## Run end-to-end tests (stamp-based; only reruns what changed) +test-e2e: cleanup-namespace-sut .PHONY: lint lint: ## Run golangci-lint linter @@ -167,35 +202,17 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - $(CONTAINER_TOOL) buildx rm gitops-reverser-builder rm Dockerfile.cross -.PHONY: build-installer -build-installer: manifests helm-sync ## Generate a consolidated YAML from Helm chart for easy installation. - @echo "πŸ“¦ Generating install.yaml from Helm chart..." +CHART_INPUTS := charts/gitops-reverser/Chart.yaml \ + charts/gitops-reverser/values.yaml \ + $(shell find charts/gitops-reverser/templates -type f) + +dist/install.yaml: $(HELM_SYNC_OUTPUTS) $(CHART_INPUTS) ## Generate consolidated YAML from Helm chart. @mkdir -p dist - @$(HELM) template gitops-reverser charts/gitops-reverser \ - --namespace gitops-reverser \ + @$(HELM) template $(INSTALL_NAME) charts/gitops-reverser \ + --namespace $(NAMESPACE) \ --set labels.managedBy=kubectl \ --set createNamespace=true \ - --include-crds > dist/install.yaml - @echo "βœ… Generated dist/install.yaml ($(shell wc -l < dist/install.yaml) lines)" - -##@ Deployment - -.PHONY: install -install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=true -f - - -.PHONY: deploy -deploy: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG} - $(KUSTOMIZE) build config | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config | $(KUBECTL) delete --ignore-not-found=true -f - + --include-crds > $@ ##@ Dependencies @@ -226,32 +243,12 @@ setup-envtest: ## Setup envtest binaries for unit tests ##@ E2E Stamp Targets (cluster-parameterized; pass CTX=kind- to target a different cluster) -# DEPLOY_INPUTS: all config files except CRDs (tracked separately by crds.applied). -# Only relevant for the main e2e cluster (controller.deployed is not used by quickstart targets). -# When PROJECT_IMAGE is set (CI), skip the local image build stamp. -ifeq ($(PROJECT_IMAGE),) -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(CS)/sut.namespace.ready \ - $(IS)/controller.id \ - $(shell find config -type f -not -path 'config/crd/*') -else -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(CS)/sut.namespace.ready \ - $(shell find config -type f -not -path 'config/crd/*') -endif - $(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml mkdir -p $(CS) KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ -$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml - mkdir -p $(CS) - kubectl --context $(CTX) apply -f config/namespace.yaml - kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted - touch $@ - CERT_MANAGER_WAIT_TIMEOUT ?= 600s CERT_MANAGER_VERSION ?= v1.19.4 CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml @@ -313,12 +310,23 @@ $(CS)/image.loaded: $(IS)/controller.id $(CS)/ready $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) touch $@ +# DEPLOY_INPUTS: all config files except CRDs (tracked separately by crds.applied). +# Only relevant for the main e2e cluster (controller.deployed is not used by quickstart targets). +# When PROJECT_IMAGE is set (CI), skip the local image build stamp. +ifeq ($(PROJECT_IMAGE),) +DEPLOY_INPUTS := install \ + $(CS)/sut.namespace.ready \ + $(IS)/controller.id \ + $(shell find config -type f -not -path 'config/crd/*') +else +DEPLOY_INPUTS := install \ + $(CS)/sut.namespace.ready \ + $(shell find config -type f -not -path 'config/crd/*') +endif $(CS)/controller.deployed: $(DEPLOY_INPUTS) mkdir -p $(CS) @[ -z "$(PROJECT_IMAGE)" ] || $(CONTAINER_TOOL) pull $(E2E_IMAGE) - $(KIND) load docker-image $(E2E_IMAGE) --name $(CLUSTER_FROM_CTX) - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true + $(KIND) load docker-image $(E2E_IMAGE) --name $(CLUSTER_FROM_CTX) cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_IMAGE) $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - @if [ -z "$(PROJECT_IMAGE)" ]; then \ @@ -329,21 +337,14 @@ $(CS)/controller.deployed: $(DEPLOY_INPUTS) kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s touch $@ + .PHONY: $(CS)/portforward.running $(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed mkdir -p $(CS) - if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ - curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then \ - echo "port-forwards already healthy, skipping restart"; \ - touch $@; exit 0; \ - fi - kubectl config use-context $(CTX) - bash test/e2e/scripts/setup-port-forwards.sh - curl -fsS http://localhost:13000/api/healthz >/dev/null || { echo "Gitea health check failed"; exit 1; } - curl -fsS http://localhost:19090/-/healthy >/dev/null || { echo "Prometheus health check failed"; exit 1; } + E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh touch $@ -$(CS)/e2e.passed: $(CS)/controller.deployed $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') +$(CS)/e2e.passed: install $(CS)/controller.deployed $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') mkdir -p $(CS) kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_IMAGE) \ @@ -351,7 +352,81 @@ $(CS)/e2e.passed: $(CS)/controller.deployed $(CS)/portforward.running $(CS)/age- go test ./test/e2e/ -v -ginkgo.v touch $@ -##@ E2E Quickstart Tests +# INSTALL_MODE controls installation workflow: helm|plain-manifests-file|config-dir +INSTALL_MODE ?= config-dir +INSTALL_NAME ?= gitops-reverser +NAMESPACE ?= gitops-reverser +INSTALL_LOCK := $(CS)/$(NAMESPACE)/install.method +VALID_INSTALL_MODES := config-dir helm plain-manifests-file +ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) + $(error INSTALL_MODE must be one of: $(VALID_INSTALL_MODES)) +endif + +define ASSERT_INSTALL_METHOD +mkdir -p $(@D); \ +if [ -f "$(INSTALL_LOCK)" ]; then \ + existing="$$(cat "$(INSTALL_LOCK)")"; \ + if [ "$$existing" != "$(INSTALL_MODE)" ]; then \ + echo "ERROR: Namespace $(NAMESPACE) already installed using '$$existing'."; \ + echo "Refusing to install with '$(INSTALL_MODE)'."; \ + echo "Cleanup that namespace and remove $(INSTALL_LOCK) to switch methods."; \ + exit 2; \ + fi; \ +else \ + echo "$(INSTALL_MODE)" > "$(INSTALL_LOCK)"; \ +fi; +endef + +.PHONY: install install-helm install-plain-manifests-file install-config-dir +install: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) + +install-helm: $(CS)/$(NAMESPACE)/install-helm + +install-plain-manifests-file: $(CS)/$(NAMESPACE)/install-plain-manifests-file + +install-config-dir: $(CS)/$(NAMESPACE)/install-config-dir + +##@ Deployments of all manifests needed to run +$(CS)/$(NAMESPACE)/install-helm: INSTALL_MODE := helm +$(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via Helm into CTX cluster + @set -euo pipefail; \ + $(ASSERT_INSTALL_METHOD) \ + helm_args=( \ + upgrade --install $(INSTALL_NAME) charts/gitops-reverser \ + --kube-context $(CTX) \ + --namespace $(NAMESPACE) \ + --create-namespace \ + ); \ + if kubectl --context $(CTX) get crd \ + gitproviders.configbutler.ai \ + gittargets.configbutler.ai \ + watchrules.configbutler.ai \ + clusterwatchrules.configbutler.ai >/dev/null 2>&1; then \ + helm_args+=(--skip-crds); \ + fi; \ + helm "$${helm_args[@]}"; \ + touch $@ + +$(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir +$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy controller to the K8s cluster specified in ~/.kube/config. + @set -euo pipefail; \ + $(ASSERT_INSTALL_METHOD) + (cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG}) + (cd config && $(KUSTOMIZE) build .) | $(KUBECTL) apply -f - + touch $@ + +$(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file +$(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via manifest into CTX cluster + @set -euo pipefail; \ + ctx="$(CTX)"; \ + ns="$(NAMESPACE)"; \ + $(ASSERT_INSTALL_METHOD) \ + ( \ + cd dist && \ + kustomize edit set namespace "$$ns" >/dev/null && \ + kustomize build . \ + ) | kubectl --context "$$ctx" apply -f -; \ + touch $@ .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster @@ -374,7 +449,7 @@ test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-installer; \ + $(MAKE) build-plain-manifests-installer; \ $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ fi diff --git a/PROD_DEBUG_REPORT.md b/PROD_DEBUG_REPORT.md new file mode 100644 index 000000000..624cb55ac --- /dev/null +++ b/PROD_DEBUG_REPORT.md @@ -0,0 +1,234 @@ +# Production Debug Report: gitops-reverser + +Date: 2026-02-26 +Workspace: `/workspaces/gitops-reverser2` + +## Scope +- Investigate why configmap/secret changes in `gitops-reverser-example-audit-namespace` are not being tracked. +- Read-only investigation against cluster, except allowed restart of gitops-reverser and changes in `gitops-reverser-example-audit` namespace. + +## Findings Log + +### 1) Context and deployment health + +Command: +```bash +kubectl config current-context +kubectl -n gitops-reverser get all +helm list -A | rg gitops-reverser +``` + +Result: +- Current context: `k3s` +- Pod/Deployment/Service for gitops-reverser are healthy in namespace `gitops-reverser` +- Helm release: `gitops-reverser-gitops-reverser` chart `gitops-reverser-0.17.0+01bf7040249c` + +--- + +### 2) Config objects and watch scope + +Command: +```bash +kubectl -n gitops-reverser-example-audit get gittargets,watchrules -o yaml +kubectl explain watchrule.spec +kubectl get clusterwatchrules -A -o yaml +``` + +Result: +- `WatchRule only-configmaps` exists in `gitops-reverser-example-audit` +- Rule watches resources: `configmaps`, `secrets` +- `WatchRule` is namespace-scoped (per `kubectl explain watchrule.spec`) +- No `ClusterWatchRule` exists + +Interpretation: +- Current setup only tracks objects in namespace `gitops-reverser-example-audit`. + +--- + +### 3) Namespace mismatch check + +Command: +```bash +kubectl get ns | rg gitops-reverser-example +kubectl get ns gitops-reverser-example-audit-namespace -o yaml +``` + +Result: +- Existing namespace: `gitops-reverser-example-audit` +- Namespace `gitops-reverser-example-audit-namespace` does **not** exist + +Interpretation: +- If changes were made in `gitops-reverser-example-audit-namespace`, they cannot be tracked because the namespace does not exist. + +--- + +### 4) Metrics before test + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep "^gitopsreverser_" | sort' +``` + +Key values before controlled test: +- `gitopsreverser_events_received_total`: 6089 +- `gitopsreverser_events_processed_total`: 18 +- `gitopsreverser_commits_total`: 1 +- `gitopsreverser_objects_written_total`: 1 + +--- + +### 5) Controlled end-to-end test in `gitops-reverser-example-audit` + +Command: +```bash +ns="gitops-reverser-example-audit" +cm="codex-debug-cm-081911" +sec="codex-debug-secret-081911" + +kubectl -n "$ns" create configmap "$cm" --from-literal=foo=one +kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=one +kubectl -n "$ns" create configmap "$cm" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - +kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - +kubectl -n "$ns" delete configmap "$cm" --wait=true +kubectl -n "$ns" delete secret "$sec" --wait=true +``` + +Log correlation command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=5m | rg 'codex-debug-cm-081911|codex-debug-secret-081911|Created commit|All events pushed|writeEvents operation|Starting git commit and push' +``` + +Observed logs (highlights): +- `Starting git commit and push` with `eventCount: 6` +- `Created commit` for: + - `CREATE v1/configmaps/codex-debug-cm-081911` + - `CREATE v1/secrets/codex-debug-secret-081911` + - `UPDATE v1/configmaps/codex-debug-cm-081911` + - `UPDATE v1/secrets/codex-debug-secret-081911` + - `DELETE v1/configmaps/codex-debug-cm-081911` + - `DELETE v1/secrets/codex-debug-secret-081911` +- `All events pushed to remote` after each event + +Post-test metrics/status: + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep -E "^gitopsreverser_(events_received_total|events_processed_total|objects_written_total|commits_total|git_operations_total|git_commit_queue_size)" | sort' +kubectl -n gitops-reverser-example-audit get gittarget to-folder-live-cluster -o jsonpath='{.status.lastCommit}{"\n"}{.status.lastReconcileTime}{"\n"}{.status.snapshot.stats.created}{"\n"}{.status.snapshot.stats.deleted}{"\n"}' +``` + +Result: +- `events_processed_total`: 18 -> 24 +- `commits_total`: 1 -> 2 +- `git_operations_total`: 1 -> 7 +- `objects_written_total`: 1 -> 7 +- `GitTarget.status.lastCommit`: changed to `3e4cfd329e3b3e7638253bb3db3beb61ce363a0c` + +Interpretation: +- ConfigMap/Secret tracking is functioning in `gitops-reverser-example-audit`. + +--- + +### 6) Additional finding: startup seed race (likely non-blocking after startup) + +Command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=20m | rg -i 'Failed to route event to GitTargetEventStream|dropping event' +``` + +Observed error: +- `Failed to route event to GitTargetEventStream - dropping event` +- `error: no GitTargetEventStream registered for gitops-reverser-example-audit/to-folder-live-cluster` +- Appears around startup/initial seed window. + +Interpretation: +- There is a race during initial seed listing where events can be dropped before the stream is registered. +- This did **not** prevent later live create/update/delete processing in this test. + +--- + +## Conclusion + +- The system is actively tracking ConfigMap and Secret changes in `gitops-reverser-example-audit` and pushing them to Git. +- Main issue found in this debugging session: namespace mismatch (`gitops-reverser-example-audit-namespace` does not exist; watch rule is namespace-scoped to `gitops-reverser-example-audit`). +- Secondary issue found: startup seed event-drop race exists in logs and may affect initial catch-up behavior at startup. + +--- + +### 7) Root cause for "committed then immediately removed" (`oeps3`) + +#### 7.1 Reproduction evidence from logs + +Command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=90m | rg -n 'oeps3|Starting git commit and push|Deleted file from repository|Created commit' +``` + +Observed sequence: +- `08:21:02` -> `Created commit` `operation:"CREATE"` `resource:"v1/configmaps/oeps3"` +- `08:21:10` -> `Deleted file from repository` `file:"live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml"` +- `08:21:10` -> `Created commit` `operation:"DELETE"` `resource:"live-cluster/v1/configmaps/oeps3"` + +At the same time, object still exists in cluster: + +Command: +```bash +kubectl -n gitops-reverser-example-audit get cm oeps3 -o yaml +``` + +Result: +- `ConfigMap oeps3` exists (`creationTimestamp: 2026-02-26T08:20:55Z`). + +#### 7.2 Why this happens (code-path analysis) + +1) Repo-state reconciliation asks BranchWorker to list resources under GitTarget path: + +- [`internal/watch/event_router.go`](/workspaces/gitops-reverser2/internal/watch/event_router.go:151) + - `worker.ListResourcesInPath(gitTarget.Spec.Path)` + +2) BranchWorker walks `basePath = repoPath + path`, but computes relative path against **repo root**: + +- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:358) +- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:371) + - `relPath := filepath.Rel(repoPath, path)` + +This means files are parsed like: +- `live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml` + +3) Path parser interprets first segment as API group for 5-part paths: + +- [`internal/git/helpers.go`](/workspaces/gitops-reverser2/internal/git/helpers.go:97) + - For 5 parts, sets `group = parts[0]`. + +So parsed identifier becomes effectively: +- `group=live-cluster, version=v1, resource=configmaps, namespace=gitops-reverser-example-audit, name=oeps3` + +instead of core group (`group=""`). + +4) FolderReconciler compares cluster identifiers vs git identifiers and emits deletes for git-only items: + +- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:126) +- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:145) + +Because keys mismatch (`"/v1/..."` vs `"live-cluster/v1/..."`), it emits DELETE. + +5) Event stream behavior makes this asymmetrical: + +- [`internal/reconcile/git_target_event_stream.go`](/workspaces/gitops-reverser2/internal/reconcile/git_target_event_stream.go:161) + - Events with `Object=nil` are skipped **except DELETE**. + +So reconcile-created CREATE events are dropped, but reconcile-created DELETE events are processed, causing file removal commits. + +#### 7.3 Additional corroboration + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'find /tmp/gitops-reverser-workers/gitops-reverser-example-audit/your-repo/test-gitops-reverser/repos/54d901857c780feacd94c0da35957f7a/live-cluster -type f | sort' +``` + +Result at capture time: +- `live-cluster/v1/configmaps/...` file for `oeps3` was absent in repo cache +- `oeps3` still existed in Kubernetes + +This confirms Git side deletion without cluster deletion. + diff --git a/SIMPLE_FIX_PLAN.md b/SIMPLE_FIX_PLAN.md new file mode 100644 index 000000000..582d9695a --- /dev/null +++ b/SIMPLE_FIX_PLAN.md @@ -0,0 +1,77 @@ +# Simple Fix Plan: "CREATE then immediate DELETE" for namespaced resources + +## Goal +Stop false DELETE commits (for example `oeps3`) while the resource still exists in the cluster. + +## Root Cause (confirmed) +Repo-state listing parses file paths relative to repo root instead of GitTarget base path. + +Example today: +- Actual file: `live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml` +- Parsed as if `group=live-cluster` (wrong) +- Cluster state uses core group (`group=""`) +- Reconciler sees mismatch and emits DELETE for the Git file. + +## Design Principle +Keep the fix minimal and local: +- Fix path normalization at the source (`BranchWorker.listResourceIdentifiersInPath`) +- Do not redesign reconciliation or event stream behavior in this change + +## Implementation Plan +1. Update path handling in `listResourceIdentifiersInPath` +- File: `internal/git/branch_worker.go` +- Change: + - Keep scanning `basePath := repoPath + targetPath`. + - Compute `relPath` relative to `basePath` (not `repoPath`) when `targetPath` is set. + - Normalize separators with `filepath.ToSlash(relPath)` before parsing. +- Expected effect: + - Parser receives `v1/configmaps//.yaml` instead of `live-cluster/v1/...`. + +2. Add regression unit test for path-prefixed repos +- File: `internal/git/branch_worker_test.go` (or dedicated small test file in `internal/git/`) +- Test scenario: + - Create test repo tree containing `live-cluster/v1/configmaps/ns1/oeps3.yaml`. + - Call `ListResourcesInPath("live-cluster")`. + - Assert returned identifier is: + - `Group: ""` + - `Version: "v1"` + - `Resource: "configmaps"` + - `Namespace: "ns1"` + - `Name: "oeps3"` +- Add negative case: + - Ensure marker files (like `.configbutler`) are still ignored. + +3. Add focused reconciler regression test (small) +- File: `internal/reconcile/folder_reconciler_test.go` (or integration test if already present) +- Scenario: + - Cluster state has `/v1/configmaps/ns1/oeps3`. + - Repo state from parsed path should match same key. + - Assert no DELETE is emitted for identical resource identity. + +## Validation Plan +Run in this order: + +```bash +make fmt +make vet +make lint +make test +``` + +Then run required suite before merge: + +```bash +docker info +make test-e2e +make test-e2e-quickstart-manifest +make test-e2e-quickstart-helm +``` + +## Out of Scope (for this fix) +- Changing `GitTargetEventStream` behavior for nil-object CREATE events. +- Reworking snapshot/reconcile architecture. + +## Risk +Low. +- Change is isolated to repo path parsing in one function. +- Regression tests directly cover the broken case. diff --git a/charts/gitops-reverser/templates/rbac.yaml b/charts/gitops-reverser/templates/rbac.yaml index 4b6bd0d5d..786f1d73c 100644 --- a/charts/gitops-reverser/templates/rbac.yaml +++ b/charts/gitops-reverser/templates/rbac.yaml @@ -1,7 +1,7 @@ {{- if .Values.rbac.create -}} -{{- $roleYaml := .Files.Get "config/cluster-role.yaml" -}} +{{- $roleYaml := .Files.Get "config/role.yaml" -}} {{- if eq $roleYaml "" -}} -{{- fail "charts/gitops-reverser/config/cluster-role.yaml is missing. Run `make helm-sync` to regenerate chart RBAC artifacts before templating/installing." -}} +{{- fail "charts/gitops-reverser/config/role.yaml is missing. Run `make helm-sync` to regenerate chart RBAC artifacts before templating/installing." -}} {{- end -}} {{- $roleFile := $roleYaml | fromYaml }} diff --git a/config/rbac/cluster-role.yaml b/config/rbac/cluster-role.yaml deleted file mode 100644 index 799c25126..000000000 --- a/config/rbac/cluster-role.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: gitops-reverser -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - patch - - update -- apiGroups: - - configbutler.ai - resources: - - clusterwatchrules - - gitproviders - - gittargets - - watchrules - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - configbutler.ai - resources: - - clusterwatchrules/status - - gitproviders/status - - gittargets/status - - watchrules/status - verbs: - - get - - patch - - update -- apiGroups: - - configbutler.ai - resources: - - gitproviders/finalizers - verbs: - - update diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 7b51eb49b..86f409456 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - sa.yaml - - cluster-role.yaml + - role.yaml - cluster-rolebinding.yaml - cluster-rolebinding-jane.yaml # E2E-only helper: allows impersonated user jane@acme.com writes in tests. diff --git a/dist/kustomization.yaml b/dist/kustomization.yaml new file mode 100644 index 000000000..9e7ab5845 --- /dev/null +++ b/dist/kustomization.yaml @@ -0,0 +1,8 @@ +# Helper to facilitate quick deployment to another ns (currently used for e2e) +# kustomize edit set namespace blaat +# kustomize build . | kubectl apply -f - +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- install.yaml + diff --git a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md index d4c8857d4..51bf90b07 100644 --- a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md +++ b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md @@ -73,7 +73,7 @@ Outcome: - Run `make test-e2e-install-helm` or `make test-e2e-install-manifest` with no `PROJECT_IMAGE`. - Make cleans cluster first (clean-install validation), then sets up e2e infra. - Make auto-builds and Kind-loads `$(E2E_LOCAL_IMAGE)`, then runs smoke install using that image. -- For `test-e2e-install-manifest`, `build-installer` is run first to regenerate `dist/install.yaml`. +- For `test-e2e-install-manifest`, `build-plain-manifests-installer` is run first to regenerate `dist/install.yaml`. Outcome: - Automatic local behavior with explicit clean-install validation for smoke tests. @@ -97,7 +97,7 @@ Outcome: - local fallback path: cleanup cluster, setup infra, build/load local image. - `test-e2e-install-helm`: wrapper to `test-e2e-install`. - `test-e2e-install-manifest`: - - local path: run `build-installer` first. + - local path: run `build-plain-manifests-installer` first. - prebuilt path: use existing manifest artifact. ### Go (`test/e2e/e2e_suite_test.go`) @@ -144,7 +144,7 @@ Outcome: - Image pull failures in Kind: - Ensure local build/load ran or `PROJECT_IMAGE` points to reachable registry. - Manifest smoke using stale image: - - Local path: verify `build-installer` ran before smoke target. + - Local path: verify `build-plain-manifests-installer` ran before smoke target. - CI/prebuilt path: verify artifact source and `kubectl set image` override message. - Pod readiness says "no matching resources": - Verify selector in smoke logs (`Pod selector: ...`) and deployment selector labels. diff --git a/docs/e2e-shared-infra-parallel-plan.md b/docs/e2e-shared-infra-parallel-plan.md index b2c0a286b..7f8a1bb05 100644 --- a/docs/e2e-shared-infra-parallel-plan.md +++ b/docs/e2e-shared-infra-parallel-plan.md @@ -104,8 +104,8 @@ Shared prerequisites: Deploy/install actions (to add/refine): - `e2e-deploy-full` (run-scoped install/deploy for full flow) -- `e2e-deploy-quickstart-helm` (run-scoped Helm quickstart deploy) -- `e2e-deploy-quickstart-manifest` (run-scoped manifest quickstart deploy) +- `$(CS)/$(NAMESPACE)/deploy-helm` (run-scoped Helm quickstart deploy) +- `deploy-installer` (run-scoped manifest quickstart deploy) Expected inputs for these targets: diff --git a/docs/e2e/kubectl-context.md b/docs/e2e/kubectl-context.md new file mode 100644 index 000000000..b4704bb7b --- /dev/null +++ b/docs/e2e/kubectl-context.md @@ -0,0 +1,76 @@ +We will preserve the user chosen kubectl context: but inside the the tests we will everywhere switch to the right context. + +Make it an environment contract: Make exports either KUBECONFIG and/or a β€œready-to-append” CTX (or KUBE_CONTEXT) and every script uses that consistently. + +Option A (my go-to): export KUBECONFIG + CTX from Make + +Makefile: + +KUBECONFIG ?= +KUBE_CONTEXT ?= + +export KUBECONFIG +export CTX := $(if $(KUBE_CONTEXT),--context $(KUBE_CONTEXT),) + +.PHONY: e2e +e2e: + ./hack/run-e2e.sh + +Script(s): + +#!/usr/bin/env bash +set -euo pipefail + +# CTX is either empty or "--context name" +kubectl ${CTX:-} get ns +kubectl ${CTX:-} -n kube-system get pods + +Usage: + +make e2e +KUBE_CONTEXT=prod make e2e +KUBECONFIG=./kubeconfigs/prod.yaml make e2e +KUBECONFIG=./kubeconfigs/prod.yaml KUBE_CONTEXT=prod make e2e + +This works because Make exports env vars to subprocesses, and your scripts just append ${CTX:-}. + +Option B: export KUBE_CONTEXT and have scripts add --context + +Makefile: + +export KUBECONFIG ?= +export KUBE_CONTEXT ?= + +Script: + +kubectl_ctx=() +if [[ -n "${KUBE_CONTEXT:-}" ]]; then + kubectl_ctx+=(--context "$KUBE_CONTEXT") +fi + +kubectl "${kubectl_ctx[@]}" get ns + +This is more robust for quoting (contexts with weird chars/spaces), and avoids word-splitting issues. + +I’d pick B if you can touch scripts + +Because CTX="--context foo" relies on correct splitting. It’s usually fine (context names almost never have spaces), but arrays are bulletproof. + +Tiny helper to keep scripts clean + +Drop this in hack/kubectl.sh and source it from scripts: + +kubectl_cmd() { + local args=() + [[ -n "${KUBECONFIG:-}" ]] && export KUBECONFIG + [[ -n "${KUBE_CONTEXT:-}" ]] && args+=(--context "$KUBE_CONTEXT") + command kubectl "${args[@]}" "$@" +} + +Then in scripts: + +source "$(dirname "$0")/kubectl.sh" +kubectl_cmd get ns +kubectl_cmd -n default get pods + +If you show one of your bash scripts (how it currently calls kubectl, and whether it already reads env), I’ll adapt this to your exact style (and keep it consistent with your existing $(CTX) usage). \ No newline at end of file diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index e2e905f7d..0e7f90317 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -320,7 +320,7 @@ test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-installer; \ + $(MAKE) build-plain-manifests-installer; \ $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ fi @@ -329,7 +329,7 @@ test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - bash test/e2e/scripts/run-quickstart.sh manifest ``` -The manifest target also calls `make build-installer` (regenerates `dist/install.yaml`) before loading the image if `PROJECT_IMAGE` is not set. +The manifest target also calls `make build-plain-manifests-installer` (regenerates `dist/install.yaml`) before loading the image if `PROJECT_IMAGE` is not set. ### `run-quickstart.sh` is self-contained diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md new file mode 100644 index 000000000..f11768cd2 --- /dev/null +++ b/docs/makefile-e2e-overview-and-plan.md @@ -0,0 +1,172 @@ +# Makefile E2E Overview and Next-Step Plan + +## Purpose + +This document explains the current Makefile-driven E2E model and proposes the next implementation steps to reach: + +- seed-based, run-scoped test namespaces +- explicit suite x install-method combinations +- quickstart assertions implemented in Go (not shell) +- CI-aligned developer entrypoints + +## Current State + +## Full E2E (`make test-e2e`) + +Current entrypoint: + +- `test-e2e -> $(CS)/e2e.passed` + +Current dependency chain for full E2E: + +- `$(CS)/e2e.passed` depends on: + - `install` (default `INSTALL_MODE=config-dir`) + - `$(CS)/controller.deployed` + - `$(CS)/portforward.running` + - `$(CS)/age-key.applied` + - `test/e2e/**/*.go` + +`$(CS)/controller.deployed` currently: + +- loads image into Kind (`$(E2E_IMAGE)`) +- deletes validating webhook configuration +- sets image in `config/` via `kustomize edit set image` +- applies `kustomize build config` +- rolls out `deploy/gitops-reverser` in namespace `sut` + +Important relationship: + +- `controller.deployed` is the cluster-state gate for full suite execution. +- `e2e.passed` is the test-result stamp that runs `go test ./test/e2e`. + +## Quickstart E2E + +Current entrypoints: + +- `make test-e2e-quickstart-helm` +- `make test-e2e-quickstart-manifest` + +These flows: + +- always cleanup dedicated Kind cluster first +- install cert-manager + gitea via shared stamps +- load image for local runs +- execute `test/e2e/scripts/run-quickstart.sh` in `helm` or `manifest` mode + +CI alignment is already strong: + +- `.github/workflows/ci.yml` runs: + - `make test-e2e` + - matrix: `make test-e2e-quickstart-helm|manifest` + +## Seed and Namespace Today + +What exists today: + +- Ginkgo seed is exported to `make` as `E2E_GINKGO_RANDOM_SEED`. +- quickstart framework installer namespace uses `run-`. + +What is missing: + +- full E2E still uses fixed namespace constants (`sut`) in Go helpers/tests. +- Makefile does not derive namespace/build stamp identity from seed. +- no run-scoped namespace stamp used by full E2E test resources. + +## Gaps Against Target Vision + +1. Seed is propagated but not structurally used for run scoping. +2. Suite x install-method model is partial: + - quickstart: `helm`, `manifest` + - full: effectively `config-dir` + extra deploy path + - `config-dir` is not clearly positioned as either product path or internal test path. +3. Quickstart Go framework lacks parity with `run-quickstart.sh` assertions (commit progression, encrypted secret checks, invalid-credential message checks). +4. Full E2E still contains significant infra-level shell/Kubernetes orchestration in Go (`setup-gitea.sh`, many direct `kubectl`). +5. Local quickstart manifest target has a defect: + - `test-e2e-quickstart-manifest` calls `build-plain-manifests-installer`, but no such target exists. + +## Proposed End-State Model + +## Test matrix + +Define explicit, stable matrix naming: + +- `full-config-dir` (current full suite path) +- `quickstart-helm` +- `quickstart-manifest` + +Optional future (only if needed): + +- `full-helm` +- `full-manifest` + +## Seed contract + +Single run contract shared by Make and Go: + +- `E2E_RUN_SEED` (defaults to ginkgo seed when available) +- `E2E_RUN_NS=run-$(E2E_RUN_SEED)` +- `E2E_CONTROLLER_NS` (keep `sut` initially, parameterize later) + +## Ownership split + +- Makefile owns cluster infra and deployment/install orchestration. +- Go tests own scenario logic and assertions. +- Go should not perform install/deploy internals directly. + +## Phased Plan + +## Phase 0: Stabilize current paths (small PR) + +1. Fix missing target reference in `test-e2e-quickstart-manifest`: + - replace `build-plain-manifests-installer` with existing `dist/install.yaml` dependency path. +2. Remove `.PHONY` from `$(CS)/portforward.running` so stamp behavior is real. +3. Add/refresh Makefile overview doc (this file) and link it from `docs/make-e2e-deps.md`. + +## Phase 1: Introduce run seed + run namespace primitives + +1. Add Make variables: + - `E2E_RUN_SEED ?= $(if $(E2E_GINKGO_RANDOM_SEED),$(E2E_GINKGO_RANDOM_SEED),$(shell date +%s))` + - `E2E_RUN_NS ?= run-$(E2E_RUN_SEED)` +2. Add run namespace stamp: + - `$(CS)/run-ns.ready` applying/labeling `$(E2E_RUN_NS)`. +3. Pass `E2E_RUN_SEED`, `E2E_RUN_NS`, `E2E_CONTROLLER_NS` into `go test` env. +4. In Go tests, replace hardcoded test namespace constants with env-backed helper. + +## Phase 2: Explicit suite x install-method targets + +1. Introduce clear top-level targets: + - `test-e2e-full-config-dir` + - `test-e2e-quickstart-helm` + - `test-e2e-quickstart-manifest` +2. Keep `test-e2e` as alias to `test-e2e-full-config-dir` for compatibility. +3. Add `test-e2e-ci` as local equivalent of CI flow: + - full + quickstart-helm + quickstart-manifest. + +## Phase 3: Port `run-quickstart.sh` assertions into Go + +Port these checks into `test/e2e/quickstart_framework_e2e_test.go`: + +1. resource readiness (`GitProvider`, `GitTarget`, `WatchRule`) +2. generated SOPS secret exists + backup-warning + age-recipient annotation +3. create/update/delete ConfigMap produces expected repo file transitions and commit count increments +4. Secret commit is encrypted (no plaintext/base64 leakage) +5. encrypted file decrypts with generated age key and shows updated value +6. invalid credentials produce actionable `ConnectionFailed` status message + +Then reduce `run-quickstart.sh` to thin install/bootstrap wrapper or retire it. + +## Phase 4: Minimize infra/Kubernetes orchestration in Go + +1. Move remaining setup operations behind Make targets/stamps. +2. Keep Go focused on test scenario inputs + assertions. +3. Add one convenience target for developers (candidate name): + - `make last-deployment` or `make e2e-prepare` to provision infra without running tests. + +## Recommended Immediate Next PR + +1. Apply Phase 0 fixes. +2. Start Phase 3 by porting one vertical slice in Go: + - ConfigMap create/update/delete + repo file + commit assertions. +3. Keep CI job structure unchanged while moving logic from shell to Go. + +This gives fast signal improvement without destabilizing the current pipeline. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ad119805e..908d020fa 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -21,12 +21,18 @@ package e2e import ( "fmt" "os" + "os/exec" + "strings" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/ConfigButler/gitops-reverser/test/utils" ) +const e2eGinkgoSeedEnv = "E2E_GINKGO_RANDOM_SEED" + // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, // temporary environment to validate project changes with the purpose of being used in CI jobs. // The default setup requires Kind, builds/loads the Manager Docker image locally, and installs @@ -45,7 +51,84 @@ var _ = BeforeSuite(func() { } else { By(fmt.Sprintf("using pre-built image: %s", img)) } + + By("ensuring shared e2e port-forwards are active via Makefile target") + ensureE2EPortForwards() }) var _ = AfterSuite(func() { }) + +func ensureE2EPortForwards() { + ctx := resolveE2EContext() + target := fmt.Sprintf(".stamps/cluster/%s/portforward.running", ctx) + cmd := makeCommandWithSeed(fmt.Sprintf("CTX=%s", ctx), target) + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to run make target for port-forwards") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) +} + +func makeCommandWithSeed(args ...string) *exec.Cmd { + seed := ginkgoRandomSeed() + seedPrefix := e2eGinkgoSeedEnv + "=" + filteredArgs := make([]string, 0, len(args)+1) + for _, arg := range args { + if strings.HasPrefix(arg, seedPrefix) { + continue + } + filteredArgs = append(filteredArgs, arg) + } + filteredArgs = append(filteredArgs, fmt.Sprintf("%s=%d", e2eGinkgoSeedEnv, seed)) + cmd := exec.Command("make", filteredArgs...) + _, _ = fmt.Fprintf( + GinkgoWriter, + "make invocation: seed=%d arg=%s args=%v\n", + seed, + fmt.Sprintf("%s=%d", e2eGinkgoSeedEnv, seed), + filteredArgs, + ) + return cmd +} + +func ginkgoRandomSeed() int64 { + suiteConfig, _ := GinkgoConfiguration() + return suiteConfig.RandomSeed +} + +func resolveE2EContext() string { + if value := strings.TrimSpace(os.Getenv("CTX")); value != "" { + return value + } + + if cluster := strings.TrimSpace(os.Getenv("KIND_CLUSTER")); cluster != "" { + if strings.HasPrefix(cluster, "kind-") { + return cluster + } + return fmt.Sprintf("kind-%s", cluster) + } + + cmd := exec.Command("kubectl", "config", "current-context") + output, err := utils.Run(cmd) + if err == nil { + if value := strings.TrimSpace(output); value != "" { + return value + } + } + + return "kind-gitops-reverser-test-e2e" +} + +/* + +At startup of tests we should define the seed of the test run and use that to create a namespace and to run tests in it, also the build should be depending on that. + +We should be cominbing instllation methods with testsuites + +helm +manifests +config (is there still use for this?!) + +full +quickstart + +*/ diff --git a/test/e2e/gitea-values.yaml b/test/e2e/gitea-values.yaml index 103368f16..551c7c747 100644 --- a/test/e2e/gitea-values.yaml +++ b/test/e2e/gitea-values.yaml @@ -75,11 +75,9 @@ service: # Resource limits for CI environments resources: limits: - memory: "512Mi" - cpu: "500m" + memory: "1024Mi" requests: - memory: "256Mi" - cpu: "250m" + memory: "512Mi" podSecurityContext: runAsNonRoot: true diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go new file mode 100644 index 000000000..b862b6655 --- /dev/null +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -0,0 +1,229 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/ConfigButler/gitops-reverser/test/utils" +) + +const ( + quickstartFrameworkEnabledEnv = "E2E_ENABLE_QUICKSTART_FRAMEWORK" + quickstartFrameworkModeEnv = "E2E_QUICKSTART_MODE" + quickstartSetupNamespace = "sut" +) + +type quickstartFrameworkRun struct { + mode string + installNamespace string + testID string + repoName string + checkoutDir string + repoURL string + providerName string + targetName string + watchRuleName string + invalidProvName string + encryptionName string +} + +var _ = Describe("Quickstart Framework", Label("quickstart-framework"), Ordered, func() { + var run quickstartFrameworkRun + + BeforeAll(func() { + if !quickstartFrameworkEnabled() { + Skip(fmt.Sprintf( + "quickstart framework is disabled; set %s=true to run", + quickstartFrameworkEnabledEnv, + )) + } + + run = newQuickstartFrameworkRun() + }) + + It("sets up quickstart flow via Go framework", func() { + By("resetting install state for clean quickstart framework execution") + run.resetInstallState() + + By(fmt.Sprintf("installing controller for quickstart mode %q", run.mode)) + run.installController() + run.waitForControllerRollout() + + By("creating dedicated Gitea repository and bootstrap credentials") + run.setupGiteaRepository() + + By("applying quickstart resources from Go") + run.applyQuickstartResources() + }) + +}) + +func quickstartFrameworkEnabled() bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(quickstartFrameworkEnabledEnv))) + return value == "1" || value == "true" || value == "yes" +} + +func quickstartFrameworkMode() string { + mode := strings.ToLower(strings.TrimSpace(os.Getenv(quickstartFrameworkModeEnv))) + if mode == "" { + return "helm" + } + + Expect(mode == "config-dir" || mode == "helm" || mode == "plain-manifests-file").To( + BeTrue(), + fmt.Sprintf( + "unsupported %s value %q (expected config-dir|helm|plain-manifests-file)", + quickstartFrameworkModeEnv, + mode, + ), + ) + return mode +} + +func newQuickstartFrameworkRun() quickstartFrameworkRun { + mode := quickstartFrameworkMode() + testID := strconv.FormatInt(time.Now().UnixNano(), 10) + repoName := fmt.Sprintf("quickstart-framework-%s-%s", mode, testID) + checkoutDir := filepath.Join("/tmp/gitops-reverser", repoName) + + return quickstartFrameworkRun{ + mode: mode, + installNamespace: quickstartInstallerNamespace(), + testID: testID, + repoName: repoName, + checkoutDir: checkoutDir, + repoURL: fmt.Sprintf("http://gitea-http.gitea-e2e.svc.cluster.local:13000/testorg/%s.git", repoName), + providerName: fmt.Sprintf("quickstart-provider-%s", testID), + targetName: fmt.Sprintf("quickstart-target-%s", testID), + watchRuleName: fmt.Sprintf("quickstart-watchrule-%s", testID), + invalidProvName: fmt.Sprintf("quickstart-invalid-provider-%s", testID), + encryptionName: fmt.Sprintf("quickstart-sops-age-key-%s", testID), + } +} + +func (r *quickstartFrameworkRun) resetInstallState() { + deleteClusterScopeObjects := [][]string{ + { + "delete", "clusterrole", + "gitops-reverser", + "gitops-reverser-manager-role", + "gitops-reverser-metrics-reader", + "gitops-reverser-proxy-role", + "--ignore-not-found=true", + }, + { + "delete", "clusterrolebinding", + "gitops-reverser", + "gitops-reverser-manager-rolebinding", + "gitops-reverser-proxy-rolebinding", + "--ignore-not-found=true", + }, + {"delete", "validatingwebhookconfiguration", "gitops-reverser-validating-webhook-configuration", + "--ignore-not-found=true"}, + } + + for _, args := range deleteClusterScopeObjects { + cmd := exec.Command("kubectl", args...) + _, _ = utils.Run(cmd) + } + + deleteNamespace := exec.Command( + "kubectl", "delete", "namespace", r.installNamespace, "--wait=true", "--ignore-not-found=true", + ) + _, _ = utils.Run(deleteNamespace) +} + +func (r *quickstartFrameworkRun) installController() { + cmd := r.installCommand(r.mode) + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to install %s controller via make install", r.mode)) + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) +} + +func (r *quickstartFrameworkRun) installCommand(mode string) *exec.Cmd { + ctx := resolveE2EContext() + args := []string{ + fmt.Sprintf("CTX=%s", ctx), + "install", + fmt.Sprintf("INSTALL_MODE=%s", mode), + fmt.Sprintf("NAMESPACE=%s", r.installNamespace), + fmt.Sprintf("INSTALL_NAMESPACE=%s", r.installNamespace), + fmt.Sprintf("INSTALL_NAME=%s", r.installNamespace), + } + return makeCommandWithSeed(args...) +} + +func (r *quickstartFrameworkRun) waitForControllerRollout() { + rolloutCmd := exec.Command( + "kubectl", "-n", r.installNamespace, "rollout", "status", + "deployment/gitops-reverser", "--timeout=120s", + ) + _, err := utils.Run(rolloutCmd) + Expect(err).NotTo(HaveOccurred(), "controller rollout did not complete") +} + +func quickstartInstallerNamespace() string { + suiteConfig, _ := GinkgoConfiguration() + return fmt.Sprintf("run-%d", suiteConfig.RandomSeed) +} + +func (r *quickstartFrameworkRun) setupGiteaRepository() { + cmd := exec.Command("bash", "test/e2e/scripts/setup-gitea.sh", r.repoName, r.checkoutDir) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to bootstrap gitea repository for quickstart") +} + +func (r *quickstartFrameworkRun) applyQuickstartResources() { + createGitProviderWithURL(r.providerName, "main", "git-creds", r.repoURL) + + createGitTargetWithEncryptionOptions( + r.targetName, + quickstartSetupNamespace, + r.providerName, + "live-cluster", + "main", + r.encryptionName, + true, + ) + + watchRuleData := struct { + Name string + Namespace string + DestinationName string + }{ + Name: r.watchRuleName, + Namespace: quickstartSetupNamespace, + DestinationName: r.targetName, + } + + err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, quickstartSetupNamespace) + Expect(err).NotTo(HaveOccurred(), "failed to apply quickstart watchrule") + + createGitProviderWithURL(r.invalidProvName, "main", "git-creds-invalid", r.repoURL) +} diff --git a/test/e2e/scripts/setup-gitea.sh b/test/e2e/scripts/setup-gitea.sh index 043cfd39e..f2f455da4 100755 --- a/test/e2e/scripts/setup-gitea.sh +++ b/test/e2e/scripts/setup-gitea.sh @@ -49,21 +49,46 @@ test_api_connectivity() { # Function to create organization and get token setup_organization_and_token() { - echo "🏒 Creating test organization '$ORG_NAME'..." - ORG_RESPONSE=$(curl -s -w "%{http_code}" -o /tmp/org_response.json \ - -X POST "$API_URL/orgs" \ - -H "Content-Type: application/json" \ - -u "$ADMIN_USER:$ADMIN_PASS" \ - -d "{\"username\":\"$ORG_NAME\",\"full_name\":\"Test Organization\",\"description\":\"E2E Test Organization\"}") + organization_exists() { + local status + status=$(curl -s -w "%{http_code}" -o /tmp/org_get_response.json \ + -X GET "$API_URL/orgs/$ORG_NAME" \ + -H "Content-Type: application/json" \ + -u "$ADMIN_USER:$ADMIN_PASS" || true) + [[ "$status" == "200" ]] + } - if [[ "$ORG_RESPONSE" == "201" ]]; then - echo "βœ… Organization created successfully" - elif [[ "$ORG_RESPONSE" == "409" ]] || [[ "$ORG_RESPONSE" == "422" ]]; then + echo "🏒 Ensuring test organization '$ORG_NAME' exists..." + if organization_exists; then echo "ℹ️ Organization already exists" else - echo "⚠️ Unexpected response creating organization: $ORG_RESPONSE" - cat /tmp/org_response.json 2>/dev/null || true - echo "ℹ️ Continuing setup despite organization creation issue..." + local attempt + for attempt in {1..5}; do + ORG_RESPONSE=$(curl -s -w "%{http_code}" -o /tmp/org_response.json \ + -X POST "$API_URL/orgs" \ + -H "Content-Type: application/json" \ + -u "$ADMIN_USER:$ADMIN_PASS" \ + -d "{\"username\":\"$ORG_NAME\",\"full_name\":\"Test Organization\",\"description\":\"E2E Test Organization\"}" || true) + + if [[ "$ORG_RESPONSE" == "201" ]]; then + echo "βœ… Organization created successfully" + break + fi + if [[ "$ORG_RESPONSE" == "409" ]] || [[ "$ORG_RESPONSE" == "422" ]]; then + echo "ℹ️ Organization already exists" + break + fi + if organization_exists; then + echo "ℹ️ Organization already exists" + break + fi + + echo "⚠️ Attempt $attempt/5: failed to create organization (status=${ORG_RESPONSE:-none})" + cat /tmp/org_response.json 2>/dev/null || true + if [[ "$attempt" -lt 5 ]]; then + sleep 1 + fi + done fi # Generate or reuse access token @@ -349,4 +374,4 @@ echo " # Cleanup temporary files rm -f /tmp/org_response.json /tmp/repo_response.json /tmp/token_response.json /tmp/token_list.json \ - /tmp/ssh_key_response.json + /tmp/ssh_key_response.json /tmp/org_get_response.json diff --git a/test/e2e/scripts/setup-port-forwards.sh b/test/e2e/scripts/setup-port-forwards.sh index df0347f29..a886ad142 100755 --- a/test/e2e/scripts/setup-port-forwards.sh +++ b/test/e2e/scripts/setup-port-forwards.sh @@ -5,6 +5,23 @@ set -euo pipefail GITEA_NAMESPACE=${GITEA_NAMESPACE:-gitea-e2e} PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-prometheus-operator} PROMETHEUS_INSTANCE_NAME=${PROMETHEUS_INSTANCE_NAME:-prometheus-shared-e2e} +KUBE_CONTEXT=${E2E_KUBECONTEXT:-${CTX:-${KUBECONTEXT:-}}} + +if [[ -z "${KUBE_CONTEXT}" ]]; then + KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" +fi + +if [[ -z "${KUBE_CONTEXT}" ]]; then + echo "❌ Kubernetes context is required (set E2E_KUBECONTEXT or CTX)" + exit 1 +fi + +# Fast path: keep existing healthy forwards instead of restarting. +if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ + curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then + echo "βœ… Existing port-forwards are healthy for context ${KUBE_CONTEXT}; skipping restart" + exit 0 +fi # Cleanup old port-forwards echo "🧹 Cleaning up old port-forwards..." @@ -13,21 +30,22 @@ pkill -f "kubectl port-forward.*13000" || true sleep 1 echo "πŸ”Œ Setting up port-forwards for e2e testing..." +echo "🎯 Using kube context: ${KUBE_CONTEXT}" echo "⏳ Waiting for Prometheus pod to be ready..." -kubectl wait --for=condition=ready pod \ +kubectl --context "$KUBE_CONTEXT" wait --for=condition=ready pod \ -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" \ -n "$PROMETHEUS_NAMESPACE" \ --timeout=180s || { echo "❌ Prometheus pod failed to become ready" - kubectl get pods -n "$PROMETHEUS_NAMESPACE" -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" || true + kubectl --context "$KUBE_CONTEXT" get pods -n "$PROMETHEUS_NAMESPACE" -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" || true exit 1 } echo "βœ… Prometheus pod is ready" echo "⏳ Waiting for Gitea pod to be ready..." -kubectl wait --for=condition=ready pod \ +kubectl --context "$KUBE_CONTEXT" wait --for=condition=ready pod \ -l app.kubernetes.io/name=gitea \ -n "$GITEA_NAMESPACE" \ --timeout=180s || { @@ -54,7 +72,7 @@ setup_port_forward() { echo "πŸ”Œ Starting ${service_name} port-forward..." local pf_log="/tmp/${service_name}-pf.log" - kubectl port-forward --address 127.0.0.1 \ + kubectl --context "$KUBE_CONTEXT" port-forward --address 127.0.0.1 \ -n "$namespace" "svc/${service}" "${local_port}:${remote_port}" > "$pf_log" 2>&1 & local pf_pid=$! @@ -94,6 +112,16 @@ setup_port_forward() { setup_port_forward "Prometheus" "$PROMETHEUS_NAMESPACE" "prometheus-operated" "19090" "9090" setup_port_forward "Gitea" "$GITEA_NAMESPACE" "gitea-http" "13000" "13000" +# Validate HTTP endpoints before returning success. +curl -fsS http://localhost:13000/api/healthz >/dev/null || { + echo "❌ Gitea health check failed after port-forward setup" + exit 1 +} +curl -fsS http://localhost:19090/-/healthy >/dev/null || { + echo "❌ Prometheus health check failed after port-forward setup" + exit 1 +} + echo "" echo "πŸ“Š Prometheus: http://localhost:19090" echo "πŸ“ Gitea: http://localhost:13000" From 4affc1ea66eb555bb8096d80e7601b4c2bb2682a Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 08:44:33 +0000 Subject: [PATCH 02/30] ci: Let's see if we can e2e in GH again --- Makefile | 54 +++-- docs/make-e2e-deps.md | 2 + docs/makefile-e2e-overview-and-plan.md | 201 ++++++------------ test/e2e/e2e_suite_test.go | 36 +++- .../e2e/scripts/ensure-prometheus-operator.sh | 26 ++- 5 files changed, 156 insertions(+), 163 deletions(-) diff --git a/Makefile b/Makefile index 07daa1f37..585711a5d 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,12 @@ CS := .stamps/cluster/$(CTX) IS := .stamps/image GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum +# INSTALL_MODE controls installation workflow: helm|plain-manifests-file|config-dir +# These defaults must be defined before any target names that reference $(NAMESPACE). +INSTALL_MODE ?= config-dir +INSTALL_NAME ?= gitops-reverser +NAMESPACE ?= gitops-reverser + KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) @@ -139,18 +145,28 @@ cleanup-namespace-sut: $(CS)/sut.namespace.ready kubectl --context $(CTX) delete validatingwebhookconfiguration \ gitops-reverser-validating-webhook-configuration --ignore-not-found=true +$(CS)/sut.namespace.cleaned: $(CS)/sut.namespace.ready + kubectl --context $(CTX) delete validatingwebhookconfiguration \ + gitops-reverser-validating-webhook-configuration --ignore-not-found=true + touch $@ + $(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml mkdir -p $(CS) kubectl --context $(CTX) apply -f config/namespace.yaml kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted touch $@ -// This is called by the full e2e suite -> it should for now clean the sut namespace, recreate it and run the installer and build and install new image. Let's use dependecies as much as possible -> this code can't call the e2e golang stuff: this is what we will call from golang -$(CS)/$(NAMESPACE)/e2e/prepare: +# Called by the full e2e suite. +# For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. +# Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). +$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/sut.namespace.cleaned $(CS)/controller.deployed $(CS)/portforward.running \ + $(CS)/age-key.applied + mkdir -p $(@D) + touch $@ -// This is still what I would like to call! make test-e2e is the classic command that I'm used to +# Keep `make test-e2e` as the classic entrypoint. .PHONY: test-e2e -test-e2e: cleanup-namespace-sut +test-e2e: $(CS)/e2e.passed .PHONY: lint lint: ## Run golangci-lint linter @@ -277,6 +293,10 @@ PROMETHEUS_SETUP_MANIFESTS := $(shell find test/e2e/setup/prometheus -type f -na $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-operator.sh $(PROMETHEUS_SETUP_MANIFESTS) mkdir -p $(CS) KUBECONTEXT=$(CTX) bash test/e2e/scripts/ensure-prometheus-operator.sh + kubectl --context $(CTX) wait --for=condition=Established \ + crd/prometheuses.monitoring.coreos.com \ + crd/servicemonitors.monitoring.coreos.com \ + --timeout=180s kubectl --context $(CTX) apply -n prometheus-operator -f test/e2e/setup/prometheus kubectl --context $(CTX) wait --for=condition=Available prometheus/prometheus-shared-e2e -n prometheus-operator --timeout=180s kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s @@ -314,12 +334,12 @@ $(CS)/image.loaded: $(IS)/controller.id $(CS)/ready # Only relevant for the main e2e cluster (controller.deployed is not used by quickstart targets). # When PROJECT_IMAGE is set (CI), skip the local image build stamp. ifeq ($(PROJECT_IMAGE),) -DEPLOY_INPUTS := install \ +DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ $(CS)/sut.namespace.ready \ $(IS)/controller.id \ $(shell find config -type f -not -path 'config/crd/*') else -DEPLOY_INPUTS := install \ +DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ $(CS)/sut.namespace.ready \ $(shell find config -type f -not -path 'config/crd/*') endif @@ -333,29 +353,31 @@ $(CS)/controller.deployed: $(DEPLOY_INPUTS) IMAGE_ID="$$(cat $(IS)/controller.id)"; \ kubectl --context $(CTX) -n sut patch deploy/gitops-reverser --type=merge \ -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"configbutler.ai/e2e-image-id\":\"$$IMAGE_ID\"}}}}}"; \ + kubectl --context $(CTX) -n sut patch deploy/gitops-reverser --type=json \ + -p '[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]'; \ fi kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s touch $@ - -.PHONY: $(CS)/portforward.running $(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh touch $@ -$(CS)/e2e.passed: install $(CS)/controller.deployed $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') +.PHONY: portforward-ensure +portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure port-forwards are running (always checks) + mkdir -p $(CS) + E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh + +$(CS)/e2e.passed: $(CS)/$(NAMESPACE)/e2e/prepare $(shell find test/e2e -name '*.go') mkdir -p $(CS) kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_IMAGE) \ + CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ -# INSTALL_MODE controls installation workflow: helm|plain-manifests-file|config-dir -INSTALL_MODE ?= config-dir -INSTALL_NAME ?= gitops-reverser -NAMESPACE ?= gitops-reverser +# INSTALL_MODE, INSTALL_NAME, NAMESPACE defaults are defined near the E2E variables section above. INSTALL_LOCK := $(CS)/$(NAMESPACE)/install.method VALID_INSTALL_MODES := config-dir helm plain-manifests-file ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) @@ -412,7 +434,7 @@ $(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CS)/cert-manager.in @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) (cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG}) - (cd config && $(KUSTOMIZE) build .) | $(KUBECTL) apply -f - + (cd config && $(KUSTOMIZE) build .) | $(KUBECTL) --context $(CTX) apply -f - touch $@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file @@ -449,7 +471,7 @@ test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-plain-manifests-installer; \ + $(MAKE) dist/install.yaml; \ $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ fi diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index 0e7f90317..125f29c7d 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -1,5 +1,7 @@ # E2E test dependencies with Make stamps +For the roadmap/implementation plan, see `docs/makefile-e2e-overview-and-plan.md`. + ## What the old setup did (replaced) `make test-e2e` used to run these prerequisites unconditionally on every invocation: diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md index f11768cd2..39ade537d 100644 --- a/docs/makefile-e2e-overview-and-plan.md +++ b/docs/makefile-e2e-overview-and-plan.md @@ -1,172 +1,111 @@ -# Makefile E2E Overview and Next-Step Plan +# Makefile E2E Overview and Implementation Plan -## Purpose +This is the high-level roadmap for the Makefile-driven E2E work. For the detailed stamp dependency graph, see +`docs/make-e2e-deps.md`. -This document explains the current Makefile-driven E2E model and proposes the next implementation steps to reach: +## Goals (from Makefile + suite comments) -- seed-based, run-scoped test namespaces -- explicit suite x install-method combinations -- quickstart assertions implemented in Go (not shell) -- CI-aligned developer entrypoints +- Keep `make test-e2e` as the classic developer entrypoint. +- Use a **single seed** to scope a test run (namespaces + stamps), shared by Make and Go. +- Make the E2E model explicit as a **matrix**: `suite` Γ— `install-method`. +- Keep Go focused on scenario logic/assertions; Make owns cluster orchestration (Kind/Helm/kubectl/image/stamps). -## Current State +## Current reality (as of Feb 26, 2026) -## Full E2E (`make test-e2e`) +- There is a stamp-based infra model under `.stamps/cluster//...` and `.stamps/image/...`. +- The full-suite Go code already shells out to Make in `BeforeSuite` (currently to ensure port-forwards). +- `Makefile` contains WIP placeholders: + - `$(CS)/$(NAMESPACE)/e2e/prepare:` exists but has no recipe yet. + - `test-e2e` is currently wired to only run `cleanup-namespace-sut` (it does not run the suite). + - `test-e2e-quickstart-manifest` references `build-plain-manifests-installer` (not defined). + - `$(CS)/portforward.running` is `.PHONY`, so stamps can’t skip restarts. -Current entrypoint: +## Design contract (end-state) -- `test-e2e -> $(CS)/e2e.passed` +### 1) One β€œprepare” target that Go calls -Current dependency chain for full E2E: +Go should call exactly one Make target in `BeforeSuite`: -- `$(CS)/e2e.passed` depends on: - - `install` (default `INSTALL_MODE=config-dir`) - - `$(CS)/controller.deployed` - - `$(CS)/portforward.running` - - `$(CS)/age-key.applied` - - `test/e2e/**/*.go` +- `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp; no `go test` inside) -`$(CS)/controller.deployed` currently: +That stamp is responsible for: -- loads image into Kind (`$(E2E_IMAGE)`) -- deletes validating webhook configuration -- sets image in `config/` via `kustomize edit set image` -- applies `kustomize build config` -- rolls out `deploy/gitops-reverser` in namespace `sut` +- cleaning + recreating the controller namespace (`sut` for now) +- running the selected installer (`INSTALL_MODE=...`) +- ensuring the controller is deployed with the correct image +- ensuring shared test dependencies are ready (port-forwards, age key, etc.) -Important relationship: +Then Go runs tests normally and only owns scenario logic/assertions. -- `controller.deployed` is the cluster-state gate for full suite execution. -- `e2e.passed` is the test-result stamp that runs `go test ./test/e2e`. +### 2) Seed β†’ run namespace (shared by Make + Go) -## Quickstart E2E +Inputs/outputs: -Current entrypoints: +- input: `E2E_GINKGO_RANDOM_SEED` (already used by the suite) +- derived: `E2E_RUN_SEED` (defaults to `E2E_GINKGO_RANDOM_SEED`, fallback to timestamp) +- derived: `E2E_RUN_NS=run-$(E2E_RUN_SEED)` -- `make test-e2e-quickstart-helm` -- `make test-e2e-quickstart-manifest` - -These flows: - -- always cleanup dedicated Kind cluster first -- install cert-manager + gitea via shared stamps -- load image for local runs -- execute `test/e2e/scripts/run-quickstart.sh` in `helm` or `manifest` mode - -CI alignment is already strong: - -- `.github/workflows/ci.yml` runs: - - `make test-e2e` - - matrix: `make test-e2e-quickstart-helm|manifest` - -## Seed and Namespace Today - -What exists today: - -- Ginkgo seed is exported to `make` as `E2E_GINKGO_RANDOM_SEED`. -- quickstart framework installer namespace uses `run-`. +The full suite should start using `E2E_RUN_NS` for test-created resources so concurrent runs don’t collide. +Keep `sut` as the controller namespace initially (parameterize later). -What is missing: +### 3) Makefile matrix: suite Γ— install-method -- full E2E still uses fixed namespace constants (`sut`) in Go helpers/tests. -- Makefile does not derive namespace/build stamp identity from seed. -- no run-scoped namespace stamp used by full E2E test resources. +Keep the matrix intentionally small: -## Gaps Against Target Vision +- `suite=full` with `INSTALL_MODE=config-dir` (internal/dev install path) +- `suite=quickstart` with `INSTALL_MODE=helm|plain-manifests-file` (user-facing install paths) -1. Seed is propagated but not structurally used for run scoping. -2. Suite x install-method model is partial: - - quickstart: `helm`, `manifest` - - full: effectively `config-dir` + extra deploy path - - `config-dir` is not clearly positioned as either product path or internal test path. -3. Quickstart Go framework lacks parity with `run-quickstart.sh` assertions (commit progression, encrypted secret checks, invalid-credential message checks). -4. Full E2E still contains significant infra-level shell/Kubernetes orchestration in Go (`setup-gitea.sh`, many direct `kubectl`). -5. Local quickstart manifest target has a defect: - - `test-e2e-quickstart-manifest` calls `build-plain-manifests-installer`, but no such target exists. +Stable entrypoints: -## Proposed End-State Model - -## Test matrix - -Define explicit, stable matrix naming: - -- `full-config-dir` (current full suite path) -- `quickstart-helm` -- `quickstart-manifest` - -Optional future (only if needed): - -- `full-helm` -- `full-manifest` - -## Seed contract +- `make test-e2e` (alias for full + config-dir) +- `make test-e2e-quickstart-helm` +- `make test-e2e-quickstart-manifest` -Single run contract shared by Make and Go: +Optional later (only if it proves valuable): `suite=full` with `INSTALL_MODE=helm|plain-manifests-file`. -- `E2E_RUN_SEED` (defaults to ginkgo seed when available) -- `E2E_RUN_NS=run-$(E2E_RUN_SEED)` -- `E2E_CONTROLLER_NS` (keep `sut` initially, parameterize later) +## Implementation plan (small PRs with clear outcomes) -## Ownership split +### PR1 β€” Fix-first correctness (unblocks everything) -- Makefile owns cluster infra and deployment/install orchestration. -- Go tests own scenario logic and assertions. -- Go should not perform install/deploy internals directly. +- Convert the `// ...` β€œnotes” in `Makefile` into Make comments (`# ...`) so they are not treated as recipe lines. +- Re-wire `make test-e2e` to actually run the full E2E suite again (and keep the alias stable). +- Fix `test-e2e-quickstart-manifest` to depend on `dist/install.yaml` (or rename/add the intended target). +- Make `$(CS)/portforward.running` a real stamp target (remove `.PHONY`) and add a health-check fast-path. -## Phased Plan +Acceptance: -## Phase 0: Stabilize current paths (small PR) +- `make test-e2e` runs the suite end-to-end on a prepared cluster. +- Quickstart helm+manifest targets work locally without undefined targets. -1. Fix missing target reference in `test-e2e-quickstart-manifest`: - - replace `build-plain-manifests-installer` with existing `dist/install.yaml` dependency path. -2. Remove `.PHONY` from `$(CS)/portforward.running` so stamp behavior is real. -3. Add/refresh Makefile overview doc (this file) and link it from `docs/make-e2e-deps.md`. +### PR2 β€” Implement `e2e/prepare` as the single orchestration contract -## Phase 1: Introduce run seed + run namespace primitives +- Implement `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp) as the β€œone target Go calls”. +- Update Go `BeforeSuite` to call `e2e/prepare` (instead of calling `portforward.running` directly). -1. Add Make variables: - - `E2E_RUN_SEED ?= $(if $(E2E_GINKGO_RANDOM_SEED),$(E2E_GINKGO_RANDOM_SEED),$(shell date +%s))` - - `E2E_RUN_NS ?= run-$(E2E_RUN_SEED)` -2. Add run namespace stamp: - - `$(CS)/run-ns.ready` applying/labeling `$(E2E_RUN_NS)`. -3. Pass `E2E_RUN_SEED`, `E2E_RUN_NS`, `E2E_CONTROLLER_NS` into `go test` env. -4. In Go tests, replace hardcoded test namespace constants with env-backed helper. +Acceptance: -## Phase 2: Explicit suite x install-method targets +- `go test ./test/e2e/...` works when invoked from CI/Make and always provisions infra via `e2e/prepare`. -1. Introduce clear top-level targets: - - `test-e2e-full-config-dir` - - `test-e2e-quickstart-helm` - - `test-e2e-quickstart-manifest` -2. Keep `test-e2e` as alias to `test-e2e-full-config-dir` for compatibility. -3. Add `test-e2e-ci` as local equivalent of CI flow: - - full + quickstart-helm + quickstart-manifest. +### PR3 β€” Run scoping via seed -## Phase 3: Port `run-quickstart.sh` assertions into Go +- Add `E2E_RUN_SEED` + `E2E_RUN_NS` to Make. +- Add a run-namespace stamp and propagate env vars into `go test`. +- Update Go helpers/tests to use `E2E_RUN_NS` for test-created resources. -Port these checks into `test/e2e/quickstart_framework_e2e_test.go`: +Acceptance: -1. resource readiness (`GitProvider`, `GitTarget`, `WatchRule`) -2. generated SOPS secret exists + backup-warning + age-recipient annotation -3. create/update/delete ConfigMap produces expected repo file transitions and commit count increments -4. Secret commit is encrypted (no plaintext/base64 leakage) -5. encrypted file decrypts with generated age key and shows updated value -6. invalid credentials produce actionable `ConnectionFailed` status message +- Two runs with different seeds can share a cluster without namespace collisions. -Then reduce `run-quickstart.sh` to thin install/bootstrap wrapper or retire it. +### PR4 β€” Reduce shell quickstart assertions (Go parity) -## Phase 4: Minimize infra/Kubernetes orchestration in Go +- Port `run-quickstart.sh` assertions into Go (commit progression, encryption checks, invalid-credential status). +- Keep the shell script as a thin bootstrap wrapper or retire it once parity is reached. -1. Move remaining setup operations behind Make targets/stamps. -2. Keep Go focused on test scenario inputs + assertions. -3. Add one convenience target for developers (candidate name): - - `make last-deployment` or `make e2e-prepare` to provision infra without running tests. +Acceptance: -## Recommended Immediate Next PR +- Quickstart smoke tests’ β€œreal assertions” live in Go, not shell. -1. Apply Phase 0 fixes. -2. Start Phase 3 by porting one vertical slice in Go: - - ConfigMap create/update/delete + repo file + commit assertions. -3. Keep CI job structure unchanged while moving logic from shell to Go. +## Notes / non-goals (for now) -This gives fast signal improvement without destabilizing the current pipeline. +- Don’t expand the suite/install matrix until the `e2e/prepare` contract and run scoping are solid. +- Avoid moving Kubernetes orchestration into Go; prefer Make stamps and verified gates. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 908d020fa..d007ca025 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -45,26 +45,37 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { if img := os.Getenv("PROJECT_IMAGE"); img == "" { - // In local runs, the Makefile guarantees the cluster and image are ready - // before go test is invoked. Nothing to do here. - By("local run: cluster and image prepared by Makefile") + By("local run: preparing cluster via Makefile target") } else { By(fmt.Sprintf("using pre-built image: %s", img)) } - By("ensuring shared e2e port-forwards are active via Makefile target") - ensureE2EPortForwards() + By("preparing e2e prerequisites via Makefile target") + ensureE2EPrepared() }) var _ = AfterSuite(func() { }) -func ensureE2EPortForwards() { +func ensureE2EPrepared() { ctx := resolveE2EContext() - target := fmt.Sprintf(".stamps/cluster/%s/portforward.running", ctx) - cmd := makeCommandWithSeed(fmt.Sprintf("CTX=%s", ctx), target) + ns := resolveE2ENamespace() + target := fmt.Sprintf(".stamps/cluster/%s/%s/e2e/prepare", ctx, ns) + cmd := makeCommandWithSeed( + fmt.Sprintf("CTX=%s", ctx), + fmt.Sprintf("NAMESPACE=%s", ns), + target, + "portforward-ensure", + ) output, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "failed to run make target for port-forwards") + Expect(err).NotTo(HaveOccurred(), "failed to run make target for e2e prepare") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + + // Many e2e helpers invoke kubectl without an explicit --context flag. Ensure kubectl is pointed at the + // intended cluster context for the remainder of the test run. + useCtx := exec.Command("kubectl", "config", "use-context", ctx) + output, err = utils.Run(useCtx) + Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) } @@ -90,6 +101,13 @@ func makeCommandWithSeed(args ...string) *exec.Cmd { return cmd } +func resolveE2ENamespace() string { + if value := strings.TrimSpace(os.Getenv("NAMESPACE")); value != "" { + return value + } + return "gitops-reverser" // does it wrok? +} + func ginkgoRandomSeed() int64 { suiteConfig, _ := GinkgoConfiguration() return suiteConfig.RandomSeed diff --git a/test/e2e/scripts/ensure-prometheus-operator.sh b/test/e2e/scripts/ensure-prometheus-operator.sh index a94b5391e..819cd1d52 100755 --- a/test/e2e/scripts/ensure-prometheus-operator.sh +++ b/test/e2e/scripts/ensure-prometheus-operator.sh @@ -3,11 +3,21 @@ set -euo pipefail PROM_OPERATOR_VERSION="${PROM_OPERATOR_VERSION:-0.89.0}" PROM_OPERATOR_NAMESPACE="${PROM_OPERATOR_NAMESPACE:-prometheus-operator}" +KUBE_CONTEXT="${KUBECONTEXT:-${CTX:-}}" + +if [[ -z "${KUBE_CONTEXT}" ]]; then + KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" +fi + +if [[ -z "${KUBE_CONTEXT}" ]]; then + echo "❌ Kubernetes context is required (set KUBECONTEXT or CTX)" + exit 1 +fi is_fully_installed() { - kubectl get deployment prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" >/dev/null 2>&1 && - kubectl get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1 && - kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 + kubectl --context "${KUBE_CONTEXT}" get deployment prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" >/dev/null 2>&1 && + kubectl --context "${KUBE_CONTEXT}" get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1 && + kubectl --context "${KUBE_CONTEXT}" get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 } if is_fully_installed; then @@ -16,7 +26,8 @@ if is_fully_installed; then fi echo "Installing Prometheus Operator v${PROM_OPERATOR_VERSION} in namespace ${PROM_OPERATOR_NAMESPACE}..." -kubectl create namespace "${PROM_OPERATOR_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl --context "${KUBE_CONTEXT}" create namespace "${PROM_OPERATOR_NAMESPACE}" --dry-run=client -o yaml | \ + kubectl --context "${KUBE_CONTEXT}" apply -f - tmp_dir="$(mktemp -d)" cleanup() { @@ -32,12 +43,13 @@ EOF ( cd "${tmp_dir}" NAMESPACE="${PROM_OPERATOR_NAMESPACE}" kustomize edit set namespace "${PROM_OPERATOR_NAMESPACE}" - if ! kubectl create -k .; then + if ! kubectl --context "${KUBE_CONTEXT}" create -k .; then echo "kubectl create -k failed (likely already exists), reconciling with server-side apply ..." - kubectl apply --server-side -k . + kubectl --context "${KUBE_CONTEXT}" apply --server-side -k . fi ) -kubectl rollout status deployment/prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" --timeout=180s +kubectl --context "${KUBE_CONTEXT}" rollout status deployment/prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" \ + --timeout=180s echo "βœ… Prometheus Operator is ready" From 6c99fd212fe0cfe51046b51ea3798656286ab2aa Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 09:09:04 +0000 Subject: [PATCH 03/30] ci: Small steps towards the end of this Makefile ride --- Makefile | 3 +- docs/make-e2e-deps.md | 35 ++++++++-------- docs/makefile-e2e-overview-and-plan.md | 51 +++++++++++++++-------- test/e2e/e2e_suite_test.go | 50 ++++++++++++---------- test/e2e/quickstart_framework_e2e_test.go | 2 +- 5 files changed, 82 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 585711a5d..a68b62696 100644 --- a/Makefile +++ b/Makefile @@ -369,7 +369,8 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh -$(CS)/e2e.passed: $(CS)/$(NAMESPACE)/e2e/prepare $(shell find test/e2e -name '*.go') +E2E_TEST_INPUTS := $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) +$(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index 125f29c7d..c68914ef4 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -83,25 +83,26 @@ make CTX=kind-my-other-cluster .stamps/cluster/kind-my-other-cluster/cert-manage ``` make test-e2e (phony) └─ $(CS)/e2e.passed - β”œβ”€ test/e2e/**/*.go (any test change reruns tests) - └─ $(CS)/portforward.running - β”œβ”€ $(CS)/controller.deployed - β”‚ β”œβ”€ $(CS)/crds.applied - β”‚ β”‚ └─ $(CS)/ready - β”‚ β”‚ └─ test/e2e/kind/start-cluster.sh - β”‚ β”‚ test/e2e/kind/cluster-template.yaml - β”‚ β”œβ”€ $(CS)/cert-manager.installed - β”‚ β”‚ └─ $(CS)/ready - β”‚ β”œβ”€ $(CS)/prometheus.installed - β”‚ β”‚ └─ $(CS)/ready - β”‚ β”œβ”€ $(IS)/controller.id - β”‚ β”‚ └─ cmd/**/*.go internal/**/*.go go.mod go.sum Dockerfile - β”‚ └─ config/** (excluding config/crd/*) - β”œβ”€ $(CS)/gitea.installed - β”‚ └─ $(CS)/ready - └─ $(CS)/prometheus.installed + β”œβ”€ test/e2e/** (any test change reruns tests) + └─ go test ./test/e2e + └─ BeforeSuite calls Make: + make CTX= INSTALL_NAME= NAMESPACE= \ + $(CS)/$(NAMESPACE)/e2e/prepare portforward-ensure + +$(CS)/$(NAMESPACE)/e2e/prepare (stamp) + β”œβ”€ $(CS)/sut.namespace.cleaned + β”œβ”€ $(CS)/controller.deployed + β”œβ”€ $(CS)/portforward.running + └─ $(CS)/age-key.applied ``` +Notes: + +- The Makefile stays seed-agnostic. The Go suite derives seed-suffixed `INSTALL_NAME` and `NAMESPACE` and passes + them into Make so install-time cluster-scoped names can be unique per run. +- `portforward-ensure` always runs a health check + (re)starts port-forwards as needed for the current test + process. The stamp target `$(CS)/portforward.running` is still used as a cached gate for other Make targets. + What this buys you in practice: | Change you make | Steps that actually rerun | diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md index 39ade537d..6d5d084d6 100644 --- a/docs/makefile-e2e-overview-and-plan.md +++ b/docs/makefile-e2e-overview-and-plan.md @@ -10,15 +10,15 @@ This is the high-level roadmap for the Makefile-driven E2E work. For the detaile - Make the E2E model explicit as a **matrix**: `suite` Γ— `install-method`. - Keep Go focused on scenario logic/assertions; Make owns cluster orchestration (Kind/Helm/kubectl/image/stamps). -## Current reality (as of Feb 26, 2026) +## Current reality (as of Feb 27, 2026) - There is a stamp-based infra model under `.stamps/cluster//...` and `.stamps/image/...`. -- The full-suite Go code already shells out to Make in `BeforeSuite` (currently to ensure port-forwards). -- `Makefile` contains WIP placeholders: - - `$(CS)/$(NAMESPACE)/e2e/prepare:` exists but has no recipe yet. - - `test-e2e` is currently wired to only run `cleanup-namespace-sut` (it does not run the suite). - - `test-e2e-quickstart-manifest` references `build-plain-manifests-installer` (not defined). - - `$(CS)/portforward.running` is `.PHONY`, so stamps can’t skip restarts. +- The full-suite Go code shells out to Make in `BeforeSuite`: + - runs `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp; no tests inside) + - runs `portforward-ensure` to guarantee live port-forwards for the test process + - sets `kubectl config use-context $(CTX)` for helpers that don’t pass `--context` +- `make test-e2e` runs `go test ./test/e2e/...` and the Go suite owns infra prep via the Make targets above. +- Quickstart manifest uses `dist/install.yaml` (no missing Make targets). ## Design contract (end-state) @@ -41,12 +41,13 @@ Then Go runs tests normally and only owns scenario logic/assertions. Inputs/outputs: -- input: `E2E_GINKGO_RANDOM_SEED` (already used by the suite) -- derived: `E2E_RUN_SEED` (defaults to `E2E_GINKGO_RANDOM_SEED`, fallback to timestamp) -- derived: `E2E_RUN_NS=run-$(E2E_RUN_SEED)` +- input: Ginkgo random seed (internal to the Go suite) +- output: seed-suffixed `INSTALL_NAME` + `NAMESPACE` passed into Make from Go + - rationale: the install name must include the seed because quickstart installs include cluster-scoped resources + that would otherwise collide by name -The full suite should start using `E2E_RUN_NS` for test-created resources so concurrent runs don’t collide. -Keep `sut` as the controller namespace initially (parameterize later). +The full suite should start using the seed-suffixed run namespace for test-created resources so concurrent runs +don’t collide. Keep `sut` as the controller namespace initially (parameterize later). ### 3) Makefile matrix: suite Γ— install-method @@ -65,7 +66,7 @@ Optional later (only if it proves valuable): `suite=full` with `INSTALL_MODE=hel ## Implementation plan (small PRs with clear outcomes) -### PR1 β€” Fix-first correctness (unblocks everything) +### PR1 β€” Fix-first correctness (done) - Convert the `// ...` β€œnotes” in `Makefile` into Make comments (`# ...`) so they are not treated as recipe lines. - Re-wire `make test-e2e` to actually run the full E2E suite again (and keep the alias stable). @@ -77,10 +78,10 @@ Acceptance: - `make test-e2e` runs the suite end-to-end on a prepared cluster. - Quickstart helm+manifest targets work locally without undefined targets. -### PR2 β€” Implement `e2e/prepare` as the single orchestration contract +### PR2 β€” Implement `e2e/prepare` as the single orchestration contract (done) - Implement `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp) as the β€œone target Go calls”. -- Update Go `BeforeSuite` to call `e2e/prepare` (instead of calling `portforward.running` directly). +- Update Go `BeforeSuite` to call `e2e/prepare` (and `portforward-ensure`) and set kubectl context. Acceptance: @@ -88,9 +89,12 @@ Acceptance: ### PR3 β€” Run scoping via seed -- Add `E2E_RUN_SEED` + `E2E_RUN_NS` to Make. -- Add a run-namespace stamp and propagate env vars into `go test`. -- Update Go helpers/tests to use `E2E_RUN_NS` for test-created resources. +- Use the Go suite’s seed as the single run identifier and pass only: + - `INSTALL_NAME=-` + - `NAMESPACE=-` +- Decide which resources should be seed-scoped: + - quickstart installs: must be seed-scoped (cluster-scoped resource name collisions) + - full suite: start with seed-scoped stamps only, then migrate test-created resources next Acceptance: @@ -109,3 +113,14 @@ Acceptance: - Don’t expand the suite/install matrix until the `e2e/prepare` contract and run scoping are solid. - Avoid moving Kubernetes orchestration into Go; prefer Make stamps and verified gates. + +## Why `manifests` and `helm-sync` are `.PHONY` + +They are command-style β€œentrypoint” targets, not real files. + +- They don’t produce a file named `manifests` or `helm-sync`; the real outputs are file targets like + `config/crd/bases/*.yaml` and `charts/gitops-reverser/crds/*.yaml`. +- Marking them `.PHONY` avoids accidental β€œup to date” behavior if a file named `manifests` or `helm-sync` + exists in the repo (or gets created by tooling). +- Incremental behavior still comes from the file targets: `.PHONY` only forces Make to *check* the graph when + you run `make manifests` / `make helm-sync`. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d007ca025..62f6011c3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "testing" @@ -31,8 +32,6 @@ import ( "github.com/ConfigButler/gitops-reverser/test/utils" ) -const e2eGinkgoSeedEnv = "E2E_GINKGO_RANDOM_SEED" - // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, // temporary environment to validate project changes with the purpose of being used in CI jobs. // The default setup requires Kind, builds/loads the Manager Docker image locally, and installs @@ -59,10 +58,13 @@ var _ = AfterSuite(func() { func ensureE2EPrepared() { ctx := resolveE2EContext() - ns := resolveE2ENamespace() + seed := ginkgoRandomSeed() + ns := resolveE2EStampNamespace(seed) + installName := resolveE2EInstallName(seed) target := fmt.Sprintf(".stamps/cluster/%s/%s/e2e/prepare", ctx, ns) - cmd := makeCommandWithSeed( + cmd := makeCommand( fmt.Sprintf("CTX=%s", ctx), + fmt.Sprintf("INSTALL_NAME=%s", installName), fmt.Sprintf("NAMESPACE=%s", ns), target, "portforward-ensure", @@ -77,35 +79,39 @@ func ensureE2EPrepared() { output, err = utils.Run(useCtx) Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) -} -func makeCommandWithSeed(args ...string) *exec.Cmd { - seed := ginkgoRandomSeed() - seedPrefix := e2eGinkgoSeedEnv + "=" - filteredArgs := make([]string, 0, len(args)+1) - for _, arg := range args { - if strings.HasPrefix(arg, seedPrefix) { - continue - } - filteredArgs = append(filteredArgs, arg) + // The Makefile prepares the age key under the stamp directory. When running `go test` directly (without + // `make test-e2e`), ensure the suite uses that prepared key file. + if strings.TrimSpace(os.Getenv("E2E_AGE_KEY_FILE")) == "" { + wd, err := os.Getwd() + Expect(err).NotTo(HaveOccurred(), "failed to get working directory for e2e run") + ageKeyPath := filepath.Join(wd, ".stamps", "cluster", ctx, "age-key.txt") + Expect(os.Setenv("E2E_AGE_KEY_FILE", ageKeyPath)).To(Succeed()) } - filteredArgs = append(filteredArgs, fmt.Sprintf("%s=%d", e2eGinkgoSeedEnv, seed)) - cmd := exec.Command("make", filteredArgs...) +} + +func makeCommand(args ...string) *exec.Cmd { + cmd := exec.Command("make", args...) _, _ = fmt.Fprintf( GinkgoWriter, - "make invocation: seed=%d arg=%s args=%v\n", - seed, - fmt.Sprintf("%s=%d", e2eGinkgoSeedEnv, seed), - filteredArgs, + "make invocation: args=%v\n", + args, ) return cmd } -func resolveE2ENamespace() string { +func resolveE2EInstallName(seed int64) string { + if value := strings.TrimSpace(os.Getenv("INSTALL_NAME")); value != "" { + return value + } + return fmt.Sprintf("gitops-reverser-%d", seed) +} + +func resolveE2EStampNamespace(seed int64) string { if value := strings.TrimSpace(os.Getenv("NAMESPACE")); value != "" { return value } - return "gitops-reverser" // does it wrok? + return fmt.Sprintf("gitops-reverser-%d", seed) } func ginkgoRandomSeed() int64 { diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index b862b6655..abb43590a 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -176,7 +176,7 @@ func (r *quickstartFrameworkRun) installCommand(mode string) *exec.Cmd { fmt.Sprintf("INSTALL_NAMESPACE=%s", r.installNamespace), fmt.Sprintf("INSTALL_NAME=%s", r.installNamespace), } - return makeCommandWithSeed(args...) + return makeCommand(args...) } func (r *quickstartFrameworkRun) waitForControllerRollout() { From 353a56e197f10c745c4d1e4832505c51b538bc98 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 09:26:52 +0000 Subject: [PATCH 04/30] ci: Would this help? --- Makefile | 1 - test/e2e/e2e_suite_test.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a68b62696..6bc10cdb7 100644 --- a/Makefile +++ b/Makefile @@ -372,7 +372,6 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p E2E_TEST_INPUTS := $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) - kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 62f6011c3..ea47685d5 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -80,6 +80,17 @@ func ensureE2EPrepared() { Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + By("ensuring IceCreamOrder CRD is removed before tests") + deleteIceCreamOrderCRD := exec.Command( + "kubectl", + "--context", ctx, + "delete", "crd", "icecreamorders.shop.example.com", + "--ignore-not-found=true", + ) + output, err = utils.Run(deleteIceCreamOrderCRD) + Expect(err).NotTo(HaveOccurred(), "failed to delete IceCreamOrder CRD before tests") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + // The Makefile prepares the age key under the stamp directory. When running `go test` directly (without // `make test-e2e`), ensure the suite uses that prepared key file. if strings.TrimSpace(os.Getenv("E2E_AGE_KEY_FILE")) == "" { From 22420fd3e0555386e6e2ada6cc6eded2d877bafb Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 10:27:00 +0000 Subject: [PATCH 05/30] ci: Now we can run in any namespace --- Makefile | 59 +++++++++++------------ config/kustomization.yaml | 47 ++++++++++++++++++ config/namespace.yaml | 1 + test/e2e/E2E_DEBUGGING.md | 4 +- test/e2e/e2e_suite_test.go | 9 +--- test/e2e/helpers.go | 8 +-- test/e2e/namespace.go | 33 +++++++++++++ test/e2e/quickstart_framework_e2e_test.go | 19 ++++++-- test/e2e/scripts/run-quickstart.sh | 6 +-- test/e2e/scripts/setup-gitea.sh | 2 +- 10 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 test/e2e/namespace.go diff --git a/Makefile b/Makefile index 6bc10cdb7..dad6eea4f 100644 --- a/Makefile +++ b/Makefile @@ -141,26 +141,25 @@ cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps rm -rf .stamps/cluster/kind-$$cluster; \ done -cleanup-namespace-sut: $(CS)/sut.namespace.ready - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true - -$(CS)/sut.namespace.cleaned: $(CS)/sut.namespace.ready +$(CS)/$(NAMESPACE)/namespace.cleaned: Makefile $(CS)/$(NAMESPACE)/namespace.ready + kubectl --context $(CTX) delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true kubectl --context $(CTX) delete validatingwebhookconfiguration \ gitops-reverser-validating-webhook-configuration --ignore-not-found=true touch $@ -$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml - mkdir -p $(CS) - kubectl --context $(CTX) apply -f config/namespace.yaml - kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted +$(CS)/$(NAMESPACE)/namespace.ready: Makefile $(CS)/ready + mkdir -p $(@D) + kubectl --context $(CTX) create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl --context $(CTX) apply -f - + kubectl --context $(CTX) label --overwrite ns $(NAMESPACE) \ + app.kubernetes.io/managed-by=kustomize app.kubernetes.io/name=gitops-reverser control-plane=gitops-reverser \ + pod-security.kubernetes.io/enforce=restricted touch $@ # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/sut.namespace.cleaned $(CS)/controller.deployed $(CS)/portforward.running \ - $(CS)/age-key.applied +$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/$(NAMESPACE)/namespace.cleaned $(CS)/$(NAMESPACE)/controller.deployed \ + $(CS)/portforward.running $(CS)/$(NAMESPACE)/age-key.applied mkdir -p $(@D) touch $@ @@ -265,7 +264,7 @@ $(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml kubectl --context $(CTX) get ns >/dev/null touch $@ -CERT_MANAGER_WAIT_TIMEOUT ?= 600s +CERT_MANAGER_WAIT_TIMEOUT ?= 300s CERT_MANAGER_VERSION ?= v1.19.4 CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml $(CS)/cert-manager.installed: $(CS)/ready @@ -274,7 +273,7 @@ $(CS)/cert-manager.installed: $(CS)/ready kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-webhook --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - touch $@ + $(CERT_MANAGER_VERSION) > $@ $(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml mkdir -p $(CS) @@ -302,12 +301,12 @@ $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-opera kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ -$(CS)/age-key.applied: $(CS)/sut.namespace.ready test/e2e/tools/gen-age-key/main.go - mkdir -p $(CS) +$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/namespace.ready test/e2e/tools/gen-age-key/main.go + mkdir -p $(@D) go run ./test/e2e/tools/gen-age-key \ --key-file $(CS)/age-key.txt \ --secret-file $(CS)/age-key-secret.yaml \ - --namespace sut \ + --namespace $(NAMESPACE) \ --secret-name sops-age-key kubectl --context $(CTX) apply -f $(CS)/age-key-secret.yaml touch $@ @@ -335,28 +334,29 @@ $(CS)/image.loaded: $(IS)/controller.id $(CS)/ready # When PROJECT_IMAGE is set (CI), skip the local image build stamp. ifeq ($(PROJECT_IMAGE),) DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ - $(CS)/sut.namespace.ready \ + $(CS)/$(NAMESPACE)/namespace.ready \ $(IS)/controller.id \ $(shell find config -type f -not -path 'config/crd/*') else DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ - $(CS)/sut.namespace.ready \ + $(CS)/$(NAMESPACE)/namespace.ready \ $(shell find config -type f -not -path 'config/crd/*') endif -$(CS)/controller.deployed: $(DEPLOY_INPUTS) - mkdir -p $(CS) +$(CS)/$(NAMESPACE)/controller.deployed: Makefile $(DEPLOY_INPUTS) + mkdir -p $(@D) @[ -z "$(PROJECT_IMAGE)" ] || $(CONTAINER_TOOL) pull $(E2E_IMAGE) $(KIND) load docker-image $(E2E_IMAGE) --name $(CLUSTER_FROM_CTX) + cd config && $(KUSTOMIZE) edit set namespace $(NAMESPACE) cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_IMAGE) $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - @if [ -z "$(PROJECT_IMAGE)" ]; then \ IMAGE_ID="$$(cat $(IS)/controller.id)"; \ - kubectl --context $(CTX) -n sut patch deploy/gitops-reverser --type=merge \ + kubectl --context $(CTX) -n $(NAMESPACE) patch deploy/gitops-reverser --type=merge \ -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"configbutler.ai/e2e-image-id\":\"$$IMAGE_ID\"}}}}}"; \ - kubectl --context $(CTX) -n sut patch deploy/gitops-reverser --type=json \ + kubectl --context $(CTX) -n $(NAMESPACE) patch deploy/gitops-reverser --type=json \ -p '[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]'; \ fi - kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s + kubectl --context $(CTX) -n $(NAMESPACE) rollout status deploy/gitops-reverser --timeout=180s touch $@ $(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed @@ -372,7 +372,7 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p E2E_TEST_INPUTS := $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) - CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ + CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) NAMESPACE=$(NAMESPACE) \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ @@ -430,9 +430,11 @@ $(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.install touch $@ $(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy controller to the K8s cluster specified in ~/.kube/config. +$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed \ + $(CS)/$(NAMESPACE)/namespace.ready ## Deploy controller to the K8s cluster specified in ~/.kube/config. @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) + (cd config && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)") (cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG}) (cd config && $(KUSTOMIZE) build .) | $(KUBECTL) --context $(CTX) apply -f - touch $@ @@ -485,10 +487,3 @@ cleanup-port-forwards: ## Stop all port-forwards @-pkill -f "kubectl.*port-forward.*13000" 2>/dev/null || true @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true @echo "βœ… Port-forwards stopped" - -.PHONY: cleanup-gitea-e2e -cleanup-gitea-e2e: cleanup-port-forwards ## Clean up Gitea e2e environment - @echo "🧹 Cleaning up Gitea e2e environment..." - @$(HELM) uninstall gitea --namespace $(GITEA_NAMESPACE) 2>/dev/null || true - @$(KUBECTL) delete namespace $(GITEA_NAMESPACE) 2>/dev/null || true - @echo "βœ… Gitea cleanup completed" diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 570f21c28..1ff22d995 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -12,3 +12,50 @@ images: - name: gitops-reverser newName: gitops-reverser newTag: e2e-local +replacements: +- source: + fieldPath: metadata.namespace + kind: ServiceAccount + name: gitops-reverser + targets: + - fieldPaths: + - metadata.name + select: + kind: Namespace + name: sut + - fieldPaths: + - subjects.0.namespace + select: + kind: ClusterRoleBinding + name: gitops-reverser + - fieldPaths: + - webhooks.0.clientConfig.service.namespace + select: + kind: ValidatingWebhookConfiguration + name: gitops-reverser-validating-webhook-configuration + - fieldPaths: + - metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: / + select: + kind: ValidatingWebhookConfiguration + name: gitops-reverser-validating-webhook-configuration + - fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: . + index: 1 + select: + kind: Certificate + name: gitops-reverser-admission-server-cert + - fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: . + index: 1 + select: + kind: Certificate + name: gitops-reverser-audit-server-cert +namespace: gitops-reverser diff --git a/config/namespace.yaml b/config/namespace.yaml index a79738084..4dd1788cc 100644 --- a/config/namespace.yaml +++ b/config/namespace.yaml @@ -5,4 +5,5 @@ metadata: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: gitops-reverser control-plane: gitops-reverser + pod-security.kubernetes.io/enforce: restricted name: sut diff --git a/test/e2e/E2E_DEBUGGING.md b/test/e2e/E2E_DEBUGGING.md index baff12228..4bf78ef97 100644 --- a/test/e2e/E2E_DEBUGGING.md +++ b/test/e2e/E2E_DEBUGGING.md @@ -77,7 +77,7 @@ Kind Cluster β”‚ └─ Prometheus Operator + Prometheus (scrapes metrics via ServiceMonitor) β”œβ”€ gitea-e2e namespace β”‚ └─ Gitea (Git server) - └─ sut namespace (System Under Test) + └─ controller namespace (System Under Test; set via `NAMESPACE`) └─ Controller pods (2 replicas, HTTPS metrics) ``` @@ -101,7 +101,7 @@ Kind Cluster 4. **View controller logs:** ```bash - kubectl logs -n sut -l control-plane=gitops-reverser --tail=100 + kubectl logs -n "${NAMESPACE}" -l control-plane=gitops-reverser --tail=100 ``` 5. **Check Prometheus scrape status:** diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ea47685d5..d158f5a11 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -59,7 +59,7 @@ var _ = AfterSuite(func() { func ensureE2EPrepared() { ctx := resolveE2EContext() seed := ginkgoRandomSeed() - ns := resolveE2EStampNamespace(seed) + ns := resolveE2ENamespace() installName := resolveE2EInstallName(seed) target := fmt.Sprintf(".stamps/cluster/%s/%s/e2e/prepare", ctx, ns) cmd := makeCommand( @@ -118,13 +118,6 @@ func resolveE2EInstallName(seed int64) string { return fmt.Sprintf("gitops-reverser-%d", seed) } -func resolveE2EStampNamespace(seed int64) string { - if value := strings.TrimSpace(os.Getenv("NAMESPACE")); value != "" { - return value - } - return fmt.Sprintf("gitops-reverser-%d", seed) -} - func ginkgoRandomSeed() int64 { suiteConfig, _ := GinkgoConfiguration() return suiteConfig.RandomSeed diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index ac1ecb1e2..a66289842 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -41,7 +41,7 @@ import ( ) // namespace where the project is deployed in. -const namespace = "sut" +var namespace = resolveE2ENamespace() //nolint:gochecknoglobals // used across e2e tests const metricWaitDefaultTimeout = 30 * time.Second const e2eEncryptionRefName = "sops-age-key" @@ -175,8 +175,6 @@ func getBaseFolder() string { } // createGitTarget creates a GitTarget that binds a GitProvider, branch and path. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity in template calls func createGitTarget(name, namespace, providerName, path, branch string) { createGitTargetWithEncryptionOptions( name, @@ -225,8 +223,6 @@ func createGitTargetWithEncryptionOptions( } // cleanupGitTarget deletes a GitTarget resource. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity func cleanupGitTarget(name, namespace string) { By(fmt.Sprintf("cleaning up GitTarget '%s' in ns '%s'", name, namespace)) ctx := context.Background() @@ -236,8 +232,6 @@ func cleanupGitTarget(name, namespace string) { } // cleanupWatchRule deletes a WatchRule resource. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity func cleanupWatchRule(name, namespace string) { By(fmt.Sprintf("cleaning up WatchRule '%s' in ns '%s'", name, namespace)) ctx := context.Background() diff --git a/test/e2e/namespace.go b/test/e2e/namespace.go new file mode 100644 index 000000000..145e0b347 --- /dev/null +++ b/test/e2e/namespace.go @@ -0,0 +1,33 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os" + "strings" +) + +const defaultE2ENamespace = "sut" + +func resolveE2ENamespace() string { + if value := strings.TrimSpace(os.Getenv("NAMESPACE")); value != "" { + return value + } + return defaultE2ENamespace +} diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index abb43590a..d2e37a9c0 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -36,7 +36,7 @@ import ( const ( quickstartFrameworkEnabledEnv = "E2E_ENABLE_QUICKSTART_FRAMEWORK" quickstartFrameworkModeEnv = "E2E_QUICKSTART_MODE" - quickstartSetupNamespace = "sut" + quickstartSetupNamespaceEnv = "QUICKSTART_NAMESPACE" ) type quickstartFrameworkRun struct { @@ -195,6 +195,7 @@ func quickstartInstallerNamespace() string { func (r *quickstartFrameworkRun) setupGiteaRepository() { cmd := exec.Command("bash", "test/e2e/scripts/setup-gitea.sh", r.repoName, r.checkoutDir) + cmd.Env = append(os.Environ(), fmt.Sprintf("SUT_NAMESPACE=%s", quickstartSetupNamespace())) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "failed to bootstrap gitea repository for quickstart") } @@ -204,7 +205,7 @@ func (r *quickstartFrameworkRun) applyQuickstartResources() { createGitTargetWithEncryptionOptions( r.targetName, - quickstartSetupNamespace, + quickstartSetupNamespace(), r.providerName, "live-cluster", "main", @@ -218,12 +219,22 @@ func (r *quickstartFrameworkRun) applyQuickstartResources() { DestinationName string }{ Name: r.watchRuleName, - Namespace: quickstartSetupNamespace, + Namespace: quickstartSetupNamespace(), DestinationName: r.targetName, } - err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, quickstartSetupNamespace) + err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, quickstartSetupNamespace()) Expect(err).NotTo(HaveOccurred(), "failed to apply quickstart watchrule") createGitProviderWithURL(r.invalidProvName, "main", "git-creds-invalid", r.repoURL) } + +func quickstartSetupNamespace() string { + if value := strings.TrimSpace(os.Getenv(quickstartSetupNamespaceEnv)); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("SUT_NAMESPACE")); value != "" { + return value + } + return "sut" +} diff --git a/test/e2e/scripts/run-quickstart.sh b/test/e2e/scripts/run-quickstart.sh index 1887ec720..befc47a74 100755 --- a/test/e2e/scripts/run-quickstart.sh +++ b/test/e2e/scripts/run-quickstart.sh @@ -2,8 +2,8 @@ set -euo pipefail MODE="${1:-}" -NAMESPACE="gitops-reverser" -QUICKSTART_NAMESPACE="sut" +NAMESPACE="${NAMESPACE:-gitops-reverser}" +QUICKSTART_NAMESPACE="${QUICKSTART_NAMESPACE:-sut}" HELM_CHART_SOURCE="${HELM_CHART_SOURCE:-charts/gitops-reverser}" WAIT_TIMEOUT="${WAIT_TIMEOUT:-60s}" QUICKSTART_TIMEOUT_SECONDS="${QUICKSTART_TIMEOUT_SECONDS:-180}" @@ -416,7 +416,7 @@ run_quickstart_flow() { echo "Setting up quickstart repo and credentials" run_or_debug \ "Creating dedicated Gitea repo for quickstart smoke (${REPO_NAME})" \ - bash test/e2e/scripts/setup-gitea.sh "${REPO_NAME}" "${CHECKOUT_DIR}" + env SUT_NAMESPACE="${QUICKSTART_NAMESPACE}" bash test/e2e/scripts/setup-gitea.sh "${REPO_NAME}" "${CHECKOUT_DIR}" echo "Applying minimal GitProvider/GitTarget/WatchRule quickstart resources" cat < Date: Fri, 27 Feb 2026 14:18:58 +0000 Subject: [PATCH 06/30] fix: Getting the image insertion at least a little bit straight --- Makefile | 167 +++++++++++------- docs/make-e2e-further.md | 2 +- ...file-e2e-image-and-deploy-refactor-plan.md | 159 +++++++++++++++++ docs/makefile-e2e-overview-and-plan.md | 7 + test/e2e/scripts/run-quickstart.sh | 133 +++++--------- 5 files changed, 320 insertions(+), 148 deletions(-) create mode 100644 docs/makefile-e2e-image-and-deploy-refactor-plan.md diff --git a/Makefile b/Makefile index dad6eea4f..3b63f388c 100644 --- a/Makefile +++ b/Makefile @@ -97,8 +97,12 @@ KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-test-e2e-quickstart-manifest # KIND_CLUSTER is used by cleanup-cluster; defaults to the main e2e cluster. KIND_CLUSTER ?= $(KIND_CLUSTER_E2E) E2E_LOCAL_IMAGE ?= gitops-reverser:e2e-local -# In CI, PROJECT_IMAGE is the pre-built image from the docker-build job; locally we build E2E_LOCAL_IMAGE. -E2E_IMAGE := $(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) +# Capture whether PROJECT_IMAGE was provided (CI sets it; local runs typically don't). +PROJECT_IMAGE_PROVIDED ?= $(strip $(PROJECT_IMAGE)) +export PROJECT_IMAGE_PROVIDED +# PROJECT_IMAGE is the single source of truth after this point. +PROJECT_IMAGE ?= $(E2E_LOCAL_IMAGE) +export PROJECT_IMAGE E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. @@ -118,6 +122,7 @@ NAMESPACE ?= gitops-reverser KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) +HELM_CHART_SOURCE ?= charts/gitops-reverser .PHONY: cleanup-cluster cleanup-cluster: ## Tear down the Kind cluster used for e2e tests and remove its stamps @@ -158,8 +163,12 @@ $(CS)/$(NAMESPACE)/namespace.ready: Makefile $(CS)/ready # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/$(NAMESPACE)/namespace.cleaned $(CS)/$(NAMESPACE)/controller.deployed \ - $(CS)/portforward.running $(CS)/$(NAMESPACE)/age-key.applied +$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/$(NAMESPACE)/namespace.cleaned \ + $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ + $(CS)/image.loaded \ + $(CS)/$(NAMESPACE)/controller.deployed \ + $(CS)/portforward.running \ + $(CS)/$(NAMESPACE)/age-key.applied mkdir -p $(@D) touch $@ @@ -273,7 +282,7 @@ $(CS)/cert-manager.installed: $(CS)/ready kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-webhook --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - $(CERT_MANAGER_VERSION) > $@ + echo $(CERT_MANAGER_VERSION) > $@ $(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml mkdir -p $(CS) @@ -319,44 +328,85 @@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile -t $(E2E_LOCAL_IMAGE) . $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(E2E_LOCAL_IMAGE) > $@ +ifeq ($(PROJECT_IMAGE_PROVIDED),) +$(IS)/project-image.ready: $(IS)/controller.id +endif +$(IS)/project-image.ready: Makefile + mkdir -p $(IS) + @set -euo pipefail; \ + if [ -n "$(PROJECT_IMAGE_PROVIDED)" ]; then \ + if ! $(CONTAINER_TOOL) image inspect "$(PROJECT_IMAGE)" >/dev/null 2>&1; then \ + echo "Pulling PROJECT_IMAGE=$(PROJECT_IMAGE)"; \ + $(CONTAINER_TOOL) pull "$(PROJECT_IMAGE)"; \ + fi; \ + fi + touch $@ + $(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) mkdir -p $(CS) kubectl --context $(CTX) apply -k config/crd kubectl --context $(CTX) wait --for=condition=Established crd --all --timeout=120s touch $@ -$(CS)/image.loaded: $(IS)/controller.id $(CS)/ready - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) +$(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready + $(KIND) load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX) touch $@ -# DEPLOY_INPUTS: all config files except CRDs (tracked separately by crds.applied). -# Only relevant for the main e2e cluster (controller.deployed is not used by quickstart targets). -# When PROJECT_IMAGE is set (CI), skip the local image build stamp. -ifeq ($(PROJECT_IMAGE),) -DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ - $(CS)/$(NAMESPACE)/namespace.ready \ - $(IS)/controller.id \ - $(shell find config -type f -not -path 'config/crd/*') -else -DEPLOY_INPUTS := $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ - $(CS)/$(NAMESPACE)/namespace.ready \ - $(shell find config -type f -not -path 'config/crd/*') -endif -$(CS)/$(NAMESPACE)/controller.deployed: Makefile $(DEPLOY_INPUTS) +$(CS)/$(NAMESPACE)/controller.deployed: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) $(CS)/image.loaded mkdir -p $(@D) - @[ -z "$(PROJECT_IMAGE)" ] || $(CONTAINER_TOOL) pull $(E2E_IMAGE) - $(KIND) load docker-image $(E2E_IMAGE) --name $(CLUSTER_FROM_CTX) - cd config && $(KUSTOMIZE) edit set namespace $(NAMESPACE) - cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_IMAGE) - $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - IMAGE_ID="$$(cat $(IS)/controller.id)"; \ - kubectl --context $(CTX) -n $(NAMESPACE) patch deploy/gitops-reverser --type=merge \ - -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"configbutler.ai/e2e-image-id\":\"$$IMAGE_ID\"}}}}}"; \ - kubectl --context $(CTX) -n $(NAMESPACE) patch deploy/gitops-reverser --type=json \ - -p '[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]'; \ - fi - kubectl --context $(CTX) -n $(NAMESPACE) rollout status deploy/gitops-reverser --timeout=180s + @set -euo pipefail; \ + ctx="$(CTX)"; \ + ns="$(NAMESPACE)"; \ + project_image="$(PROJECT_IMAGE)"; \ + if [ -z "$$project_image" ]; then \ + echo "ERROR: PROJECT_IMAGE must be non-empty"; \ + exit 2; \ + fi; \ + deployments="$$(kubectl --context "$$ctx" -n "$$ns" get deployments -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' || true)"; \ + deployments="$$(printf '%s\n' "$$deployments" | sed '/^$$/d')"; \ + deploy_count=0; \ + if [ -n "$$deployments" ]; then \ + deploy_count="$$(printf '%s\n' "$$deployments" | wc -l | tr -d ' ')"; \ + fi; \ + if [ "$$deploy_count" -ne 1 ]; then \ + echo "ERROR: Expected exactly 1 Deployment in namespace '$$ns', found $$deploy_count"; \ + kubectl --context "$$ctx" -n "$$ns" get deployments -o wide || true; \ + exit 1; \ + fi; \ + deploy="$$(printf '%s\n' "$$deployments" | head -n1)"; \ + default_container="$$(kubectl --context "$$ctx" -n "$$ns" get deployment "$$deploy" -o jsonpath='{.spec.template.metadata.annotations.kubectl\\.kubernetes\\.io/default-container}' 2>/dev/null || true)"; \ + containers="$$(kubectl --context "$$ctx" -n "$$ns" get deployment "$$deploy" -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{"\t"}{.image}{"\n"}{end}')"; \ + container=""; \ + if [ -n "$$default_container" ] && printf '%s\n' "$$containers" | cut -f1 | grep -qx "$$default_container"; then \ + container="$$default_container"; \ + else \ + matches="$$(printf '%s\n' "$$containers" | awk -F'\t' '$$2 ~ /gitops-reverser/ {print $$1}')"; \ + matches="$$(printf '%s\n' "$$matches" | sed '/^$$/d')"; \ + match_count=0; \ + if [ -n "$$matches" ]; then \ + match_count="$$(printf '%s\n' "$$matches" | wc -l | tr -d ' ')"; \ + fi; \ + if [ "$$match_count" -eq 1 ]; then \ + container="$$(printf '%s\n' "$$matches" | head -n1)"; \ + else \ + container_count=0; \ + if [ -n "$$containers" ]; then \ + container_count="$$(printf '%s\n' "$$containers" | sed '/^$$/d' | wc -l | tr -d ' ')"; \ + fi; \ + if [ "$$container_count" -eq 1 ]; then \ + container="$$(printf '%s\n' "$$containers" | cut -f1)"; \ + else \ + echo "ERROR: Unable to uniquely select a container to patch in deployment '$$deploy'"; \ + echo "Containers (name image):"; \ + printf '%s\n' "$$containers" | awk -F'\t' '{print $$1, $$2}'; \ + exit 1; \ + fi; \ + fi; \ + fi; \ + echo "Ensuring deployment '$$deploy' (container '$$container') uses image '$$project_image'"; \ + kubectl --context "$$ctx" -n "$$ns" patch deployment "$$deploy" --type=strategic \ + -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$$container\",\"image\":\"$$project_image\",\"imagePullPolicy\":\"IfNotPresent\"}]}}}}"; \ + kubectl --context "$$ctx" -n "$$ns" rollout status "deployment/$$deploy" --timeout=180s touch $@ $(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed @@ -408,13 +458,15 @@ install-plain-manifests-file: $(CS)/$(NAMESPACE)/install-plain-manifests-file install-config-dir: $(CS)/$(NAMESPACE)/install-config-dir +CONFIG_DIR_INPUTS := $(shell find config -type f) + ##@ Deployments of all manifests needed to run $(CS)/$(NAMESPACE)/install-helm: INSTALL_MODE := helm $(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via Helm into CTX cluster @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) \ helm_args=( \ - upgrade --install $(INSTALL_NAME) charts/gitops-reverser \ + upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ --kube-context $(CTX) \ --namespace $(NAMESPACE) \ --create-namespace \ @@ -430,13 +482,16 @@ $(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.install touch $@ $(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed \ +$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed \ $(CS)/$(NAMESPACE)/namespace.ready ## Deploy controller to the K8s cluster specified in ~/.kube/config. @set -euo pipefail; \ - $(ASSERT_INSTALL_METHOD) - (cd config && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)") - (cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG}) - (cd config && $(KUSTOMIZE) build .) | $(KUBECTL) --context $(CTX) apply -f - + $(ASSERT_INSTALL_METHOD) \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + cp -R config "$$tmpdir/config"; \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)"); \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set image gitops-reverser="$(PROJECT_IMAGE)"); \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) | $(KUBECTL) --context "$(CTX)" apply -f - touch $@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file @@ -445,8 +500,11 @@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-ma ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ $(ASSERT_INSTALL_METHOD) \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + cp -R dist "$$tmpdir/dist"; \ ( \ - cd dist && \ + cd "$$tmpdir/dist" && \ kustomize edit set namespace "$$ns" >/dev/null && \ kustomize build . \ ) | kubectl --context "$$ctx" apply -f -; \ @@ -455,31 +513,20 @@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-ma .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/image.loaded; \ - fi + $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm \ + .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed \ + .stamps/cluster/$(KUBECONTEXT_QS_HELM)/$(NAMESPACE)/controller.deployed kubectl config use-context $(KUBECONTEXT_QS_HELM) - PROJECT_IMAGE=$(E2E_IMAGE) \ - bash test/e2e/scripts/run-quickstart.sh helm + bash test/e2e/scripts/run-quickstart.sh helm .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) dist/install.yaml; \ - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ - fi + $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file \ + .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed \ + .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/$(NAMESPACE)/controller.deployed kubectl config use-context $(KUBECONTEXT_QS_MANIFEST) - PROJECT_IMAGE=$(E2E_IMAGE) \ - bash test/e2e/scripts/run-quickstart.sh manifest + bash test/e2e/scripts/run-quickstart.sh manifest .PHONY: cleanup-port-forwards cleanup-port-forwards: ## Stop all port-forwards diff --git a/docs/make-e2e-further.md b/docs/make-e2e-further.md index c5b3317e3..fdfe97d56 100644 --- a/docs/make-e2e-further.md +++ b/docs/make-e2e-further.md @@ -139,7 +139,7 @@ Example: $(CS)/e2e.passed: $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') mkdir -p $(CS) kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_IMAGE) \ + KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ diff --git a/docs/makefile-e2e-image-and-deploy-refactor-plan.md b/docs/makefile-e2e-image-and-deploy-refactor-plan.md new file mode 100644 index 000000000..d4ca15a8f --- /dev/null +++ b/docs/makefile-e2e-image-and-deploy-refactor-plan.md @@ -0,0 +1,159 @@ +# Makefile E2E Image + Deployment Refactor Plan + +This plan refactors two E2E Makefile abstractions that are currently spread across multiple targets and scripts: + +1) **Image resolution**: β€œwhat image should the tests use?” (local dev build vs CI-provided image) +2) **Deployment readiness**: β€œafter *any* install method, the controller is running with the desired image” + +It is intended to complement: + +- `docs/makefile-e2e-overview-and-plan.md` (roadmap) +- `docs/e2e-shared-infra-parallel-plan.md` (shared-infra ownership contract) + +## Problems to solve (current state) + +- Image logic is duplicated: + - `E2E_IMAGE := $(if $(PROJECT_IMAGE),...)` hides whether we’re using a CI-provided image or a local build. + - Multiple targets check `PROJECT_IMAGE` and branch differently (main e2e vs quickstart flows). +- `$(CS)/$(NAMESPACE)/controller.deployed` is not a β€œdeployment contract” yet: + - It applies `config/` (install-method-specific). + - It patches a deployment **by name** (`deploy/gitops-reverser`). + - It implicitly assumes config-dir installation behavior. + +## Target contracts (end-state) + +### Contract A β€” `PROJECT_IMAGE` becomes the single source of truth + +- If CI pre-builds an image, it passes `PROJECT_IMAGE=` into `make test-e2e` / quickstart targets. +- If `PROJECT_IMAGE` is **not** provided, Make chooses a default local image reference and builds it. +- Only one place in the Makefile β€œdecides” and β€œbranches” based on whether `PROJECT_IMAGE` was provided. +- After that point, all recipes can assume `PROJECT_IMAGE` is non-empty. + +Practical consequences: + +- No more `E2E_IMAGE` indirection (or it becomes an alias of `PROJECT_IMAGE`). +- No more scattered `if [ -z "$(PROJECT_IMAGE)" ] ...` checks outside the image-resolve/build step. + +### Contract B β€” `controller.deployed` becomes install-method-agnostic + +`$(CS)/$(NAMESPACE)/controller.deployed` means: + +- β€œThere is exactly one Deployment in `$(NAMESPACE)`” +- β€œThat Deployment is configured to run `$(PROJECT_IMAGE)`” +- β€œThat Deployment has `imagePullPolicy` set to the desired value for E2E (expected: `IfNotPresent`)” +- β€œThat Deployment has completed rollout” + +Constraints: + +- It finds the Deployment dynamically (not by a hardcoded name). +- It fails fast if it finds **0** deployments or **>1** deployments (and prints what it found). +- It does **not** apply `config/` or otherwise β€œinstall” anything. Installation is a separate concern. + +## Proposed Makefile structure + +### 1) Resolve/build the image once + +Add one stamp target that is the only place allowed to branch on β€œwas `PROJECT_IMAGE` provided?”: + +- `$(IS)/project-image.ready` + +Recipe behavior: + +- If `PROJECT_IMAGE` was provided: + - Ensure the image exists in the local container runtime (`docker pull` or `docker image inspect` + pull-on-miss). + - Write the stamp. +- If `PROJECT_IMAGE` was not provided: + - Set `PROJECT_IMAGE` to the default local tag (e.g. `gitops-reverser:e2e-local`). + - Build it (reusing `$(IS)/controller.id` for cache correctness and optional annotations). + - Write the stamp. + +Makefile variable pattern to support this cleanly: + +- Capture β€œprovided or not” *before* defaulting: + - `PROJECT_IMAGE_PROVIDED := $(strip $(PROJECT_IMAGE))` +- Then default and export: + - `PROJECT_IMAGE ?= $(E2E_LOCAL_IMAGE)` + - `export PROJECT_IMAGE` + +Only `$(IS)/project-image.ready` should read `PROJECT_IMAGE_PROVIDED`. + +### 2) Always load the resolved image into Kind (all E2E clusters) + +Update `$(CS)/image.loaded` to: + +- depend on `$(IS)/project-image.ready` and `$(CS)/ready` +- run `kind load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX)` + +Rationale: + +- Kind nodes should not need registry credentials for CI images. +- The rest of the system can rely on `imagePullPolicy=IfNotPresent` and fast startup. + +### 3) Separate install from β€œimage + rollout” deployment contract + +Make `install-$(INSTALL_MODE)` responsible only for β€œresources exist” (Helm / dist manifests / config-dir). + +Make `$(CS)/$(NAMESPACE)/controller.deployed` responsible only for: + +1. Assert exactly one Deployment in namespace +2. Patch its container image to `$(PROJECT_IMAGE)` +3. Patch its `imagePullPolicy` (expected: `IfNotPresent`) +4. Wait for rollout readiness + +Container selection strategy (no deployment-name assumptions): + +- Prefer a single container whose image contains `gitops-reverser` (expected unique match). +- If that’s ambiguous, fail with a diagnostic dump of container names/images. + +## Dependency graph impact (high level) + +- `$(CS)/$(NAMESPACE)/e2e/prepare` should depend on: + - `$(CS)/$(NAMESPACE)/namespace.cleaned` + - `$(CS)/$(NAMESPACE)/install-$(INSTALL_MODE)` + - `$(CS)/image.loaded` + - `$(CS)/$(NAMESPACE)/controller.deployed` + - `$(CS)/portforward.running` + - `$(CS)/$(NAMESPACE)/age-key.applied` + +Quickstart targets should also rely on the same image resolution + loading primitives (no custom branching). + +## Phased implementation plan (small PRs) + +### PR5a β€” Normalize `PROJECT_IMAGE` (image resolution) + +- Introduce `PROJECT_IMAGE_PROVIDED`, default `PROJECT_IMAGE`, and `export PROJECT_IMAGE`. +- Add `$(IS)/project-image.ready` and move all branching there. +- Update `$(CS)/image.loaded` to load `$(PROJECT_IMAGE)` (not `$(E2E_LOCAL_IMAGE)`). +- Remove `E2E_IMAGE` (or make it a strict alias), and remove `PROJECT_IMAGE` checks from quickstart targets. + +Acceptance: + +- `make test-e2e` and quickstart targets have exactly one β€œdecision point” for CI vs local image. +- Both local and CI runs result in `$(PROJECT_IMAGE)` existing locally and being loadable into Kind. + +### PR5b β€” Redefine `controller.deployed` as the install-agnostic contract + +- Stop applying `config/` in `$(CS)/$(NAMESPACE)/controller.deployed`. +- Implement β€œexactly one Deployment in namespace” discovery + failure modes. +- Patch image + pull policy + rollout wait without assuming a deployment name. + +Acceptance: + +- `controller.deployed` works after `install-config-dir`, `install-helm`, and `install-plain-manifests-file`. +- If the namespace contains 0 or >1 deployments, `controller.deployed` fails with a clear diagnostic. + +### PR5c β€” Align quickstart smoke harness with the new contract + +- Make quickstart Make targets perform install via `install-$(mode)` + `controller.deployed` (instead of patching in + `run-quickstart.sh`). +- Remove or simplify the `PROJECT_IMAGE` override logic in `test/e2e/scripts/run-quickstart.sh` (it should no longer + need to know deployment names or how the install happened). + +Acceptance: + +- Quickstart smoke tests no longer patch deployments directly and no longer depend on a fixed deployment name. + +## Non-goals (for this refactor) + +- Changing the Go E2E run-context ownership model from `docs/e2e-shared-infra-parallel-plan.md`. +- Expanding the suite Γ— install-method matrix beyond what already exists. diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md index 6d5d084d6..21d89a7f2 100644 --- a/docs/makefile-e2e-overview-and-plan.md +++ b/docs/makefile-e2e-overview-and-plan.md @@ -109,6 +109,13 @@ Acceptance: - Quickstart smoke tests’ β€œreal assertions” live in Go, not shell. +### PR5 β€” Simplify image resolution + make deployment install-agnostic + +This refactor makes `PROJECT_IMAGE` the single source of truth and turns `$(CS)/$(NAMESPACE)/controller.deployed` +into a true β€œdeployment contract” that is independent of the chosen install method. + +Plan: `docs/makefile-e2e-image-and-deploy-refactor-plan.md`. + ## Notes / non-goals (for now) - Don’t expand the suite/install matrix until the `e2e/prepare` contract and run scoping are solid. diff --git a/test/e2e/scripts/run-quickstart.sh b/test/e2e/scripts/run-quickstart.sh index befc47a74..506d30ba7 100755 --- a/test/e2e/scripts/run-quickstart.sh +++ b/test/e2e/scripts/run-quickstart.sh @@ -4,10 +4,8 @@ set -euo pipefail MODE="${1:-}" NAMESPACE="${NAMESPACE:-gitops-reverser}" QUICKSTART_NAMESPACE="${QUICKSTART_NAMESPACE:-sut}" -HELM_CHART_SOURCE="${HELM_CHART_SOURCE:-charts/gitops-reverser}" WAIT_TIMEOUT="${WAIT_TIMEOUT:-60s}" QUICKSTART_TIMEOUT_SECONDS="${QUICKSTART_TIMEOUT_SECONDS:-180}" -PROJECT_IMAGE="${PROJECT_IMAGE:-}" TEST_ID="$(date +%s)-$RANDOM" REPO_NAME="quickstart-smoke-${MODE}-${TEST_ID}" @@ -30,65 +28,52 @@ if [[ -z "${MODE}" ]]; then echo "usage: $0 " exit 1 fi +if [[ "${MODE}" != "helm" && "${MODE}" != "manifest" ]]; then + echo "unsupported mode: ${MODE} (expected helm or manifest)" + exit 1 +fi -get_controller_pod_selector() { - local selector - selector="$(kubectl -n "${NAMESPACE}" get deployment gitops-reverser \ - -o jsonpath='{range $k,$v := .spec.selector.matchLabels}{$k}={$v},{end}' 2>/dev/null || true)" - selector="${selector%,}" +discover_controller_deployment() { + local deployments deploy_count + deployments="$( + kubectl -n "${NAMESPACE}" get deployments -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true + )" + deployments="$(printf '%s\n' "${deployments}" | sed '/^$/d')" + if [[ -z "${deployments}" ]]; then + echo "No Deployments found in namespace ${NAMESPACE}" >&2 + kubectl -n "${NAMESPACE}" get deployments -o wide >&2 || true + return 1 + fi - if [[ -z "${selector}" ]]; then - selector="app.kubernetes.io/name=gitops-reverser" + deploy_count="$(printf '%s\n' "${deployments}" | wc -l | tr -d ' ')" + if [[ "${deploy_count}" != "1" ]]; then + echo "Expected exactly 1 Deployment in namespace ${NAMESPACE}, found ${deploy_count}" >&2 + kubectl -n "${NAMESPACE}" get deployments -o wide >&2 || true + return 1 fi - printf '%s' "${selector}" + printf '%s' "${deployments}" } -install_helm() { - local helm_image_args=() - local helm_crd_args=() - - if [[ -n "${PROJECT_IMAGE}" ]]; then - local image_no_digest image_repo image_tag - image_no_digest="${PROJECT_IMAGE%%@*}" - if [[ "${image_no_digest##*/}" == *:* ]]; then - image_repo="${image_no_digest%:*}" - image_tag="${image_no_digest##*:}" - else - image_repo="${image_no_digest}" - image_tag="latest" - fi - helm_image_args+=(--set "image.repository=${image_repo}" --set "image.tag=${image_tag}") - echo "Overriding chart image from PROJECT_IMAGE=${PROJECT_IMAGE}" +get_controller_pod_selector() { + local deploy selector + deploy="$(discover_controller_deployment 2>/dev/null || true)" + selector="" + if [[ -n "${deploy}" ]]; then + selector="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" \ + -o jsonpath='{range $k,$v := .spec.selector.matchLabels}{$k}={$v},{end}' 2>/dev/null || true)" + selector="${selector%,}" fi - if kubectl get crd gitproviders.configbutler.ai gittargets.configbutler.ai \ - watchrules.configbutler.ai clusterwatchrules.configbutler.ai >/dev/null 2>&1; then - helm_crd_args+=(--skip-crds) - echo "Detected existing CRDs in cluster; installing Helm release with --skip-crds" + if [[ -z "${selector}" ]]; then + selector="app.kubernetes.io/name=gitops-reverser" fi - echo "Installing from Helm chart (mode=helm, source=${HELM_CHART_SOURCE})" - helm upgrade --install "name-is-cool-but-not-relevant" "${HELM_CHART_SOURCE}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set fullnameOverride=gitops-reverser \ - "${helm_crd_args[@]}" \ - "${helm_image_args[@]}" -} - -install_manifest() { - echo "Installing from generated dist/install.yaml (mode=manifest)" - kubectl apply -f dist/install.yaml - - if [[ -n "${PROJECT_IMAGE}" ]]; then - echo "Overriding manifest deployment image from PROJECT_IMAGE=${PROJECT_IMAGE}" - kubectl -n "${NAMESPACE}" set image deployment/gitops-reverser manager="${PROJECT_IMAGE}" - fi + printf '%s' "${selector}" } print_debug_info() { - local pod_selector + local pod_selector deploy pod_selector="$(get_controller_pod_selector)" echo @@ -97,11 +82,15 @@ print_debug_info() { echo "Quickstart namespace: ${QUICKSTART_NAMESPACE}" echo "Pod selector: ${pod_selector}" echo - echo "Deployment status:" - kubectl -n "${NAMESPACE}" get deployment gitops-reverser -o wide || true - echo - echo "Deployment describe:" - kubectl -n "${NAMESPACE}" describe deployment gitops-reverser || true + echo "Deployments:" + kubectl -n "${NAMESPACE}" get deployments -o wide || true + + deploy="$(discover_controller_deployment 2>/dev/null || true)" + if [[ -n "${deploy}" ]]; then + echo + echo "Deployment describe (${deploy}):" + kubectl -n "${NAMESPACE}" describe deployment "${deploy}" || true + fi echo echo "Pods:" kubectl -n "${NAMESPACE}" get pods -o wide || true @@ -172,33 +161,18 @@ ensure_gitea_api_port_forward() { return 1 } -reset_install_state() { - echo "Resetting install state for clean first-time install validation" - kubectl delete clusterrole \ - gitops-reverser \ - gitops-reverser-metrics-reader \ - gitops-reverser-proxy-role \ - --ignore-not-found=true >/dev/null || true - kubectl delete clusterrolebinding \ - gitops-reverser \ - gitops-reverser-proxy-rolebinding \ - --ignore-not-found=true >/dev/null || true - kubectl delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration \ - --ignore-not-found=true >/dev/null || true - kubectl delete namespace "${NAMESPACE}" --wait=true --ignore-not-found=true >/dev/null || true -} - verify_installation() { - local pod_selector + local pod_selector deploy + deploy="$(discover_controller_deployment)" pod_selector="$(get_controller_pod_selector)" run_or_debug \ "Waiting for deployment rollout (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" rollout status deployment/gitops-reverser --timeout="${WAIT_TIMEOUT}" + kubectl -n "${NAMESPACE}" rollout status "deployment/${deploy}" --timeout="${WAIT_TIMEOUT}" run_or_debug \ "Checking deployment availability (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" wait --for=condition=available deployment/gitops-reverser --timeout="${WAIT_TIMEOUT}" + kubectl -n "${NAMESPACE}" wait --for=condition=available "deployment/${deploy}" --timeout="${WAIT_TIMEOUT}" run_or_debug \ "Checking pod readiness (selector=${pod_selector}, timeout=${WAIT_TIMEOUT})" \ @@ -571,21 +545,6 @@ EOF wait_for_connection_failed_actionable_message "${INVALID_PROVIDER_NAME}" "${QUICKSTART_TIMEOUT_SECONDS}" } -case "${MODE}" in - helm) - reset_install_state - install_helm - ;; - manifest) - reset_install_state - install_manifest - ;; - *) - echo "unsupported mode: ${MODE} (expected helm or manifest)" - exit 1 - ;; -esac - verify_installation run_or_debug "Running quickstart functional smoke checks" run_quickstart_flow echo "Install quickstart smoke test passed (${MODE})" From 34bfcf4c17e64616906ff6a9e4c93ec5127cad29 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 15:53:31 +0000 Subject: [PATCH 07/30] chore: Cleaning it up a bit more --- Makefile | 6 +- test/e2e/e2e_suite_test.go | 4 +- test/e2e/quickstart_framework_e2e_test.go | 125 ++++++---------------- 3 files changed, 36 insertions(+), 99 deletions(-) diff --git a/Makefile b/Makefile index 3b63f388c..2d4ab95c4 100644 --- a/Makefile +++ b/Makefile @@ -160,6 +160,10 @@ $(CS)/$(NAMESPACE)/namespace.ready: Makefile $(CS)/ready pod-security.kubernetes.io/enforce=restricted touch $@ +# Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. +.PHONY: e2e-prepare +e2e-prepare: $(CS)/$(NAMESPACE)/e2e/prepare portforward-ensure ## Prepare E2E prerequisites for Go tests + # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). @@ -167,11 +171,11 @@ $(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/$(NAMESPACE)/namespace.cleaned \ $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ - $(CS)/portforward.running \ $(CS)/$(NAMESPACE)/age-key.applied mkdir -p $(@D) touch $@ + # Keep `make test-e2e` as the classic entrypoint. .PHONY: test-e2e test-e2e: $(CS)/e2e.passed diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d158f5a11..bf4e705fc 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -61,13 +61,11 @@ func ensureE2EPrepared() { seed := ginkgoRandomSeed() ns := resolveE2ENamespace() installName := resolveE2EInstallName(seed) - target := fmt.Sprintf(".stamps/cluster/%s/%s/e2e/prepare", ctx, ns) cmd := makeCommand( fmt.Sprintf("CTX=%s", ctx), fmt.Sprintf("INSTALL_NAME=%s", installName), fmt.Sprintf("NAMESPACE=%s", ns), - target, - "portforward-ensure", + "e2e-prepare", ) output, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "failed to run make target for e2e prepare") diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index d2e37a9c0..6b421b9bf 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -40,17 +40,16 @@ const ( ) type quickstartFrameworkRun struct { - mode string - installNamespace string - testID string - repoName string - checkoutDir string - repoURL string - providerName string - targetName string - watchRuleName string - invalidProvName string - encryptionName string + mode string + testID string + repoName string + checkoutDir string + repoURL string + providerName string + targetName string + watchRuleName string + invalidProvName string + encryptionName string } var _ = Describe("Quickstart Framework", Label("quickstart-framework"), Ordered, func() { @@ -68,18 +67,14 @@ var _ = Describe("Quickstart Framework", Label("quickstart-framework"), Ordered, }) It("sets up quickstart flow via Go framework", func() { - By("resetting install state for clean quickstart framework execution") - run.resetInstallState() - - By(fmt.Sprintf("installing controller for quickstart mode %q", run.mode)) - run.installController() - run.waitForControllerRollout() - By("creating dedicated Gitea repository and bootstrap credentials") run.setupGiteaRepository() By("applying quickstart resources from Go") run.applyQuickstartResources() + + By("verifying quickstart resources become Ready") + run.verifyQuickstartResourcesReady() }) }) @@ -113,84 +108,17 @@ func newQuickstartFrameworkRun() quickstartFrameworkRun { checkoutDir := filepath.Join("/tmp/gitops-reverser", repoName) return quickstartFrameworkRun{ - mode: mode, - installNamespace: quickstartInstallerNamespace(), - testID: testID, - repoName: repoName, - checkoutDir: checkoutDir, - repoURL: fmt.Sprintf("http://gitea-http.gitea-e2e.svc.cluster.local:13000/testorg/%s.git", repoName), - providerName: fmt.Sprintf("quickstart-provider-%s", testID), - targetName: fmt.Sprintf("quickstart-target-%s", testID), - watchRuleName: fmt.Sprintf("quickstart-watchrule-%s", testID), - invalidProvName: fmt.Sprintf("quickstart-invalid-provider-%s", testID), - encryptionName: fmt.Sprintf("quickstart-sops-age-key-%s", testID), - } -} - -func (r *quickstartFrameworkRun) resetInstallState() { - deleteClusterScopeObjects := [][]string{ - { - "delete", "clusterrole", - "gitops-reverser", - "gitops-reverser-manager-role", - "gitops-reverser-metrics-reader", - "gitops-reverser-proxy-role", - "--ignore-not-found=true", - }, - { - "delete", "clusterrolebinding", - "gitops-reverser", - "gitops-reverser-manager-rolebinding", - "gitops-reverser-proxy-rolebinding", - "--ignore-not-found=true", - }, - {"delete", "validatingwebhookconfiguration", "gitops-reverser-validating-webhook-configuration", - "--ignore-not-found=true"}, - } - - for _, args := range deleteClusterScopeObjects { - cmd := exec.Command("kubectl", args...) - _, _ = utils.Run(cmd) - } - - deleteNamespace := exec.Command( - "kubectl", "delete", "namespace", r.installNamespace, "--wait=true", "--ignore-not-found=true", - ) - _, _ = utils.Run(deleteNamespace) -} - -func (r *quickstartFrameworkRun) installController() { - cmd := r.installCommand(r.mode) - output, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to install %s controller via make install", r.mode)) - _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) -} - -func (r *quickstartFrameworkRun) installCommand(mode string) *exec.Cmd { - ctx := resolveE2EContext() - args := []string{ - fmt.Sprintf("CTX=%s", ctx), - "install", - fmt.Sprintf("INSTALL_MODE=%s", mode), - fmt.Sprintf("NAMESPACE=%s", r.installNamespace), - fmt.Sprintf("INSTALL_NAMESPACE=%s", r.installNamespace), - fmt.Sprintf("INSTALL_NAME=%s", r.installNamespace), + mode: mode, + testID: testID, + repoName: repoName, + checkoutDir: checkoutDir, + repoURL: fmt.Sprintf("http://gitea-http.gitea-e2e.svc.cluster.local:13000/testorg/%s.git", repoName), + providerName: fmt.Sprintf("quickstart-provider-%s", testID), + targetName: fmt.Sprintf("quickstart-target-%s", testID), + watchRuleName: fmt.Sprintf("quickstart-watchrule-%s", testID), + invalidProvName: fmt.Sprintf("quickstart-invalid-provider-%s", testID), + encryptionName: fmt.Sprintf("quickstart-sops-age-key-%s", testID), } - return makeCommand(args...) -} - -func (r *quickstartFrameworkRun) waitForControllerRollout() { - rolloutCmd := exec.Command( - "kubectl", "-n", r.installNamespace, "rollout", "status", - "deployment/gitops-reverser", "--timeout=120s", - ) - _, err := utils.Run(rolloutCmd) - Expect(err).NotTo(HaveOccurred(), "controller rollout did not complete") -} - -func quickstartInstallerNamespace() string { - suiteConfig, _ := GinkgoConfiguration() - return fmt.Sprintf("run-%d", suiteConfig.RandomSeed) } func (r *quickstartFrameworkRun) setupGiteaRepository() { @@ -229,6 +157,13 @@ func (r *quickstartFrameworkRun) applyQuickstartResources() { createGitProviderWithURL(r.invalidProvName, "main", "git-creds-invalid", r.repoURL) } +func (r *quickstartFrameworkRun) verifyQuickstartResourcesReady() { + ns := quickstartSetupNamespace() + verifyResourceStatus("gitprovider", r.providerName, ns, "True", "Ready", "") + verifyResourceStatus("gittarget", r.targetName, ns, "True", "Ready", "") + verifyResourceStatus("watchrule", r.watchRuleName, ns, "True", "Ready", "") +} + func quickstartSetupNamespace() string { if value := strings.TrimSpace(os.Getenv(quickstartSetupNamespaceEnv)); value != "" { return value From 684e0399d6145cf3d1fa510229651e6c9290b1a5 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 19:15:29 +0000 Subject: [PATCH 08/30] chore: Sunsetting run-quickstart.sh, Makefile to the rescue --- Makefile | 107 ++-- .../gitops-reverser/templates/deployment.yaml | 1 - config/deployment.yaml | 1 + docs/make-e2e-deps.md | 46 +- ...file-e2e-image-and-deploy-refactor-plan.md | 6 +- docs/makefile-e2e-overview-and-plan.md | 4 +- test/e2e/e2e_test.go | 95 ++- test/e2e/kind/start-cluster.sh | 28 +- test/e2e/quickstart_framework_e2e_test.go | 379 +++++++++++- test/e2e/scripts/run-quickstart.sh | 550 ------------------ test/improving.md | 4 +- 11 files changed, 534 insertions(+), 687 deletions(-) delete mode 100755 test/e2e/scripts/run-quickstart.sh diff --git a/Makefile b/Makefile index 2d4ab95c4..2fd10bff7 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,7 @@ cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps rm -rf .stamps/cluster/kind-$$cluster; \ done +# We delete all services (over all namespaces) since we only can have ONE service listening to the cluster audit webhook at the same time $(CS)/$(NAMESPACE)/namespace.cleaned: Makefile $(CS)/$(NAMESPACE)/namespace.ready kubectl --context $(CTX) delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true kubectl --context $(CTX) delete validatingwebhookconfiguration \ @@ -343,8 +344,8 @@ $(IS)/project-image.ready: Makefile echo "Pulling PROJECT_IMAGE=$(PROJECT_IMAGE)"; \ $(CONTAINER_TOOL) pull "$(PROJECT_IMAGE)"; \ fi; \ - fi - touch $@ + fi; \ + echo "$(PROJECT_IMAGE)" > "$@" $(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) mkdir -p $(CS) @@ -353,70 +354,29 @@ $(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) touch $@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready - $(KIND) load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX) - touch $@ + @set -euo pipefail; \ + mkdir -p $(CS); \ + img_id="$$( $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(PROJECT_IMAGE) )"; \ + echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CLUSTER_FROM_CTX)"; \ + $(KIND) load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX); \ + echo "$$img_id" > "$@" -$(CS)/$(NAMESPACE)/controller.deployed: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) $(CS)/image.loaded - mkdir -p $(@D) +$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) $(CS)/image.loaded @set -euo pipefail; \ - ctx="$(CTX)"; \ - ns="$(NAMESPACE)"; \ - project_image="$(PROJECT_IMAGE)"; \ - if [ -z "$$project_image" ]; then \ - echo "ERROR: PROJECT_IMAGE must be non-empty"; \ - exit 2; \ - fi; \ - deployments="$$(kubectl --context "$$ctx" -n "$$ns" get deployments -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' || true)"; \ - deployments="$$(printf '%s\n' "$$deployments" | sed '/^$$/d')"; \ - deploy_count=0; \ - if [ -n "$$deployments" ]; then \ - deploy_count="$$(printf '%s\n' "$$deployments" | wc -l | tr -d ' ')"; \ - fi; \ - if [ "$$deploy_count" -ne 1 ]; then \ - echo "ERROR: Expected exactly 1 Deployment in namespace '$$ns', found $$deploy_count"; \ - kubectl --context "$$ctx" -n "$$ns" get deployments -o wide || true; \ + ctx="$(CTX)"; ns="$(NAMESPACE)"; img="$(PROJECT_IMAGE)"; c="$(CONTROLLER_CONTAINER)"; sel="$(CONTROLLER_DEPLOY_SELECTOR)"; \ + [ -n "$$img" ] || { echo "ERROR: PROJECT_IMAGE must be non-empty" >&2; exit 2; }; \ + [ -n "$$c" ] || { echo "ERROR: CONTROLLER_CONTAINER must be non-empty" >&2; exit 2; }; \ + count="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" --no-headers 2>/dev/null | wc -l | tr -d ' ')"; \ + [ "$$count" -eq 1 ] || { \ + echo "ERROR: Expected exactly 1 Deployment matching '$$sel' in namespace '$$ns', found $$count" >&2; \ + kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" -o wide || true; \ exit 1; \ - fi; \ - deploy="$$(printf '%s\n' "$$deployments" | head -n1)"; \ - default_container="$$(kubectl --context "$$ctx" -n "$$ns" get deployment "$$deploy" -o jsonpath='{.spec.template.metadata.annotations.kubectl\\.kubernetes\\.io/default-container}' 2>/dev/null || true)"; \ - containers="$$(kubectl --context "$$ctx" -n "$$ns" get deployment "$$deploy" -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{"\t"}{.image}{"\n"}{end}')"; \ - container=""; \ - if [ -n "$$default_container" ] && printf '%s\n' "$$containers" | cut -f1 | grep -qx "$$default_container"; then \ - container="$$default_container"; \ - else \ - matches="$$(printf '%s\n' "$$containers" | awk -F'\t' '$$2 ~ /gitops-reverser/ {print $$1}')"; \ - matches="$$(printf '%s\n' "$$matches" | sed '/^$$/d')"; \ - match_count=0; \ - if [ -n "$$matches" ]; then \ - match_count="$$(printf '%s\n' "$$matches" | wc -l | tr -d ' ')"; \ - fi; \ - if [ "$$match_count" -eq 1 ]; then \ - container="$$(printf '%s\n' "$$matches" | head -n1)"; \ - else \ - container_count=0; \ - if [ -n "$$containers" ]; then \ - container_count="$$(printf '%s\n' "$$containers" | sed '/^$$/d' | wc -l | tr -d ' ')"; \ - fi; \ - if [ "$$container_count" -eq 1 ]; then \ - container="$$(printf '%s\n' "$$containers" | cut -f1)"; \ - else \ - echo "ERROR: Unable to uniquely select a container to patch in deployment '$$deploy'"; \ - echo "Containers (name image):"; \ - printf '%s\n' "$$containers" | awk -F'\t' '{print $$1, $$2}'; \ - exit 1; \ - fi; \ - fi; \ - fi; \ - echo "Ensuring deployment '$$deploy' (container '$$container') uses image '$$project_image'"; \ - kubectl --context "$$ctx" -n "$$ns" patch deployment "$$deploy" --type=strategic \ - -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$$container\",\"image\":\"$$project_image\",\"imagePullPolicy\":\"IfNotPresent\"}]}}}}"; \ - kubectl --context "$$ctx" -n "$$ns" rollout status "deployment/$$deploy" --timeout=180s - touch $@ - -$(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed - mkdir -p $(CS) - E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh - touch $@ + }; \ + deploy="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" -o jsonpath='{.items[0].metadata.name}')"; \ + echo "Setting deployment/$$deploy container '$$c' to image '$$img'"; \ + kubectl --context "$$ctx" -n "$$ns" set image "deployment/$$deploy" "$$c=$$img" --record=false; \ + kubectl --context "$$ctx" -n "$$ns" rollout status "deployment/$$deploy" --timeout=180s; \ + touch "$@" .PHONY: portforward-ensure portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure port-forwards are running (always checks) @@ -431,6 +391,9 @@ $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile go test ./test/e2e/ -v -ginkgo.v touch $@ +CONTROLLER_CONTAINER ?= manager +CONTROLLER_DEPLOY_SELECTOR ?= app.kubernetes.io/part-of=gitops-reverser + # INSTALL_MODE, INSTALL_NAME, NAMESPACE defaults are defined near the E2E variables section above. INSTALL_LOCK := $(CS)/$(NAMESPACE)/install.method VALID_INSTALL_MODES := config-dir helm plain-manifests-file @@ -517,20 +480,18 @@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-ma .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/$(NAMESPACE)/controller.deployed - kubectl config use-context $(KUBECONTEXT_QS_HELM) - bash test/e2e/scripts/run-quickstart.sh helm + CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ + E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_HELM)/age-key.txt \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/$(NAMESPACE)/controller.deployed - kubectl config use-context $(KUBECONTEXT_QS_MANIFEST) - bash test/e2e/scripts/run-quickstart.sh manifest + CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ + E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/age-key.txt \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: cleanup-port-forwards cleanup-port-forwards: ## Stop all port-forwards diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index 49e8a379e..ad02d7813 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -6,7 +6,6 @@ metadata: labels: {{- include "gitops-reverser.labels" . | nindent 4 }} app.kubernetes.io/component: manager - app.kubernetes.io/created-by: gitops-reverser app.kubernetes.io/part-of: gitops-reverser control-plane: controller-manager spec: diff --git a/config/deployment.yaml b/config/deployment.yaml index e52e56c46..308c32a6b 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -4,6 +4,7 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: gitops-reverser + app.kubernetes.io/part-of: gitops-reverser control-plane: gitops-reverser name: gitops-reverser namespace: sut diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index c68914ef4..4c01eebce 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -282,18 +282,20 @@ Quickstart tests (`test-e2e-quickstart-helm`, `test-e2e-quickstart-manifest`) va ### Design: always start with a clean cluster -Quickstart tests **always delete and recreate their cluster** at the start of every run. This avoids a class of failures where Helm tries to install on top of resources that were previously applied with `kubectl apply` (server-side apply), which causes `metadata.managedFields must be nil` errors. Since `run-quickstart.sh` resets install state but doesn't delete CRDs or ClusterRoles, a clean cluster is the only reliable guarantee. +Quickstart tests **always delete and recreate their cluster** at the start of every run. This avoids a class of failures +where Helm tries to install on top of resources that were previously applied with `kubectl apply` (server-side apply), +which can surface as `metadata.managedFields must be nil` errors. A clean cluster is the simplest reliable guarantee. ### CTX override pattern -Quickstart targets reuse the generic stamp targets via recursive Make with `CTX` overridden: +Quickstart targets reuse the generic stamp targets by setting `CTX` before running `go test` (the Go `BeforeSuite` +invokes `make e2e-prepare`, which uses stamp targets under `.stamps/cluster/$(CTX)/...`): ```sh -$(MAKE) CTX=kind-gitops-reverser-test-e2e-quickstart-helm \ - .stamps/cluster/kind-gitops-reverser-test-e2e-quickstart-helm/cert-manager.installed +CTX=kind-gitops-reverser-test-e2e-quickstart-helm go test ./test/e2e/ ... ``` -The inner Make re-evaluates `CS` and `CLUSTER_FROM_CTX` with the overridden `CTX`, so the stamp file rules match correctly. +The stamp rules re-evaluate `CS` and `CLUSTER_FROM_CTX` with the overridden `CTX`, so all infra is isolated per cluster. ### Helm quickstart target @@ -301,16 +303,9 @@ The inner Make re-evaluates `CS` and `CLUSTER_FROM_CTX` with the overridden `CTX .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_HELM) - PROJECT_IMAGE=$(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) \ - bash test/e2e/scripts/run-quickstart.sh helm + CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework ``` ### Manifest quickstart target @@ -319,24 +314,17 @@ test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always s .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-plain-manifests-installer; \ - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_MANIFEST) - PROJECT_IMAGE=$(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) \ - bash test/e2e/scripts/run-quickstart.sh manifest + CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework ``` -The manifest target also calls `make build-plain-manifests-installer` (regenerates `dist/install.yaml`) before loading the image if `PROJECT_IMAGE` is not set. +The manifest target implicitly validates the generated `dist/install.yaml` via the `plain-manifests-file` install mode. -### `run-quickstart.sh` is self-contained +### Quickstart smoke lives in Go -`run-quickstart.sh` manages its own Gitea port-forward via an internal `ensure_gitea_api_port_forward()` function. It does **not** call `setup-port-forwards.sh` and does not need Prometheus. The Makefile only needs to ensure cert-manager, Gitea, and the image are loaded before invoking the script. +Quickstart smoke assertions run as a Go E2E test (`quickstart-framework` label). Cluster provisioning, controller install, +and port-forward setup is owned by `make e2e-prepare` (invoked from `BeforeSuite`). --- diff --git a/docs/makefile-e2e-image-and-deploy-refactor-plan.md b/docs/makefile-e2e-image-and-deploy-refactor-plan.md index d4ca15a8f..34283fd5a 100644 --- a/docs/makefile-e2e-image-and-deploy-refactor-plan.md +++ b/docs/makefile-e2e-image-and-deploy-refactor-plan.md @@ -145,9 +145,9 @@ Acceptance: ### PR5c β€” Align quickstart smoke harness with the new contract - Make quickstart Make targets perform install via `install-$(mode)` + `controller.deployed` (instead of patching in - `run-quickstart.sh`). -- Remove or simplify the `PROJECT_IMAGE` override logic in `test/e2e/scripts/run-quickstart.sh` (it should no longer - need to know deployment names or how the install happened). + a shell harness). +- Keep quickstart smoke assertions in Go (the `quickstart-framework` E2E spec) so install method changes don't require + shell logic updates. Acceptance: diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md index 21d89a7f2..0aaa85d74 100644 --- a/docs/makefile-e2e-overview-and-plan.md +++ b/docs/makefile-e2e-overview-and-plan.md @@ -102,8 +102,8 @@ Acceptance: ### PR4 β€” Reduce shell quickstart assertions (Go parity) -- Port `run-quickstart.sh` assertions into Go (commit progression, encryption checks, invalid-credential status). -- Keep the shell script as a thin bootstrap wrapper or retire it once parity is reached. +- (Done) Port quickstart smoke assertions into Go (commit progression, encryption checks, invalid-credential status). +- (Done) Retire the shell quickstart harness once parity is reached. Acceptance: diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4dadc526f..9b26a7c97 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -22,11 +22,11 @@ import ( "bytes" "encoding/base64" "encoding/json" - "errors" "fmt" "os" "os/exec" "path/filepath" + "sort" "strings" "time" @@ -1580,19 +1580,9 @@ func deriveAgeRecipient(identityString string) (string, error) { } func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) { - podCmd := exec.Command( - "kubectl", "get", "pods", - "-l", "control-plane=gitops-reverser", - "-n", namespace, - "-o", "jsonpath={.items[0].metadata.name}", - ) - podOutput, err := utils.Run(podCmd) + podName, err := discoverControllerPodName(namespace) if err != nil { - return "", fmt.Errorf("get controller pod: %w", err) - } - podName := strings.TrimSpace(podOutput) - if podName == "" { - return "", errors.New("controller pod name is empty") + return "", err } cmd := exec.Command( @@ -1610,6 +1600,85 @@ func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) return string(output), nil } +func discoverControllerPodName(ns string) (string, error) { + deploymentsCmd := exec.Command( + "kubectl", + "get", + "deployments", + "-n", + ns, + "-o", + "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}", + ) + deploymentsOutput, err := utils.Run(deploymentsCmd) + if err != nil { + return "", fmt.Errorf("get deployments in namespace %s: %w", ns, err) + } + + deployments := utils.GetNonEmptyLines(deploymentsOutput) + if len(deployments) != 1 { + return "", fmt.Errorf("expected exactly 1 Deployment in namespace %s, found %d", ns, len(deployments)) + } + deploymentName := deployments[0] + + deploymentCmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", ns, "-o", "json") + deploymentOutput, err := utils.Run(deploymentCmd) + if err != nil { + return "", fmt.Errorf("get deployment %s/%s: %w", ns, deploymentName, err) + } + + var deploymentObj unstructured.Unstructured + if unmarshalErr := json.Unmarshal([]byte(deploymentOutput), &deploymentObj); unmarshalErr != nil { + return "", fmt.Errorf("unmarshal deployment %s/%s: %w", ns, deploymentName, unmarshalErr) + } + + matchLabels, found, matchLabelErr := unstructured.NestedStringMap( + deploymentObj.Object, + "spec", + "selector", + "matchLabels", + ) + if matchLabelErr != nil { + return "", fmt.Errorf("read deployment selector labels %s/%s: %w", ns, deploymentName, matchLabelErr) + } + if !found || len(matchLabels) == 0 { + return "", fmt.Errorf("deployment selector labels are empty for %s/%s", ns, deploymentName) + } + + keys := make([]string, 0, len(matchLabels)) + for key := range matchLabels { + keys = append(keys, key) + } + sort.Strings(keys) + selectorParts := make([]string, 0, len(keys)) + for _, key := range keys { + selectorParts = append(selectorParts, fmt.Sprintf("%s=%s", key, matchLabels[key])) + } + selector := strings.Join(selectorParts, ",") + + podCmd := exec.Command( + "kubectl", + "get", + "pods", + "-n", + ns, + "-l", + selector, + "-o", + "jsonpath={.items[0].metadata.name}", + ) + podOutput, err := utils.Run(podCmd) + if err != nil { + return "", fmt.Errorf("get controller pod for selector %q in namespace %s: %w", selector, ns, err) + } + podName := strings.TrimSpace(podOutput) + if podName == "" { + return "", fmt.Errorf("controller pod name is empty for selector %q in namespace %s", selector, ns) + } + + return podName, nil +} + // createGitProviderWithURL creates a GitProvider resource with the specified URL. func createGitProviderWithURL(name, branch, secretName, repoURL string) { By(fmt.Sprintf("creating GitProvider '%s' with branch '%s', secret '%s' and URL '%s'", diff --git a/test/e2e/kind/start-cluster.sh b/test/e2e/kind/start-cluster.sh index c60a57d3d..be79906a6 100755 --- a/test/e2e/kind/start-cluster.sh +++ b/test/e2e/kind/start-cluster.sh @@ -22,7 +22,15 @@ is_known_kind_bootstrap_flake() { run_dood_self_heal() { local control_plane_container="${CLUSTER_NAME}-control-plane" - docker inspect "$control_plane_container" >/dev/null 2>&1 || return 1 + local container_found=false + for _ in $(seq 1 30); do + if docker inspect "$control_plane_container" >/dev/null 2>&1; then + container_found=true + break + fi + sleep 1 + done + [ "$container_found" = "true" ] || return 1 local ready=false for _ in $(seq 1 120); do @@ -83,12 +91,20 @@ else echo "πŸš€ Creating Kind cluster '$CLUSTER_NAME' with audit webhook support..." if create_primary_cluster; then echo "βœ… Kind cluster created successfully" - elif is_known_kind_bootstrap_flake && run_dood_self_heal; then - echo "βœ… Kind cluster self-healed successfully" else - echo "❌ Kind cluster creation failed." - echo "πŸ“„ See log: $KIND_CREATE_LOG_FILE" - exit 1 + if is_known_kind_bootstrap_flake; then + echo "⚠️ Kind bootstrap flake detected; attempting self-heal" + else + echo "⚠️ Kind cluster creation failed; attempting self-heal" + fi + + if run_dood_self_heal; then + echo "βœ… Kind cluster self-healed successfully" + else + echo "❌ Kind cluster creation failed." + echo "πŸ“„ See log: $KIND_CREATE_LOG_FILE" + exit 1 + fi fi fi diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index 6b421b9bf..5b821b03c 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -19,6 +19,8 @@ limitations under the License. package e2e import ( + "encoding/base64" + "encoding/json" "fmt" "os" "os/exec" @@ -29,6 +31,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/ConfigButler/gitops-reverser/test/utils" ) @@ -37,6 +40,7 @@ const ( quickstartFrameworkEnabledEnv = "E2E_ENABLE_QUICKSTART_FRAMEWORK" quickstartFrameworkModeEnv = "E2E_QUICKSTART_MODE" quickstartSetupNamespaceEnv = "QUICKSTART_NAMESPACE" + quickstartTimeoutSecondsEnv = "QUICKSTART_TIMEOUT_SECONDS" ) type quickstartFrameworkRun struct { @@ -75,6 +79,18 @@ var _ = Describe("Quickstart Framework", Label("quickstart-framework"), Ordered, By("verifying quickstart resources become Ready") run.verifyQuickstartResourcesReady() + + By("verifying generated encryption secret and commit flow") + generatedAgeKey := run.verifyGeneratedEncryptionSecret() + + By("verifying quickstart commits for create/update/delete") + run.verifyQuickstartConfigMapCommits() + + By("verifying quickstart encrypted Secret commit is decryptable") + run.verifyQuickstartSecretEncryption(generatedAgeKey) + + By("verifying invalid credentials provider shows actionable message") + run.verifyInvalidProviderActionableMessage() }) }) @@ -123,17 +139,26 @@ func newQuickstartFrameworkRun() quickstartFrameworkRun { func (r *quickstartFrameworkRun) setupGiteaRepository() { cmd := exec.Command("bash", "test/e2e/scripts/setup-gitea.sh", r.repoName, r.checkoutDir) - cmd.Env = append(os.Environ(), fmt.Sprintf("SUT_NAMESPACE=%s", quickstartSetupNamespace())) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "failed to bootstrap gitea repository for quickstart") + projectDir, err := utils.GetProjectDir() + Expect(err).NotTo(HaveOccurred(), "failed to resolve project directory for quickstart") + + cmd.Dir = projectDir + cmd.Env = append( + os.Environ(), + "GO111MODULE=on", + fmt.Sprintf("SUT_NAMESPACE=%s", quickstartSetupNamespace()), + ) + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to bootstrap gitea repository for quickstart: %s", output)) } func (r *quickstartFrameworkRun) applyQuickstartResources() { - createGitProviderWithURL(r.providerName, "main", "git-creds", r.repoURL) + qsNamespace := quickstartSetupNamespace() + createGitProviderWithURLInNamespace(r.providerName, qsNamespace, "main", "git-creds", r.repoURL) createGitTargetWithEncryptionOptions( r.targetName, - quickstartSetupNamespace(), + qsNamespace, r.providerName, "live-cluster", "main", @@ -147,14 +172,14 @@ func (r *quickstartFrameworkRun) applyQuickstartResources() { DestinationName string }{ Name: r.watchRuleName, - Namespace: quickstartSetupNamespace(), + Namespace: qsNamespace, DestinationName: r.targetName, } - err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, quickstartSetupNamespace()) + err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, qsNamespace) Expect(err).NotTo(HaveOccurred(), "failed to apply quickstart watchrule") - createGitProviderWithURL(r.invalidProvName, "main", "git-creds-invalid", r.repoURL) + createGitProviderWithURLInNamespace(r.invalidProvName, qsNamespace, "main", "git-creds-invalid", r.repoURL) } func (r *quickstartFrameworkRun) verifyQuickstartResourcesReady() { @@ -164,6 +189,344 @@ func (r *quickstartFrameworkRun) verifyQuickstartResourcesReady() { verifyResourceStatus("watchrule", r.watchRuleName, ns, "True", "Ready", "") } +func (r *quickstartFrameworkRun) verifyGeneratedEncryptionSecret() string { + ns := quickstartSetupNamespace() + var generatedAgeKey string + + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secret", r.encryptionName, "-n", ns, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var secretObj map[string]interface{} + g.Expect(json.Unmarshal([]byte(output), &secretObj)).To(Succeed()) + + annotations, _, annoErr := unstructured.NestedStringMap(secretObj, "metadata", "annotations") + g.Expect(annoErr).NotTo(HaveOccurred()) + g.Expect(annotations).To(HaveKeyWithValue("configbutler.ai/backup-warning", "REMOVE_AFTER_BACKUP")) + g.Expect(annotations).To(HaveKey("configbutler.ai/age-recipient")) + g.Expect(annotations["configbutler.ai/age-recipient"]).To(HavePrefix("age1")) + + secretData, found, keyErr := unstructured.NestedStringMap(secretObj, "data") + g.Expect(keyErr).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + + var sopsAgeKeyB64 string + for key, value := range secretData { + if strings.HasSuffix(key, ".agekey") { + sopsAgeKeyB64 = value + break + } + } + g.Expect(sopsAgeKeyB64).NotTo(BeEmpty()) + + keyBytes, decodeErr := base64.StdEncoding.DecodeString(sopsAgeKeyB64) + g.Expect(decodeErr).NotTo(HaveOccurred()) + generatedAgeKey = strings.TrimSpace(string(keyBytes)) + g.Expect(generatedAgeKey).To(HavePrefix("AGE-SECRET-KEY-")) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + return generatedAgeKey +} + +func (r *quickstartFrameworkRun) verifyQuickstartConfigMapCommits() { + ns := quickstartSetupNamespace() + configMapName := fmt.Sprintf("quickstart-config-%s", r.testID) + expectedFile := filepath.Join( + r.checkoutDir, + "live-cluster", + "v1", + "configmaps", + ns, + fmt.Sprintf("%s.yaml", configMapName), + ) + + _, _ = utils.Run(exec.Command( + "kubectl", + "delete", + "configmap", + configMapName, + "-n", + ns, + "--ignore-not-found=true", + )) + + commitsBefore, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = utils.Run(exec.Command( + "kubectl", + "create", + "configmap", + configMapName, + "--from-literal=value=one", + "-n", + ns, + )) + Expect(err).NotTo(HaveOccurred(), "failed to create quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("value: one")) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsBefore)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + commitsAfterCreate, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = utils.Run(exec.Command( + "kubectl", + "patch", + "configmap", + configMapName, + "-n", + ns, + "--type", + "merge", + "--patch", + `{"data":{"value":"two"}}`, + )) + Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("value: two")) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsAfterCreate)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + commitsAfterUpdate, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = utils.Run(exec.Command( + "kubectl", + "delete", + "configmap", + configMapName, + "-n", + ns, + )) + Expect(err).NotTo(HaveOccurred(), "failed to delete quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred()) + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), fmt.Sprintf("ConfigMap file should be deleted: %s", expectedFile)) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsAfterUpdate)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) +} + +func (r *quickstartFrameworkRun) verifyQuickstartSecretEncryption(generatedAgeKey string) { + ns := quickstartSetupNamespace() + secretName := fmt.Sprintf("quickstart-secret-%s", r.testID) + secretValueOne := fmt.Sprintf("quickstart-plaintext-one-%s", r.testID) + secretValueTwo := fmt.Sprintf("quickstart-plaintext-two-%s", r.testID) + secretValueOneB64 := base64.StdEncoding.EncodeToString([]byte(secretValueOne)) + secretValueTwoB64 := base64.StdEncoding.EncodeToString([]byte(secretValueTwo)) + + expectedFile := filepath.Join( + r.checkoutDir, + "live-cluster", + "v1", + "secrets", + ns, + fmt.Sprintf("%s.sops.yaml", secretName), + ) + + commitsBefore, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, _ = utils.Run(exec.Command( + "kubectl", + "delete", + "secret", + secretName, + "-n", + ns, + "--ignore-not-found=true", + )) + + _, err = utils.Run(exec.Command( + "kubectl", + "create", + "secret", + "generic", + secretName, + "--from-literal", + fmt.Sprintf("password=%s", secretValueOne), + "-n", + ns, + )) + Expect(err).NotTo(HaveOccurred(), "failed to create quickstart Secret") + + _, err = utils.Run(exec.Command( + "kubectl", + "patch", + "secret", + secretName, + "-n", + ns, + "--type", + "merge", + "--patch", + fmt.Sprintf(`{"stringData":{"password":"%s"}}`, secretValueTwo), + )) + Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart Secret") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("sops:")) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueOne)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueTwo)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueOneB64)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueTwoB64)) + + decryptedOutput, decryptErr := decryptWithControllerSOPS(content, generatedAgeKey) + g.Expect(decryptErr).NotTo(HaveOccurred(), "failed to decrypt committed secret via controller sops binary") + g.Expect(decryptedOutput).To(ContainSubstring(secretValueTwoB64)) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsBefore)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + _, _ = utils.Run(exec.Command( + "kubectl", + "delete", + "secret", + secretName, + "-n", + ns, + "--ignore-not-found=true", + )) +} + +func (r *quickstartFrameworkRun) verifyInvalidProviderActionableMessage() { + ns := quickstartSetupNamespace() + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "gitprovider", r.invalidProvName, "-n", ns, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var obj unstructured.Unstructured + g.Expect(json.Unmarshal([]byte(output), &obj)).To(Succeed()) + + conditions, found, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + g.Expect(found).To(BeTrue(), "status.conditions not found") + + var readyStatus, readyReason, readyMessage string + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + if condMap["type"] == "Ready" { + readyStatus, _ = condMap["status"].(string) + readyReason, _ = condMap["reason"].(string) + readyMessage, _ = condMap["message"].(string) + break + } + } + + g.Expect(readyStatus).To(Equal("False")) + g.Expect(readyReason).To(Equal("ConnectionFailed")) + g.Expect(strings.TrimSpace(readyMessage)).NotTo(BeEmpty()) + + message := strings.ToLower(readyMessage) + actionable := strings.Contains(message, "auth") || + strings.Contains(message, "credential") || + strings.Contains(message, "connect") || + strings.Contains(message, "repository") || + strings.Contains(message, "secret") + g.Expect(actionable).To(BeTrue(), fmt.Sprintf("expected actionable message, got: %q", readyMessage)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) +} + +func (r *quickstartFrameworkRun) gitPull() error { + pullCmd := exec.Command("git", "pull", "--ff-only") + pullCmd.Dir = r.checkoutDir + output, err := pullCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("git pull: %w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func (r *quickstartFrameworkRun) gitCommitCount() (int, error) { + cmd := exec.Command("git", "rev-list", "--count", "--all") + cmd.Dir = r.checkoutDir + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("git rev-list --count: %w: %s", err, strings.TrimSpace(string(output))) + } + + value := strings.TrimSpace(string(output)) + if value == "" { + return 0, nil + } + + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("parse git commit count %q: %w", value, err) + } + return count, nil +} + +func quickstartTimeout() time.Duration { + value := strings.TrimSpace(os.Getenv(quickstartTimeoutSecondsEnv)) + if value == "" { + return 180 * time.Second + } + + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + return 180 * time.Second + } + + return time.Duration(seconds) * time.Second +} + +func createGitProviderWithURLInNamespace(name, ns, branch, secretName, repoURL string) { + By(fmt.Sprintf("creating GitProvider '%s' in ns '%s' with branch '%s', secret '%s' and URL '%s'", + name, ns, branch, secretName, repoURL)) + + data := struct { + Name string + Namespace string + RepoURL string + Branch string + SecretName string + }{ + Name: name, + Namespace: ns, + RepoURL: repoURL, + Branch: branch, + SecretName: secretName, + } + + err := applyFromTemplate("test/e2e/templates/gitprovider.tmpl", data, ns) + Expect(err).NotTo(HaveOccurred(), "failed to apply GitProvider") +} + func quickstartSetupNamespace() string { if value := strings.TrimSpace(os.Getenv(quickstartSetupNamespaceEnv)); value != "" { return value diff --git a/test/e2e/scripts/run-quickstart.sh b/test/e2e/scripts/run-quickstart.sh deleted file mode 100755 index 506d30ba7..000000000 --- a/test/e2e/scripts/run-quickstart.sh +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MODE="${1:-}" -NAMESPACE="${NAMESPACE:-gitops-reverser}" -QUICKSTART_NAMESPACE="${QUICKSTART_NAMESPACE:-sut}" -WAIT_TIMEOUT="${WAIT_TIMEOUT:-60s}" -QUICKSTART_TIMEOUT_SECONDS="${QUICKSTART_TIMEOUT_SECONDS:-180}" - -TEST_ID="$(date +%s)-$RANDOM" -REPO_NAME="quickstart-smoke-${MODE}-${TEST_ID}" -CHECKOUT_DIR="/tmp/gitops-reverser/${REPO_NAME}" - -GIT_PROVIDER_NAME="quickstart-provider-${TEST_ID}" -GIT_TARGET_NAME="quickstart-target-${TEST_ID}" -WATCHRULE_NAME="quickstart-watchrule-${TEST_ID}" -INVALID_PROVIDER_NAME="quickstart-invalid-provider-${TEST_ID}" -ENCRYPTION_SECRET_NAME="quickstart-sops-age-key-${TEST_ID}" -CONFIGMAP_NAME="quickstart-config-${TEST_ID}" -EXPECTED_CONFIGMAP_FILE="${CHECKOUT_DIR}/live-cluster/v1/configmaps/${QUICKSTART_NAMESPACE}/${CONFIGMAP_NAME}.yaml" -SECRET_NAME="quickstart-secret-${TEST_ID}" -SECRET_VALUE_ONE="quickstart-plaintext-one-${TEST_ID}" -SECRET_VALUE_TWO="quickstart-plaintext-two-${TEST_ID}" -EXPECTED_SECRET_FILE="${CHECKOUT_DIR}/live-cluster/v1/secrets/${QUICKSTART_NAMESPACE}/${SECRET_NAME}.sops.yaml" -GITEA_PORT_FORWARD_PID="" - -if [[ -z "${MODE}" ]]; then - echo "usage: $0 " - exit 1 -fi -if [[ "${MODE}" != "helm" && "${MODE}" != "manifest" ]]; then - echo "unsupported mode: ${MODE} (expected helm or manifest)" - exit 1 -fi - -discover_controller_deployment() { - local deployments deploy_count - deployments="$( - kubectl -n "${NAMESPACE}" get deployments -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true - )" - deployments="$(printf '%s\n' "${deployments}" | sed '/^$/d')" - if [[ -z "${deployments}" ]]; then - echo "No Deployments found in namespace ${NAMESPACE}" >&2 - kubectl -n "${NAMESPACE}" get deployments -o wide >&2 || true - return 1 - fi - - deploy_count="$(printf '%s\n' "${deployments}" | wc -l | tr -d ' ')" - if [[ "${deploy_count}" != "1" ]]; then - echo "Expected exactly 1 Deployment in namespace ${NAMESPACE}, found ${deploy_count}" >&2 - kubectl -n "${NAMESPACE}" get deployments -o wide >&2 || true - return 1 - fi - - printf '%s' "${deployments}" -} - -get_controller_pod_selector() { - local deploy selector - deploy="$(discover_controller_deployment 2>/dev/null || true)" - selector="" - if [[ -n "${deploy}" ]]; then - selector="$(kubectl -n "${NAMESPACE}" get deployment "${deploy}" \ - -o jsonpath='{range $k,$v := .spec.selector.matchLabels}{$k}={$v},{end}' 2>/dev/null || true)" - selector="${selector%,}" - fi - - if [[ -z "${selector}" ]]; then - selector="app.kubernetes.io/name=gitops-reverser" - fi - - printf '%s' "${selector}" -} - -print_debug_info() { - local pod_selector deploy - pod_selector="$(get_controller_pod_selector)" - - echo - echo "Install quickstart smoke diagnostics (${MODE})" - echo "Namespace: ${NAMESPACE}" - echo "Quickstart namespace: ${QUICKSTART_NAMESPACE}" - echo "Pod selector: ${pod_selector}" - echo - echo "Deployments:" - kubectl -n "${NAMESPACE}" get deployments -o wide || true - - deploy="$(discover_controller_deployment 2>/dev/null || true)" - if [[ -n "${deploy}" ]]; then - echo - echo "Deployment describe (${deploy}):" - kubectl -n "${NAMESPACE}" describe deployment "${deploy}" || true - fi - echo - echo "Pods:" - kubectl -n "${NAMESPACE}" get pods -o wide || true - echo - echo "Controller-manager pod describe:" - kubectl -n "${NAMESPACE}" describe pod -l "${pod_selector}" || true - echo - echo "Controller-manager logs (last 200 lines):" - kubectl -n "${NAMESPACE}" logs -l "${pod_selector}" --tail=200 --all-containers=true || true - echo - echo "Quickstart resources:" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider,gittarget,watchrule \ - "${GIT_PROVIDER_NAME}" "${GIT_TARGET_NAME}" "${WATCHRULE_NAME}" "${INVALID_PROVIDER_NAME}" 2>/dev/null || true - echo - echo "Quickstart resource status dumps:" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${GIT_PROVIDER_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get gittarget "${GIT_TARGET_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get watchrule "${WATCHRULE_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${INVALID_PROVIDER_NAME}" -o yaml 2>/dev/null || true - echo - echo "Recent namespace events (${NAMESPACE}):" - kubectl -n "${NAMESPACE}" get events --sort-by=.metadata.creationTimestamp | tail -n 50 || true - echo - echo "Recent namespace events (${QUICKSTART_NAMESPACE}):" - kubectl -n "${QUICKSTART_NAMESPACE}" get events --sort-by=.metadata.creationTimestamp | tail -n 50 || true - echo - echo "Git checkout state (${CHECKOUT_DIR}):" - git -C "${CHECKOUT_DIR}" status --short 2>/dev/null || true - git -C "${CHECKOUT_DIR}" --no-pager log --oneline -n 10 2>/dev/null || true -} - -run_or_debug() { - local description="$1" - shift - echo "${description}" - if ! "$@"; then - echo "FAILED: ${description}" >&2 - print_debug_info - return 1 - fi -} - -cleanup_gitea_port_forward() { - if [[ -n "${GITEA_PORT_FORWARD_PID}" ]]; then - kill "${GITEA_PORT_FORWARD_PID}" >/dev/null 2>&1 || true - wait "${GITEA_PORT_FORWARD_PID}" >/dev/null 2>&1 || true - GITEA_PORT_FORWARD_PID="" - fi -} - -ensure_gitea_api_port_forward() { - cleanup_gitea_port_forward - pkill -f "kubectl.*port-forward.*13000:13000" >/dev/null 2>&1 || true - - kubectl -n gitea-e2e port-forward svc/gitea-http 13000:13000 >/tmp/gitea-port-forward.log 2>&1 & - GITEA_PORT_FORWARD_PID="$!" - trap cleanup_gitea_port_forward EXIT - - for _ in {1..30}; do - if curl -s -f "http://localhost:13000/api/v1/version" >/dev/null 2>&1; then - echo "Gitea API port-forward is ready on localhost:13000" - return 0 - fi - sleep 2 - done - - echo "Timed out waiting for Gitea API port-forward readiness" - return 1 -} - -verify_installation() { - local pod_selector deploy - deploy="$(discover_controller_deployment)" - pod_selector="$(get_controller_pod_selector)" - - run_or_debug \ - "Waiting for deployment rollout (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" rollout status "deployment/${deploy}" --timeout="${WAIT_TIMEOUT}" - - run_or_debug \ - "Checking deployment availability (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" wait --for=condition=available "deployment/${deploy}" --timeout="${WAIT_TIMEOUT}" - - run_or_debug \ - "Checking pod readiness (selector=${pod_selector}, timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" wait --for=condition=ready pod -l "${pod_selector}" --timeout="${WAIT_TIMEOUT}" - - echo "Checking CRDs" - kubectl get crd \ - gitproviders.configbutler.ai \ - gittargets.configbutler.ai \ - watchrules.configbutler.ai \ - clusterwatchrules.configbutler.ai >/dev/null - - echo "Checking validating webhook configuration" - kubectl get validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration >/dev/null -} - -wait_for_ready() { - local kind="$1" - local name="$2" - local namespace="$3" - local timeout_seconds="${4}" - local start_epoch now status reason message - start_epoch="$(date +%s)" - - while true; do - status="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - reason="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - message="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" - - if [[ "${status}" == "True" ]]; then - echo "${kind}/${name} Ready=True (${reason})" - return 0 - fi - - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for ${kind}/${name} Ready=True (status=${status} reason=${reason})" - echo "Last message: ${message}" - kubectl -n "${namespace}" get "${kind}" "${name}" -o yaml || true - return 1 - fi - sleep 2 - done -} - -wait_for_connection_failed_actionable_message() { - local name="$1" - local timeout_seconds="${2}" - local start_epoch now status reason message - start_epoch="$(date +%s)" - - while true; do - status="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - reason="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - message="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" - - if [[ "${status}" == "False" && "${reason}" == "ConnectionFailed" && -n "${message}" ]]; then - shopt -s nocasematch - if [[ "${message}" =~ auth|credential|connect|repository|secret ]]; then - shopt -u nocasematch - echo "gitprovider/${name} exposes actionable failure message" - return 0 - fi - shopt -u nocasematch - fi - - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for actionable ConnectionFailed message on gitprovider/${name}" - echo "Last seen status=${status} reason=${reason} message=${message}" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" -o yaml || true - return 1 - fi - sleep 2 - done -} - -git_commit_count() { - git -C "${CHECKOUT_DIR}" rev-list --count --all 2>/dev/null || echo 0 -} - -wait_for_file_exists() { - local file_path="$1" - local timeout_seconds="$2" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]]; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file to exist: ${file_path}" - return 1 - fi - sleep 2 - done -} - -wait_for_file_contains() { - local file_path="$1" - local expected_text="$2" - local timeout_seconds="$3" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]] && grep -Fq "${expected_text}" "${file_path}"; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file content: ${file_path} to contain '${expected_text}'" - return 1 - fi - sleep 2 - done -} - -wait_for_file_absent() { - local file_path="$1" - local timeout_seconds="$2" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ ! -f "${file_path}" ]]; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file to be deleted: ${file_path}" - return 1 - fi - sleep 2 - done -} - -wait_for_file_not_contains() { - local file_path="$1" - local unexpected_text="$2" - local timeout_seconds="$3" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]] && ! grep -Fq "${unexpected_text}" "${file_path}"; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file content: ${file_path} to NOT contain '${unexpected_text}'" - return 1 - fi - sleep 2 - done -} - -extract_generated_age_key() { - local secret_name="$1" - local key_data age_key - - key_data="$( - kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${secret_name}" \ - -o go-template='{{ range $k, $v := .data }}{{ printf "%s=%s\n" $k $v }}{{ end }}' 2>/dev/null | - grep '\.agekey=' | head -n1 | cut -d'=' -f2- || true - )" - if [[ -z "${key_data}" ]]; then - echo "No .agekey entry found in secret ${QUICKSTART_NAMESPACE}/${secret_name}" - return 1 - fi - - age_key="$(printf '%s' "${key_data}" | base64 -d 2>/dev/null | awk '/AGE-SECRET-KEY-/{print; exit}')" - if [[ -z "${age_key}" ]]; then - echo "Failed to decode AGE-SECRET-KEY from secret ${QUICKSTART_NAMESPACE}/${secret_name}" - return 1 - fi - - printf '%s' "${age_key}" -} - -decrypt_file_with_controller_sops() { - local file_path="$1" - local age_key="$2" - local pod_selector pod_name - - pod_selector="$(get_controller_pod_selector)" - pod_name="$( - kubectl -n "${NAMESPACE}" get pod -l "${pod_selector}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true - )" - if [[ -z "${pod_name}" ]]; then - echo "Failed to find controller pod in namespace ${NAMESPACE}" - return 1 - fi - - kubectl -n "${NAMESPACE}" exec -i "${pod_name}" -- \ - env "SOPS_AGE_KEY=${age_key}" \ - /usr/local/bin/sops --decrypt --input-type yaml --output-type yaml /dev/stdin < "${file_path}" -} - -run_quickstart_flow() { - ensure_gitea_api_port_forward - - echo "Setting up quickstart repo and credentials" - run_or_debug \ - "Creating dedicated Gitea repo for quickstart smoke (${REPO_NAME})" \ - env SUT_NAMESPACE="${QUICKSTART_NAMESPACE}" bash test/e2e/scripts/setup-gitea.sh "${REPO_NAME}" "${CHECKOUT_DIR}" - - echo "Applying minimal GitProvider/GitTarget/WatchRule quickstart resources" - cat </dev/null - local warning_anno recipient_anno - warning_anno="$(kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${ENCRYPTION_SECRET_NAME}" \ - -o jsonpath='{.metadata.annotations.configbutler\.ai/backup-warning}')" - recipient_anno="$(kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${ENCRYPTION_SECRET_NAME}" \ - -o jsonpath='{.metadata.annotations.configbutler\.ai/age-recipient}')" - if [[ "${warning_anno}" != "REMOVE_AFTER_BACKUP" ]]; then - echo "Expected backup warning annotation REMOVE_AFTER_BACKUP, got '${warning_anno}'" - return 1 - fi - if [[ "${recipient_anno}" != age1* ]]; then - echo "Expected age recipient annotation to start with age1, got '${recipient_anno}'" - return 1 - fi - - echo "Running create/update/delete quickstart functional checks" - local commits_before_create commits_after_create commits_after_update commits_after_delete - commits_before_create="$(git_commit_count)" - - kubectl -n "${QUICKSTART_NAMESPACE}" delete configmap "${CONFIGMAP_NAME}" --ignore-not-found=true >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" create configmap "${CONFIGMAP_NAME}" --from-literal=value=one >/dev/null - wait_for_file_exists "${EXPECTED_CONFIGMAP_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_contains "${EXPECTED_CONFIGMAP_FILE}" "value: one" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_create="$(git_commit_count)" - if (( commits_after_create <= commits_before_create )); then - echo "Expected commit count to increase after create (${commits_before_create} -> ${commits_after_create})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" patch configmap "${CONFIGMAP_NAME}" --type merge --patch '{"data":{"value":"two"}}' >/dev/null - wait_for_file_contains "${EXPECTED_CONFIGMAP_FILE}" "value: two" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_update="$(git_commit_count)" - if (( commits_after_update <= commits_after_create )); then - echo "Expected commit count to increase after update (${commits_after_create} -> ${commits_after_update})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" delete configmap "${CONFIGMAP_NAME}" --ignore-not-found=true >/dev/null - wait_for_file_absent "${EXPECTED_CONFIGMAP_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_delete="$(git_commit_count)" - if (( commits_after_delete <= commits_after_update )); then - echo "Expected commit count to increase after delete (${commits_after_update} -> ${commits_after_delete})" - return 1 - fi - - echo "Running encrypted Secret commit check" - local commits_before_secret commits_after_secret secret_value_one_b64 secret_value_two_b64 - commits_before_secret="$(git_commit_count)" - secret_value_one_b64="$(printf '%s' "${SECRET_VALUE_ONE}" | base64 | tr -d '\n')" - secret_value_two_b64="$(printf '%s' "${SECRET_VALUE_TWO}" | base64 | tr -d '\n')" - - kubectl -n "${QUICKSTART_NAMESPACE}" delete secret "${SECRET_NAME}" --ignore-not-found=true >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" create secret generic "${SECRET_NAME}" \ - --from-literal=password="${SECRET_VALUE_ONE}" >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" patch secret "${SECRET_NAME}" --type merge \ - --patch "{\"stringData\":{\"password\":\"${SECRET_VALUE_TWO}\"}}" >/dev/null - - wait_for_file_exists "${EXPECTED_SECRET_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_contains "${EXPECTED_SECRET_FILE}" "sops:" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${SECRET_VALUE_ONE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${SECRET_VALUE_TWO}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${secret_value_one_b64}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${secret_value_two_b64}" "${QUICKSTART_TIMEOUT_SECONDS}" - - echo "Validating encrypted Secret is decryptable with generated key" - local generated_age_key decrypted_secret - generated_age_key="$(extract_generated_age_key "${ENCRYPTION_SECRET_NAME}")" - decrypted_secret="$(decrypt_file_with_controller_sops "${EXPECTED_SECRET_FILE}" "${generated_age_key}")" - if ! grep -Fq "${secret_value_two_b64}" <<<"${decrypted_secret}"; then - echo "Expected decrypted secret payload to contain updated value (base64) after patch" - return 1 - fi - - commits_after_secret="$(git_commit_count)" - if (( commits_after_secret <= commits_before_secret )); then - echo "Expected commit count to increase after secret write (${commits_before_secret} -> ${commits_after_secret})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" delete secret "${SECRET_NAME}" --ignore-not-found=true >/dev/null - - echo "Running invalid-credentials quickstart UX check" - cat < I'm sure a lot can be removed and can be picked up by the 'normal' stuff. +The first next step used to be analysing `test/e2e/scripts/run-quickstart.sh`, but quickstart smoke assertions now live +in Go (`test/e2e/quickstart_framework_e2e_test.go`) and the shell harness is retired. It would be really cool to create a new namespace: "run-e2e-full-{same-number-as-repo-basedupon-time}" @@ -26,4 +27,3 @@ It would be really cool to create a new namespace: I'm also wondering if the tests for the quickstart shouldnt be in a go e2e_test.go like file -> in the end we are just doing the same type of assertions. It would be really awesome if we could redo all tests in the proper way. - From 38619381f2a2b3448572fb009802ea7e99a4317b Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 27 Feb 2026 19:15:46 +0000 Subject: [PATCH 09/30] fix: Kind and docker outside docker --- test/e2e/kind/start-cluster.sh | 72 ++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/test/e2e/kind/start-cluster.sh b/test/e2e/kind/start-cluster.sh index be79906a6..1160d3baf 100755 --- a/test/e2e/kind/start-cluster.sh +++ b/test/e2e/kind/start-cluster.sh @@ -9,6 +9,46 @@ CONFIG_FILE="test/e2e/kind/cluster.ignore.yaml" KIND_CREATE_LOG_FILE="${TMPDIR:-/tmp}/kind-create-${CLUSTER_NAME}.log" POD_SUBNET="${KIND_POD_SUBNET:-10.244.0.0/16}" +docker_can_mount_repo() { + local candidate="$1" + docker run --rm -v "${candidate}:/hostproj:ro" busybox:1.36.1 sh -c \ + 'test -f /hostproj/test/e2e/kind/audit/policy.yaml' >/dev/null 2>&1 +} + +resolve_host_project_path() { + local repo_pwd + repo_pwd="$(pwd -P)" + + local candidates=() + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + candidates+=("${HOST_PROJECT_PATH}") + fi + candidates+=("${repo_pwd}") + + local candidate + for candidate in "${candidates[@]}"; do + if docker_can_mount_repo "$candidate"; then + echo "$candidate" + return 0 + fi + done + + echo "❌ ERROR: Unable to determine a usable HOST_PROJECT_PATH for Kind extraMounts." >&2 + echo "Tried:" >&2 + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + echo " - HOST_PROJECT_PATH=${HOST_PROJECT_PATH}" >&2 + else + echo " - HOST_PROJECT_PATH=" >&2 + fi + echo " - pwd=${repo_pwd}" >&2 + echo "" >&2 + echo "Fix: set HOST_PROJECT_PATH to the path visible to the Docker daemon that contains this repo." >&2 + echo "Examples:" >&2 + echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${repo_pwd}'" >&2 + echo " - Docker-outside-of-Docker (daemon on your host): export HOST_PROJECT_PATH=''" >&2 + return 1 +} + create_primary_cluster() { kind create cluster --name "$CLUSTER_NAME" --config "$CONFIG_FILE" --wait 5m --retain 2>&1 | tee "$KIND_CREATE_LOG_FILE" } @@ -20,6 +60,27 @@ is_known_kind_bootstrap_flake() { "$KIND_CREATE_LOG_FILE" } +create_cluster_with_retries() { + local retries="${KIND_CREATE_RETRIES:-2}" + local attempt=0 + + while true; do + attempt=$((attempt + 1)) + if create_primary_cluster; then + return 0 + fi + + if is_known_kind_bootstrap_flake && [ "$attempt" -le "$retries" ]; then + echo "⚠️ Kind bootstrap flake detected; retrying create (attempt ${attempt}/${retries})" + kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true + sleep 2 + continue + fi + + return 1 + done +} + run_dood_self_heal() { local control_plane_container="${CLUSTER_NAME}-control-plane" local container_found=false @@ -73,13 +134,8 @@ run_dood_self_heal() { if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then echo "♻️ Reusing existing Kind cluster '$CLUSTER_NAME' (no delete/recreate)" else - # HOST_PROJECT_PATH is only needed when creating a new cluster (for host-path volume mounts). - # Clusters pre-created by CI (helm/kind-action) don't need it. - if [ -z "${HOST_PROJECT_PATH:-}" ]; then - echo "❌ ERROR: HOST_PROJECT_PATH environment variable is not set" - echo "This should be set in .devcontainer/devcontainer.json or passed by CI" - exit 1 - fi + export HOST_PROJECT_PATH + HOST_PROJECT_PATH="$(resolve_host_project_path)" echo "πŸ”§ Using HOST_PROJECT_PATH: $HOST_PROJECT_PATH" echo "πŸ“ Generating Kind cluster configuration from template..." @@ -89,7 +145,7 @@ else echo "" echo "πŸš€ Creating Kind cluster '$CLUSTER_NAME' with audit webhook support..." - if create_primary_cluster; then + if create_cluster_with_retries; then echo "βœ… Kind cluster created successfully" else if is_known_kind_bootstrap_flake; then From 7668531cdfb80b8740c0033f7ff6b8bb08da45e4 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 07:51:48 +0000 Subject: [PATCH 10/30] chore: Move kubectl calls to functions so that we always can add the --context --- Makefile | 6 +- docs/make-e2e-deps.md | 8 +- ...file-e2e-image-and-deploy-refactor-plan.md | 2 +- docs/makefile-e2e-overview-and-plan.md | 14 +- test/e2e/e2e_suite_test.go | 33 +- test/e2e/e2e_test.go | 364 +++++++++++------- test/e2e/helpers.go | 39 +- test/e2e/kubectl.go | 126 ++++++ test/e2e/quickstart_framework_e2e_test.go | 94 ++--- 9 files changed, 411 insertions(+), 275 deletions(-) create mode 100644 test/e2e/kubectl.go diff --git a/Makefile b/Makefile index 2fd10bff7..4fa715a65 100644 --- a/Makefile +++ b/Makefile @@ -162,13 +162,13 @@ $(CS)/$(NAMESPACE)/namespace.ready: Makefile $(CS)/ready touch $@ # Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. -.PHONY: e2e-prepare -e2e-prepare: $(CS)/$(NAMESPACE)/e2e/prepare portforward-ensure ## Prepare E2E prerequisites for Go tests +.PHONY: prepare-e2e +prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare E2E prerequisites for Go tests # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/e2e/prepare: $(CS)/$(NAMESPACE)/namespace.cleaned \ +$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/namespace.cleaned \ $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index 4c01eebce..c59940be5 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -87,9 +87,9 @@ make test-e2e (phony) └─ go test ./test/e2e └─ BeforeSuite calls Make: make CTX= INSTALL_NAME= NAMESPACE= \ - $(CS)/$(NAMESPACE)/e2e/prepare portforward-ensure + $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure -$(CS)/$(NAMESPACE)/e2e/prepare (stamp) +$(CS)/$(NAMESPACE)/prepare-e2e.ready (stamp) β”œβ”€ $(CS)/sut.namespace.cleaned β”œβ”€ $(CS)/controller.deployed β”œβ”€ $(CS)/portforward.running @@ -289,7 +289,7 @@ which can surface as `metadata.managedFields must be nil` errors. A clean cluste ### CTX override pattern Quickstart targets reuse the generic stamp targets by setting `CTX` before running `go test` (the Go `BeforeSuite` -invokes `make e2e-prepare`, which uses stamp targets under `.stamps/cluster/$(CTX)/...`): +invokes `make prepare-e2e`, which uses stamp targets under `.stamps/cluster/$(CTX)/...`): ```sh CTX=kind-gitops-reverser-test-e2e-quickstart-helm go test ./test/e2e/ ... @@ -324,7 +324,7 @@ The manifest target implicitly validates the generated `dist/install.yaml` via t ### Quickstart smoke lives in Go Quickstart smoke assertions run as a Go E2E test (`quickstart-framework` label). Cluster provisioning, controller install, -and port-forward setup is owned by `make e2e-prepare` (invoked from `BeforeSuite`). +and port-forward setup is owned by `make prepare-e2e` (invoked from `BeforeSuite`). --- diff --git a/docs/makefile-e2e-image-and-deploy-refactor-plan.md b/docs/makefile-e2e-image-and-deploy-refactor-plan.md index 34283fd5a..ac91fbd22 100644 --- a/docs/makefile-e2e-image-and-deploy-refactor-plan.md +++ b/docs/makefile-e2e-image-and-deploy-refactor-plan.md @@ -107,7 +107,7 @@ Container selection strategy (no deployment-name assumptions): ## Dependency graph impact (high level) -- `$(CS)/$(NAMESPACE)/e2e/prepare` should depend on: +- `$(CS)/$(NAMESPACE)/prepare-e2e.ready` should depend on: - `$(CS)/$(NAMESPACE)/namespace.cleaned` - `$(CS)/$(NAMESPACE)/install-$(INSTALL_MODE)` - `$(CS)/image.loaded` diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md index 0aaa85d74..82cc9b6c4 100644 --- a/docs/makefile-e2e-overview-and-plan.md +++ b/docs/makefile-e2e-overview-and-plan.md @@ -14,7 +14,7 @@ This is the high-level roadmap for the Makefile-driven E2E work. For the detaile - There is a stamp-based infra model under `.stamps/cluster//...` and `.stamps/image/...`. - The full-suite Go code shells out to Make in `BeforeSuite`: - - runs `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp; no tests inside) + - runs `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp; no tests inside) - runs `portforward-ensure` to guarantee live port-forwards for the test process - sets `kubectl config use-context $(CTX)` for helpers that don’t pass `--context` - `make test-e2e` runs `go test ./test/e2e/...` and the Go suite owns infra prep via the Make targets above. @@ -26,7 +26,7 @@ This is the high-level roadmap for the Makefile-driven E2E work. For the detaile Go should call exactly one Make target in `BeforeSuite`: -- `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp; no `go test` inside) +- `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp; no `go test` inside) That stamp is responsible for: @@ -78,14 +78,14 @@ Acceptance: - `make test-e2e` runs the suite end-to-end on a prepared cluster. - Quickstart helm+manifest targets work locally without undefined targets. -### PR2 β€” Implement `e2e/prepare` as the single orchestration contract (done) +### PR2 β€” Implement `prepare-e2e.ready` as the single orchestration contract (done) -- Implement `$(CS)/$(NAMESPACE)/e2e/prepare` (stamp) as the β€œone target Go calls”. -- Update Go `BeforeSuite` to call `e2e/prepare` (and `portforward-ensure`) and set kubectl context. +- Implement `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp) as the β€œone target Go calls”. +- Update Go `BeforeSuite` to call `prepare-e2e.ready` (and `portforward-ensure`) and set kubectl context. Acceptance: -- `go test ./test/e2e/...` works when invoked from CI/Make and always provisions infra via `e2e/prepare`. +- `go test ./test/e2e/...` works when invoked from CI/Make and always provisions infra via `prepare-e2e.ready`. ### PR3 β€” Run scoping via seed @@ -118,7 +118,7 @@ Plan: `docs/makefile-e2e-image-and-deploy-refactor-plan.md`. ## Notes / non-goals (for now) -- Don’t expand the suite/install matrix until the `e2e/prepare` contract and run scoping are solid. +- Don’t expand the suite/install matrix until the `prepare-e2e.ready` contract and run scoping are solid. - Avoid moving Kubernetes orchestration into Go; prefer Make stamps and verified gates. ## Why `manifests` and `helm-sync` are `.PHONY` diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index bf4e705fc..b41c7da96 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -58,6 +58,7 @@ var _ = AfterSuite(func() { func ensureE2EPrepared() { ctx := resolveE2EContext() + setE2EKubectlContext(ctx) seed := ginkgoRandomSeed() ns := resolveE2ENamespace() installName := resolveE2EInstallName(seed) @@ -65,27 +66,23 @@ func ensureE2EPrepared() { fmt.Sprintf("CTX=%s", ctx), fmt.Sprintf("INSTALL_NAME=%s", installName), fmt.Sprintf("NAMESPACE=%s", ns), - "e2e-prepare", + "prepare-e2e", ) output, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "failed to run make target for e2e prepare") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) - // Many e2e helpers invoke kubectl without an explicit --context flag. Ensure kubectl is pointed at the - // intended cluster context for the remainder of the test run. - useCtx := exec.Command("kubectl", "config", "use-context", ctx) - output, err = utils.Run(useCtx) + // Some e2e shell scripts call `kubectl` without an explicit `--context` flag. Ensure `kubectl` is + // pointed at the intended cluster context for the remainder of the test run. + output, err = kubectlRun("config", "use-context", ctx) Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) By("ensuring IceCreamOrder CRD is removed before tests") - deleteIceCreamOrderCRD := exec.Command( - "kubectl", - "--context", ctx, + output, err = kubectlRun( "delete", "crd", "icecreamorders.shop.example.com", "--ignore-not-found=true", ) - output, err = utils.Run(deleteIceCreamOrderCRD) Expect(err).NotTo(HaveOccurred(), "failed to delete IceCreamOrder CRD before tests") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) @@ -133,8 +130,7 @@ func resolveE2EContext() string { return fmt.Sprintf("kind-%s", cluster) } - cmd := exec.Command("kubectl", "config", "current-context") - output, err := utils.Run(cmd) + output, err := kubectlRun("config", "current-context") if err == nil { if value := strings.TrimSpace(output); value != "" { return value @@ -143,18 +139,3 @@ func resolveE2EContext() string { return "kind-gitops-reverser-test-e2e" } - -/* - -At startup of tests we should define the seed of the test run and use that to create a namespace and to run tests in it, also the build should be depending on that. - -We should be cominbing instllation methods with testsuites - -helm -manifests -config (is there still use for this?!) - -full -quickstart - -*/ diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 9b26a7c97..05e444c72 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,6 +20,7 @@ package e2e import ( "bytes" + "context" "encoding/base64" "encoding/json" "fmt" @@ -98,8 +99,12 @@ var _ = Describe("Manager", Ordered, func() { specReport := CurrentSpecReport() if specReport.Failed() { By("Fetching controller manager pod logs") - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace, "--tail=200") - controllerLogs, err := utils.Run(cmd) + controllerLogs, err := kubectlRunInNamespace( + namespace, + "logs", + controllerPodName, + "--tail=200", + ) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Last 200 lines of controller logs:\n %s", controllerLogs) } else { @@ -107,8 +112,12 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching Kubernetes events") - cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.metadata.creationTimestamp") - eventsOutput, err := utils.Run(cmd) + eventsOutput, err := kubectlRunInNamespace( + namespace, + "get", + "events", + "--sort-by=.metadata.creationTimestamp", + ) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events (newest last):\n%s", eventsOutput) } else { @@ -116,8 +125,7 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching controller manager pod description") - cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) - podDescription, err := utils.Run(cmd) + podDescription, err := kubectlRunInNamespace(namespace, "describe", "pod", controllerPodName) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Pod description:\n%s", podDescription) } else { @@ -136,16 +144,17 @@ var _ = Describe("Manager", Ordered, func() { By("validating that the gitops-reverser pods are running as expected") verifyControllerUp := func(g Gomega) { // Get the names of the gitops-reverser pods - cmd := exec.Command("kubectl", "get", - "pods", "-l", "control-plane=gitops-reverser", + podOutput, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + "-l", + "control-plane=gitops-reverser", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ "{{ .metadata.name }}"+ "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, ) - - podOutput, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve gitops-reverser pod information") podNames := utils.GetNonEmptyLines(podOutput) g.Expect(podNames).To(HaveLen(1), "expected exactly 1 controller pod running") @@ -154,11 +163,14 @@ var _ = Describe("Manager", Ordered, func() { // Validate all pods' status for _, podName := range podNames { - cmd = exec.Command("kubectl", "get", - "pods", podName, "-o", "jsonpath={.status.phase}", - "-n", namespace, + output, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + podName, + "-o", + "jsonpath={.status.phase}", ) - output, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("Running"), fmt.Sprintf("Incorrect status for pod %s", podName)) } @@ -170,10 +182,14 @@ var _ = Describe("Manager", Ordered, func() { By("verifying controller service selects the running controller pod") verifyWebhookService := func(g Gomega) { // Get controller service endpoints - cmd := exec.Command("kubectl", "get", "endpoints", - controllerServiceName, "-n", namespace, - "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "get", + "endpoints", + controllerServiceName, + "-o", + "jsonpath={.subsets[*].addresses[*].targetRef.name}", + ) g.Expect(err).NotTo(HaveOccurred(), "Failed to get controller service endpoints") // Filter out kubectl deprecation warnings from output @@ -203,16 +219,19 @@ var _ = Describe("Manager", Ordered, func() { It("should expose admission and audit ports on one controller service", func() { By("verifying controller service exists") - cmd := exec.Command("kubectl", "get", "svc", controllerServiceName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) Expect(err).NotTo(HaveOccurred(), "Controller service should exist") By("verifying controller service routes to the controller pod") Eventually(func(g Gomega) { - endpointsCmd := exec.Command("kubectl", "get", "endpoints", - controllerServiceName, "-n", namespace, - "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}") - output, endpointsErr := utils.Run(endpointsCmd) + output, endpointsErr := kubectlRunInNamespace( + namespace, + "get", + "endpoints", + controllerServiceName, + "-o", + "jsonpath={.subsets[*].addresses[*].targetRef.name}", + ) g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get controller service endpoints") lines := utils.GetNonEmptyLines(output) @@ -239,8 +258,7 @@ var _ = Describe("Manager", Ordered, func() { verifyWebhook := func(g Gomega) { jsonPath := "jsonpath={.items[?(@.metadata.name=='gitops-reverser-validating-webhook-configuration')]" + ".webhooks[*].name}" - cmd := exec.Command("kubectl", "get", "validatingwebhookconfigurations", "-o", jsonPath) - output, err := utils.Run(cmd) + output, err := kubectlRun("get", "validatingwebhookconfigurations", "-o", jsonPath) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("gitops-reverser.configbutler.ai"), "Event webhook should be registered") @@ -250,14 +268,12 @@ var _ = Describe("Manager", Ordered, func() { It("should ensure the metrics endpoint is serving metrics", func() { By("validating that the controller service is available for metrics") - cmd := exec.Command("kubectl", "get", "service", controllerServiceName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "get", "service", controllerServiceName) Expect(err).NotTo(HaveOccurred(), "Controller service should exist") By("waiting for the metrics endpoint to be ready") verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", controllerServiceName, "-n", namespace) - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "get", "endpoints", controllerServiceName) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") } @@ -265,8 +281,7 @@ var _ = Describe("Manager", Ordered, func() { By("verifying that the controller manager is serving the metrics server") verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "logs", controllerPodName) g.Expect(err).NotTo(HaveOccurred()) jsonMetricsLogLine := "\"logger\":\"controller-runtime.metrics\"," + "\"msg\":\"Serving metrics server\"" @@ -306,10 +321,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("πŸ“Š Baseline events: %.0f\n", baselineEvents) By("creating a ConfigMap to trigger webhook call") - cmd := exec.Command("kubectl", "create", "configmap", "webhook-test-cm", - "--namespace", namespace, - "--from-literal=test=webhook") - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + "webhook-test-cm", + "--from-literal=test=webhook", + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed with working webhook") By("waiting for webhook event metric to increment") @@ -327,8 +345,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("πŸ“Š Inspect metrics: %s\n", getPrometheusURL()) By("cleaning up webhook test resources") - cmd = exec.Command("kubectl", "delete", "configmap", "webhook-test-cm", "--namespace", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + "webhook-test-cm", + "--ignore-not-found=true", + ) }) It("should receive audit webhook events from kube-apiserver", func() { @@ -343,16 +366,23 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("πŸ“Š Baseline kind-e2e audit events: %.0f\n", baselineClusterAuditEvents) By("creating a ConfigMap to trigger audit events") - cmd := exec.Command("kubectl", "create", "configmap", "audit-test-cm", - "--namespace", namespace, - "--from-literal=test=audit") - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + "audit-test-cm", + "--from-literal=test=audit", + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed") - cmd = exec.Command("kubectl", "patch", "configmap", "audit-test-cm", - "--namespace", namespace, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "configmap", + "audit-test-cm", "--type=merge", - "--patch", `{"data":{"test":"audit-updated"}}`) - _, err = utils.Run(cmd) + "--patch", + `{"data":{"test":"audit-updated"}}`, + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap update should succeed") By("waiting for audit event metric to increment") @@ -376,8 +406,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("πŸ“Š Total audit events: %.0f\n", currentAuditEvents) By("cleaning up audit test resources") - cmd = exec.Command("kubectl", "delete", "configmap", "audit-test-cm", "--namespace", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + "audit-test-cm", + "--ignore-not-found=true", + ) }) It("should validate GitProvider with real Gitea repository", func() { @@ -431,8 +466,7 @@ var _ = Describe("Manager", Ordered, func() { showControllerLogs("before SSH test") By("πŸ“‹ Checking SSH secret exists") - cmd := exec.Command("kubectl", "get", "secret", "git-creds-ssh", "-n", namespace, "-o", "yaml") - secretOutput, err := utils.Run(cmd) + secretOutput, err := kubectlRunInNamespace(namespace, "get", "secret", "git-creds-ssh", "-o", "yaml") if err != nil { fmt.Printf("❌ SSH secret not found: %v\n", err) } else { @@ -518,20 +552,28 @@ var _ = Describe("Manager", Ordered, func() { verifyResourceStatus("gittarget", destName, namespace, "True", "Ready", "") By("creating Secret in watched namespace") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") - cmd := exec.Command("kubectl", "create", "secret", "generic", secretName, - "--from-literal=password=do-not-commit", "-n", namespace) - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") By("patching Secret once to avoid informer start race and force an update event") - patchCmd := exec.Command( - "kubectl", "patch", "secret", secretName, "-n", namespace, - "--type=merge", "--patch", `{"stringData":{"password":"never-commit-this"}}`, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"never-commit-this"}}`, ) - _, err = utils.Run(patchCmd) Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") By("verifying Secret file is committed and does not contain plaintext data") @@ -571,8 +613,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) By("cleaning up test resources") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) }) @@ -584,8 +625,13 @@ var _ = Describe("Manager", Ordered, func() { generatedSecretName := "sops-age-key-autogen" By("ensuring generated encryption secret does not exist before test") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", generatedSecretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) By("creating GitTarget with age recipient auto-generation enabled") destName := watchRuleName + "-dest" @@ -617,8 +663,14 @@ var _ = Describe("Manager", Ordered, func() { var generatedAgeKey string var generatedRecipient string Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "secret", generatedSecretName, "-n", namespace, "-o", "json") - output, getErr := utils.Run(cmd) + output, getErr := kubectlRunInNamespace( + namespace, + "get", + "secret", + generatedSecretName, + "-o", + "json", + ) g.Expect(getErr).NotTo(HaveOccurred()) var secretObj map[string]interface{} @@ -668,19 +720,27 @@ var _ = Describe("Manager", Ordered, func() { }, "30s", "2s").Should(Succeed()) By("creating Secret in watched namespace") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) - cmd := exec.Command("kubectl", "create", "secret", "generic", secretName, - "--from-literal=password=do-not-commit", "-n", namespace) - _, err = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") + _, err = kubectlRunInNamespace( + namespace, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") By("patching Secret once to avoid informer start race and force an update event") - patchCmd := exec.Command( - "kubectl", "patch", "secret", secretName, "-n", namespace, - "--type=merge", "--patch", `{"stringData":{"password":"autogen-never-commit-this"}}`, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"autogen-never-commit-this"}}`, ) - _, err = utils.Run(patchCmd) Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") By("verifying committed secret is encrypted and decryptable with generated key") @@ -716,10 +776,14 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) By("cleaning up test resources") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", generatedSecretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) }) @@ -771,9 +835,14 @@ var _ = Describe("Manager", Ordered, func() { By("waiting for controller reconciliation of ConfigMap event") verifyReconciliationLogs := func(g Gomega) { // Get controller logs from all pods (single-pod mode still uses label selector). - cmd := exec.Command("kubectl", "logs", "-l", "control-plane=gitops-reverser", - "-n", namespace, "--tail=500", "--prefix=true") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "logs", + "-l", + "control-plane=gitops-reverser", + "--tail=500", + "--prefix=true", + ) g.Expect(err).NotTo(HaveOccurred()) // Check for git commit operation in logs @@ -842,8 +911,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyGitCommit).Should(Succeed()) By("cleaning up test resources") - cmd := exec.Command("kubectl", "delete", "configmap", configMapName, "-n", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "configmap", configMapName, "--ignore-not-found=true") cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) @@ -912,8 +980,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileCreated).Should(Succeed()) By("deleting the ConfigMap to trigger DELETE operation") - cmd := exec.Command("kubectl", "delete", "configmap", configMapName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "delete", "configmap", configMapName) Expect(err).NotTo(HaveOccurred(), "ConfigMap deletion should succeed") By("verifying ConfigMap file is deleted from Git repository") @@ -983,15 +1050,18 @@ var _ = Describe("Manager", Ordered, func() { verifyResourceStatus("gittarget", destName, namespace, "True", "Ready", "") By("installing the IceCreamOrder CRD to trigger Git commit") - cmd := exec.Command("kubectl", "apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - _, err = utils.Run(cmd) + _, err = kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") Expect(err).NotTo(HaveOccurred(), "Failed to install CRD") By("waiting for CRD to be established") verifyCRDEstablished := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "crd", crdName, - "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}") - output, err := utils.Run(cmd) + output, err := kubectlRun( + "get", + "crd", + crdName, + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("True")) } @@ -1041,15 +1111,18 @@ var _ = Describe("Manager", Ordered, func() { watchRuleName := "watchrule-icecream-orders" By("installing the IceCreamOrder CRD first (needed for custom resource tests)") - cmd := exec.Command("kubectl", "apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - _, err := utils.Run(cmd) + _, err := kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") Expect(err).NotTo(HaveOccurred(), "Failed to install sample CRD") By("waiting for CRD to be established") verifyCRDEstablished := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "crd", "icecreamorders.shop.example.com", - "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}") - output, err := utils.Run(cmd) + output, err := kubectlRun( + "get", + "crd", + "icecreamorders.shop.example.com", + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("True")) } @@ -1122,9 +1195,14 @@ var _ = Describe("Manager", Ordered, func() { By("waiting for controller reconciliation of CRD instance event") verifyReconciliationLogs := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", "-l", "control-plane=gitops-reverser", - "-n", namespace, "--tail=500", "--prefix=true") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "logs", + "-l", + "control-plane=gitops-reverser", + "--tail=500", + "--prefix=true", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("git commit"), "Should see git commit operation in logs") @@ -1190,9 +1268,16 @@ var _ = Describe("Manager", Ordered, func() { By("applying status update to the IceCreamOrder CR") statusPatch := `{"status":{"orderStatus":"pending","preparationTime":"5m","totalPrice":"$12.50"}}` - statusCmd := exec.Command("kubectl", "patch", "icecreamorder", crdInstanceName, - "-n", namespace, "--type=merge", "--subresource=status", "-p", statusPatch) - statusOutput, statusErr := utils.Run(statusCmd) + statusOutput, statusErr := kubectlRunInNamespace( + namespace, + "patch", + "icecreamorder", + crdInstanceName, + "--type=merge", + "--subresource=status", + "-p", + statusPatch, + ) if statusErr != nil { // Status subresource might not be configured for this CRD, which is fine for this test By(fmt.Sprintf("⚠️ Status patch not supported (expected): %v", statusErr)) @@ -1239,8 +1324,7 @@ var _ = Describe("Manager", Ordered, func() { By("βœ… Status update verified - no Git commit created and status not in file") By("cleaning up IceCreamOrder instance") - cmd2 := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, _ = utils.Run(cmd2) + _, _ = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") @@ -1355,8 +1439,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyUpdatedFile).Should(Succeed()) By("cleaning up IceCreamOrder instance") - cmd := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") @@ -1418,8 +1501,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileCreated).Should(Succeed()) By("deleting the CR to trigger DELETE operation") - cmd := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) Expect(err).NotTo(HaveOccurred(), "CRD instance deletion should succeed") By("verifying CRD instance file is deleted from Git repository") @@ -1493,8 +1575,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileExists).Should(Succeed()) By("deleting the CRD to trigger DELETE operation") - deleteCmd := exec.Command("kubectl", "delete", "crd", crdName) - _, deleteErr := utils.Run(deleteCmd) + _, deleteErr := kubectlRun("delete", "crd", crdName) Expect(deleteErr).NotTo(HaveOccurred(), "CRD deletion should succeed") By("verifying CRD file is deleted from Git repository") verifyFileDeleted := func(g Gomega) { @@ -1539,9 +1620,12 @@ var _ = Describe("Manager", Ordered, func() { cleanupGitProvider("gitprovider-normal") // Clean up IceCreamOrder CRD - cmd := exec.Command("kubectl", "delete", "crd", - "icecreamorders.shop.example.com", "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRun( + "delete", + "crd", + "icecreamorders.shop.example.com", + "--ignore-not-found=true", + ) }) // +kubebuilder:scaffold:e2e-webhooks-checks @@ -1585,9 +1669,13 @@ func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) return "", err } - cmd := exec.Command( - "kubectl", "exec", "-i", - podName, "-n", namespace, "--", + cmd := kubectlCmdInNamespace( + context.Background(), + namespace, + "exec", + "-i", + podName, + "--", "env", fmt.Sprintf("SOPS_AGE_KEY=%s", ageKey), "/usr/local/bin/sops", "--decrypt", "--input-type", "yaml", "--output-type", "yaml", "/dev/stdin", ) @@ -1601,16 +1689,13 @@ func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) } func discoverControllerPodName(ns string) (string, error) { - deploymentsCmd := exec.Command( - "kubectl", + deploymentsOutput, err := kubectlRunInNamespace( + ns, "get", "deployments", - "-n", - ns, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}", ) - deploymentsOutput, err := utils.Run(deploymentsCmd) if err != nil { return "", fmt.Errorf("get deployments in namespace %s: %w", ns, err) } @@ -1621,8 +1706,7 @@ func discoverControllerPodName(ns string) (string, error) { } deploymentName := deployments[0] - deploymentCmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", ns, "-o", "json") - deploymentOutput, err := utils.Run(deploymentCmd) + deploymentOutput, err := kubectlRunInNamespace(ns, "get", "deployment", deploymentName, "-o", "json") if err != nil { return "", fmt.Errorf("get deployment %s/%s: %w", ns, deploymentName, err) } @@ -1656,18 +1740,15 @@ func discoverControllerPodName(ns string) (string, error) { } selector := strings.Join(selectorParts, ",") - podCmd := exec.Command( - "kubectl", + podOutput, err := kubectlRunInNamespace( + ns, "get", "pods", - "-n", - ns, "-l", selector, "-o", "jsonpath={.items[0].metadata.name}", ) - podOutput, err := utils.Run(podCmd) if err != nil { return "", fmt.Errorf("get controller pod for selector %q in namespace %s: %w", selector, ns, err) } @@ -1736,8 +1817,7 @@ func verifyResourceStatus(resourceType, name, ns, expectedStatus, expectedReason args = append(args, "-n", ns) } - cmd := exec.Command("kubectl", args...) - output, err := utils.Run(cmd) + output, err := kubectlRun(args...) g.Expect(err).NotTo(HaveOccurred()) var obj unstructured.Unstructured @@ -1776,8 +1856,13 @@ func verifyResourceStatus(resourceType, name, ns, expectedStatus, expectedReason // cleanupGitProvider deletes a GitProvider resource. func cleanupGitProvider(name string) { By(fmt.Sprintf("cleaning up GitProvider '%s'", name)) - cmd := exec.Command("kubectl", "delete", "gitprovider", name, "-n", namespace, "--ignore-not-found=true") - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace( + namespace, + "delete", + "gitprovider", + name, + "--ignore-not-found=true", + ) Expect(err).NotTo(HaveOccurred()) } @@ -1786,9 +1871,15 @@ func showControllerLogs(context string) { By(fmt.Sprintf("πŸ“‹ Controller logs %s:", context)) // Get the controller pod name dynamically - cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=gitops-reverser", - "-o", "jsonpath={.items[0].metadata.name}", "-n", namespace) - podName, err := utils.Run(cmd) + podName, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + "-l", + "control-plane=gitops-reverser", + "-o", + "jsonpath={.items[0].metadata.name}", + ) if err != nil { fmt.Printf("⚠️ Failed to get controller pod name: %v\n", err) return @@ -1800,8 +1891,7 @@ func showControllerLogs(context string) { } // Get the logs - cmd = exec.Command("kubectl", "logs", strings.TrimSpace(podName), "-n", namespace, "--tail=20") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "logs", strings.TrimSpace(podName), "--tail=20") if err != nil { fmt.Printf("❌ Failed to get controller logs: %v\n", err) return diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index a66289842..80ba1c2d4 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -26,8 +26,6 @@ import ( "bytes" "context" "fmt" - "os/exec" - "strings" "text/template" "time" @@ -36,8 +34,6 @@ import ( "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" - - "github.com/ConfigButler/gitops-reverser/test/utils" ) // namespace where the project is deployed in. @@ -147,24 +143,8 @@ func applyFromTemplate(templatePath string, data interface{}, namespace string, return err } - // 1. Define the binary explicitly - const binary = "kubectl" - - args := []string{ - "apply", "-f", "-", - } - - // Add extra arguments - args = append(args, extraArgs...) - - if namespace != "" { - args = append(args, "-n", namespace) - } - - ctx := context.Background() - cmd := exec.CommandContext(ctx, binary, args...) - cmd.Stdin = strings.NewReader(yamlContent) - _, err = utils.Run(cmd) + args := append([]string{"apply", "-f", "-"}, extraArgs...) + _, err = kubectlRunWithStdin(namespace, yamlContent, args...) return err } @@ -225,26 +205,17 @@ func createGitTargetWithEncryptionOptions( // cleanupGitTarget deletes a GitTarget resource. func cleanupGitTarget(name, namespace string) { By(fmt.Sprintf("cleaning up GitTarget '%s' in ns '%s'", name, namespace)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "gittarget", name, - "-n", namespace, "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "gittarget", name, "--ignore-not-found=true") } // cleanupWatchRule deletes a WatchRule resource. func cleanupWatchRule(name, namespace string) { By(fmt.Sprintf("cleaning up WatchRule '%s' in ns '%s'", name, namespace)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "watchrule", name, - "-n", namespace, "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "watchrule", name, "--ignore-not-found=true") } // cleanupClusterWatchRule deletes a ClusterWatchRule resource. func cleanupClusterWatchRule(name string) { By(fmt.Sprintf("cleaning up ClusterWatchRule '%s'", name)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterwatchrule", name, - "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRun("delete", "clusterwatchrule", name, "--ignore-not-found=true") } diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go new file mode 100644 index 000000000..c76ac05ff --- /dev/null +++ b/test/e2e/kubectl.go @@ -0,0 +1,126 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + "os/exec" + "strings" + + "github.com/ConfigButler/gitops-reverser/test/utils" +) + +var e2eKubectlContext string //nolint:gochecknoglobals // set once in suite, used across e2e helpers + +const kubectlNamespaceArgCount = 2 + +func setE2EKubectlContext(ctx string) { + e2eKubectlContext = strings.TrimSpace(ctx) +} + +func kubectlContext() string { + if value := strings.TrimSpace(e2eKubectlContext); value != "" { + return value + } + + // Keep compatibility with existing Makefile/shell usage. + if value := strings.TrimSpace(os.Getenv("CTX")); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("KUBE_CONTEXT")); value != "" { + return value + } + + return "" +} + +func kubectlArgs(args ...string) []string { + ctx := kubectlContext() + if ctx == "" { + return args + } + return append([]string{"--context", ctx}, args...) +} + +func kubectlArgsInNamespace(namespace string, args ...string) []string { + if strings.TrimSpace(namespace) == "" { + return kubectlArgs(args...) + } + + withCtx := kubectlArgs(args...) + if hasNamespaceArg(withCtx) { + return withCtx + } + + // Ensure we don't append namespace after a "--" separator (e.g., `kubectl exec ... -- ...`), + // since anything after "--" is passed to the executed command. + doubleDashIdx := -1 + for i, arg := range withCtx { + if arg == "--" { + doubleDashIdx = i + break + } + } + if doubleDashIdx == -1 { + return append(withCtx, "-n", namespace) + } + + updated := make([]string, 0, len(withCtx)+kubectlNamespaceArgCount) + updated = append(updated, withCtx[:doubleDashIdx]...) + updated = append(updated, "-n", namespace) + updated = append(updated, withCtx[doubleDashIdx:]...) + return updated +} + +func hasNamespaceArg(args []string) bool { + for _, arg := range args { + if arg == "-n" || arg == "--namespace" { + return true + } + if strings.HasPrefix(arg, "--namespace=") { + return true + } + } + return false +} + +func kubectlCmd(ctx context.Context, args ...string) *exec.Cmd { + //nolint:gosec // e2e helper intentionally shells out to kubectl + return exec.CommandContext(ctx, "kubectl", kubectlArgs(args...)...) +} + +func kubectlCmdInNamespace(ctx context.Context, namespace string, args ...string) *exec.Cmd { + //nolint:gosec // e2e helper intentionally shells out to kubectl + return exec.CommandContext(ctx, "kubectl", kubectlArgsInNamespace(namespace, args...)...) +} + +func kubectlRun(args ...string) (string, error) { + return utils.Run(kubectlCmd(context.Background(), args...)) +} + +func kubectlRunInNamespace(namespace string, args ...string) (string, error) { + return utils.Run(kubectlCmdInNamespace(context.Background(), namespace, args...)) +} + +func kubectlRunWithStdin(namespace, stdin string, args ...string) (string, error) { + cmd := kubectlCmdInNamespace(context.Background(), namespace, args...) + cmd.Stdin = strings.NewReader(stdin) + return utils.Run(cmd) +} diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go index 5b821b03c..710f60011 100644 --- a/test/e2e/quickstart_framework_e2e_test.go +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -194,8 +194,7 @@ func (r *quickstartFrameworkRun) verifyGeneratedEncryptionSecret() string { var generatedAgeKey string Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "secret", r.encryptionName, "-n", ns, "-o", "json") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(ns, "get", "secret", r.encryptionName, "-o", "json") g.Expect(err).NotTo(HaveOccurred()) var secretObj map[string]interface{} @@ -241,28 +240,17 @@ func (r *quickstartFrameworkRun) verifyQuickstartConfigMapCommits() { fmt.Sprintf("%s.yaml", configMapName), ) - _, _ = utils.Run(exec.Command( - "kubectl", - "delete", - "configmap", - configMapName, - "-n", - ns, - "--ignore-not-found=true", - )) + _, _ = kubectlRunInNamespace(ns, "delete", "configmap", configMapName, "--ignore-not-found=true") commitsBefore, err := r.gitCommitCount() Expect(err).NotTo(HaveOccurred()) - _, err = utils.Run(exec.Command( - "kubectl", - "create", - "configmap", + _, err = kubectlRunInNamespace( + ns, + "create", "configmap", configMapName, "--from-literal=value=one", - "-n", - ns, - )) + ) Expect(err).NotTo(HaveOccurred(), "failed to create quickstart ConfigMap") Eventually(func(g Gomega) { @@ -280,18 +268,15 @@ func (r *quickstartFrameworkRun) verifyQuickstartConfigMapCommits() { commitsAfterCreate, err := r.gitCommitCount() Expect(err).NotTo(HaveOccurred()) - _, err = utils.Run(exec.Command( - "kubectl", - "patch", - "configmap", - configMapName, - "-n", + _, err = kubectlRunInNamespace( ns, + "patch", "configmap", + configMapName, "--type", "merge", "--patch", `{"data":{"value":"two"}}`, - )) + ) Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart ConfigMap") Eventually(func(g Gomega) { @@ -309,14 +294,11 @@ func (r *quickstartFrameworkRun) verifyQuickstartConfigMapCommits() { commitsAfterUpdate, err := r.gitCommitCount() Expect(err).NotTo(HaveOccurred()) - _, err = utils.Run(exec.Command( - "kubectl", - "delete", - "configmap", - configMapName, - "-n", + _, err = kubectlRunInNamespace( ns, - )) + "delete", "configmap", + configMapName, + ) Expect(err).NotTo(HaveOccurred(), "failed to delete quickstart ConfigMap") Eventually(func(g Gomega) { @@ -352,41 +334,31 @@ func (r *quickstartFrameworkRun) verifyQuickstartSecretEncryption(generatedAgeKe commitsBefore, err := r.gitCommitCount() Expect(err).NotTo(HaveOccurred()) - _, _ = utils.Run(exec.Command( - "kubectl", - "delete", - "secret", - secretName, - "-n", + _, _ = kubectlRunInNamespace( ns, + "delete", "secret", + secretName, "--ignore-not-found=true", - )) + ) - _, err = utils.Run(exec.Command( - "kubectl", - "create", - "secret", - "generic", + _, err = kubectlRunInNamespace( + ns, + "create", "secret", "generic", secretName, "--from-literal", fmt.Sprintf("password=%s", secretValueOne), - "-n", - ns, - )) + ) Expect(err).NotTo(HaveOccurred(), "failed to create quickstart Secret") - _, err = utils.Run(exec.Command( - "kubectl", - "patch", - "secret", - secretName, - "-n", + _, err = kubectlRunInNamespace( ns, + "patch", "secret", + secretName, "--type", "merge", "--patch", fmt.Sprintf(`{"stringData":{"password":"%s"}}`, secretValueTwo), - )) + ) Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart Secret") Eventually(func(g Gomega) { @@ -409,22 +381,18 @@ func (r *quickstartFrameworkRun) verifyQuickstartSecretEncryption(generatedAgeKe g.Expect(commitsAfter).To(BeNumerically(">", commitsBefore)) }, quickstartTimeout(), 2*time.Second).Should(Succeed()) - _, _ = utils.Run(exec.Command( - "kubectl", - "delete", - "secret", - secretName, - "-n", + _, _ = kubectlRunInNamespace( ns, + "delete", "secret", + secretName, "--ignore-not-found=true", - )) + ) } func (r *quickstartFrameworkRun) verifyInvalidProviderActionableMessage() { ns := quickstartSetupNamespace() Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "gitprovider", r.invalidProvName, "-n", ns, "-o", "json") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(ns, "get", "gitprovider", r.invalidProvName, "-o", "json") g.Expect(err).NotTo(HaveOccurred()) var obj unstructured.Unstructured From 831127cbdc35a22853c07ad4261b8ec2b7affa09 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 09:37:54 +0000 Subject: [PATCH 11/30] chore: Stripping unneeded stuff, and allowing to run the three scenarios faster in local setup (no need to recreate the cluster every time). --- Makefile | 71 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 4fa715a65..24ff86e38 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,8 @@ GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum INSTALL_MODE ?= config-dir INSTALL_NAME ?= gitops-reverser NAMESPACE ?= gitops-reverser +E2E_NAMESPACE_LABEL_KEY ?= e2e +E2E_NAMESPACE_LABEL_VALUE ?= true KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) @@ -146,20 +148,6 @@ cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps rm -rf .stamps/cluster/kind-$$cluster; \ done -# We delete all services (over all namespaces) since we only can have ONE service listening to the cluster audit webhook at the same time -$(CS)/$(NAMESPACE)/namespace.cleaned: Makefile $(CS)/$(NAMESPACE)/namespace.ready - kubectl --context $(CTX) delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true - touch $@ - -$(CS)/$(NAMESPACE)/namespace.ready: Makefile $(CS)/ready - mkdir -p $(@D) - kubectl --context $(CTX) create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl --context $(CTX) apply -f - - kubectl --context $(CTX) label --overwrite ns $(NAMESPACE) \ - app.kubernetes.io/managed-by=kustomize app.kubernetes.io/name=gitops-reverser control-plane=gitops-reverser \ - pod-security.kubernetes.io/enforce=restricted - touch $@ # Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. .PHONY: prepare-e2e @@ -168,8 +156,8 @@ prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/namespace.cleaned \ - $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ +$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ + $(CS)/$(NAMESPACE)/label-namespace.ready \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ $(CS)/$(NAMESPACE)/age-key.applied @@ -315,7 +303,7 @@ $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-opera kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ -$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/namespace.ready test/e2e/tools/gen-age-key/main.go +$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) test/e2e/tools/gen-age-key/main.go mkdir -p $(@D) go run ./test/e2e/tools/gen-age-key \ --key-file $(CS)/age-key.txt \ @@ -429,7 +417,7 @@ CONFIG_DIR_INPUTS := $(shell find config -type f) ##@ Deployments of all manifests needed to run $(CS)/$(NAMESPACE)/install-helm: INSTALL_MODE := helm -$(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via Helm into CTX cluster +$(CS)/$(NAMESPACE)/install-helm: $(CS)/$(NAMESPACE)/cleanup-installs.done $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) \ helm_args=( \ @@ -449,8 +437,7 @@ $(CS)/$(NAMESPACE)/install-helm: $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.install touch $@ $(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(CS)/$(NAMESPACE)/namespace.ready ## Deploy controller to the K8s cluster specified in ~/.kube/config. +$(CS)/$(NAMESPACE)/install-config-dir: $(CS)/$(NAMESPACE)/cleanup-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) \ tmpdir="$$(mktemp -d)"; \ @@ -462,7 +449,7 @@ $(CS)/$(NAMESPACE)/install-config-dir: $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) touch $@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file -$(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via manifest into CTX cluster +$(CS)/$(NAMESPACE)/install-plain-manifests-file: $(CS)/$(NAMESPACE)/cleanup-installs.done dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via manifest into CTX cluster @set -euo pipefail; \ ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ @@ -479,16 +466,14 @@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: dist/install.yaml $(CS)/cert-ma .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_HELM)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework @@ -499,3 +484,39 @@ cleanup-port-forwards: ## Stop all port-forwards @-pkill -f "kubectl.*port-forward.*13000" 2>/dev/null || true @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true @echo "βœ… Port-forwards stopped" + +$(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) + mkdir -p $(@D) + kubectl --context $(CTX) label --overwrite ns $(NAMESPACE) \ + $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) \ + app.kubernetes.io/name=gitops-reverser control-plane=gitops-reverser \ + pod-security.kubernetes.io/enforce=restricted + @echo "βœ… Added e2e=true label to namespace $(NAMESPACE)" + touch $@ + +# Ensure e2e cleanup runs before any per-namespace stamps are rebuilt. +# Keep this stamp under the namespace directory so `rm -rf .stamps/cluster//` forces a rerun. +# We delete all services (over all namespaces) since we only can have ONE service listening to the cluster audit webhook at the same time +$(CS)/$(NAMESPACE)/cleanup-installs.done: Makefile $(CS)/ready + mkdir -p $(@D) + kubectl --context $(CTX) delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true; \ + kubectl --context $(CTX) delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true; \ + $(MAKE) cleanup-installs; \ + touch $@ + +.PHONY: cleanup-installs +cleanup-installs: ## Delete namespaces labeled for e2e runs in CTX cluster and remove their stamps + @set -euo pipefail; \ + ctx="$(CTX)"; \ + label="$(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE)"; \ + namespaces="$$(kubectl --context "$$ctx" get ns -l "$$label" -o jsonpath='{.items[*].metadata.name}')"; \ + if [ -z "$$namespaces" ]; then \ + echo "ℹ️ No namespaces with label '$$label' found in context '$$ctx'"; \ + exit 0; \ + fi; \ + echo "🧹 Deleting namespaces in context '$$ctx': $$namespaces"; \ + kubectl --context "$$ctx" delete ns $$namespaces --ignore-not-found=true; \ + for ns in $$namespaces; do \ + kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ + done; \ + for ns in $$namespaces; do rm -rf ".stamps/cluster/$$ctx/$$ns"; done From a066280585fc4b87be9befd3c3303734f8a13d06 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 10:40:45 +0000 Subject: [PATCH 12/30] chore: Clean local removal guides, only reuse same installation method --- .stamps/cluster/pf.running | 0 Makefile | 48 +++++++++++++++++--------------------- 2 files changed, 21 insertions(+), 27 deletions(-) delete mode 100644 .stamps/cluster/pf.running diff --git a/.stamps/cluster/pf.running b/.stamps/cluster/pf.running deleted file mode 100644 index e69de29bb..000000000 diff --git a/Makefile b/Makefile index 24ff86e38..d60c75d20 100644 --- a/Makefile +++ b/Makefile @@ -417,7 +417,7 @@ CONFIG_DIR_INPUTS := $(shell find config -type f) ##@ Deployments of all manifests needed to run $(CS)/$(NAMESPACE)/install-helm: INSTALL_MODE := helm -$(CS)/$(NAMESPACE)/install-helm: $(CS)/$(NAMESPACE)/cleanup-installs.done $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/install-helm: $(CLEANUP_E2E_DONE) $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) \ helm_args=( \ @@ -437,7 +437,7 @@ $(CS)/$(NAMESPACE)/install-helm: $(CS)/$(NAMESPACE)/cleanup-installs.done $(HELM touch $@ $(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/install-config-dir: $(CS)/$(NAMESPACE)/cleanup-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/install-config-dir: $(CLEANUP_E2E_DONE) $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ $(ASSERT_INSTALL_METHOD) \ tmpdir="$$(mktemp -d)"; \ @@ -449,7 +449,7 @@ $(CS)/$(NAMESPACE)/install-config-dir: $(CS)/$(NAMESPACE)/cleanup-installs.done touch $@ $(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file -$(CS)/$(NAMESPACE)/install-plain-manifests-file: $(CS)/$(NAMESPACE)/cleanup-installs.done dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed ## Deploy quickstart controller via manifest into CTX cluster +$(CS)/$(NAMESPACE)/install-plain-manifests-file: $(CLEANUP_E2E_DONE) dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ @@ -491,32 +491,26 @@ $(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/install-$( $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) \ app.kubernetes.io/name=gitops-reverser control-plane=gitops-reverser \ pod-security.kubernetes.io/enforce=restricted - @echo "βœ… Added e2e=true label to namespace $(NAMESPACE)" + @echo "βœ… Added $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) label to namespace $(NAMESPACE)" touch $@ -# Ensure e2e cleanup runs before any per-namespace stamps are rebuilt. -# Keep this stamp under the namespace directory so `rm -rf .stamps/cluster//` forces a rerun. -# We delete all services (over all namespaces) since we only can have ONE service listening to the cluster audit webhook at the same time -$(CS)/$(NAMESPACE)/cleanup-installs.done: Makefile $(CS)/ready - mkdir -p $(@D) - kubectl --context $(CTX) delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true; \ - kubectl --context $(CTX) delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true; \ - $(MAKE) cleanup-installs; \ - touch $@ - -.PHONY: cleanup-installs -cleanup-installs: ## Delete namespaces labeled for e2e runs in CTX cluster and remove their stamps +CLEANUP_E2E_DONE := $(CS)/$(NAMESPACE)/cleanup-e2e.$(INSTALL_MODE).done +$(CLEANUP_E2E_DONE): Makefile $(CS)/ready @set -euo pipefail; \ ctx="$(CTX)"; \ - label="$(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE)"; \ - namespaces="$$(kubectl --context "$$ctx" get ns -l "$$label" -o jsonpath='{.items[*].metadata.name}')"; \ - if [ -z "$$namespaces" ]; then \ - echo "ℹ️ No namespaces with label '$$label' found in context '$$ctx'"; \ - exit 0; \ + ns_list="$$(kubectl --context "$$ctx" get ns -l $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) -o jsonpath='{.items[*].metadata.name}' || true)"; \ + if [ -z "$$ns_list" ]; then \ + echo "ℹ️ No e2e=true namespaces in '$$ctx'"; \ + else \ + echo "🧹 Deleting $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) namespaces in '$$ctx': $$ns_list"; \ + kubectl --context "$$ctx" delete ns $$ns_list --ignore-not-found=true; \ + for ns in $$ns_list; do \ + kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ + rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ + done; \ fi; \ - echo "🧹 Deleting namespaces in context '$$ctx': $$namespaces"; \ - kubectl --context "$$ctx" delete ns $$namespaces --ignore-not-found=true; \ - for ns in $$namespaces; do \ - kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ - done; \ - for ns in $$namespaces; do rm -rf ".stamps/cluster/$$ctx/$$ns"; done + # Clear single-instance blockers (safe always) + kubectl --context "$$ctx" delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true || true; \ + kubectl --context "$$ctx" delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true || true; \ + mkdir -p "$(@D)"; \ + touch "$@" From 94c3b619bbc6aa4fdde0540920e998e020971929 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 11:24:39 +0000 Subject: [PATCH 13/30] chore: Preparing something else than Kind --- Makefile | 60 ++++-------- test/e2e/kind/start-cluster-k3d.sh | 142 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 40 deletions(-) create mode 100644 test/e2e/kind/start-cluster-k3d.sh diff --git a/Makefile b/Makefile index d60c75d20..24ad8e81c 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) \ +$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done \ $(CS)/$(NAMESPACE)/label-namespace.ready \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ @@ -303,14 +303,14 @@ $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-opera kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ -$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) test/e2e/tools/gen-age-key/main.go +$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/label-namespace.ready test/e2e/tools/gen-age-key/main.go mkdir -p $(@D) go run ./test/e2e/tools/gen-age-key \ - --key-file $(CS)/age-key.txt \ - --secret-file $(CS)/age-key-secret.yaml \ + --key-file $(CS)/$(NAMESPACE)/age-key.txt \ + --secret-file $(CS)/$(NAMESPACE)/age-key-secret.yaml \ --namespace $(NAMESPACE) \ --secret-name sops-age-key - kubectl --context $(CTX) apply -f $(CS)/age-key-secret.yaml + kubectl --context $(CTX) apply -f $(CS)/$(NAMESPACE)/age-key-secret.yaml touch $@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile @@ -349,7 +349,7 @@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready $(KIND) load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX); \ echo "$$img_id" > "$@" -$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) $(CS)/image.loaded +$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done $(CS)/image.loaded @set -euo pipefail; \ ctx="$(CTX)"; ns="$(NAMESPACE)"; img="$(PROJECT_IMAGE)"; c="$(CONTROLLER_CONTAINER)"; sel="$(CONTROLLER_DEPLOY_SELECTOR)"; \ [ -n "$$img" ] || { echo "ERROR: PROJECT_IMAGE must be non-empty" >&2; exit 2; }; \ @@ -371,11 +371,11 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh -E2E_TEST_INPUTS := $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) +E2E_TEST_INPUTS := $(CS)/$(NAMESPACE)/age-key.txt $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) NAMESPACE=$(NAMESPACE) \ - E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ @@ -383,43 +383,26 @@ CONTROLLER_CONTAINER ?= manager CONTROLLER_DEPLOY_SELECTOR ?= app.kubernetes.io/part-of=gitops-reverser # INSTALL_MODE, INSTALL_NAME, NAMESPACE defaults are defined near the E2E variables section above. -INSTALL_LOCK := $(CS)/$(NAMESPACE)/install.method VALID_INSTALL_MODES := config-dir helm plain-manifests-file ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) $(error INSTALL_MODE must be one of: $(VALID_INSTALL_MODES)) endif -define ASSERT_INSTALL_METHOD -mkdir -p $(@D); \ -if [ -f "$(INSTALL_LOCK)" ]; then \ - existing="$$(cat "$(INSTALL_LOCK)")"; \ - if [ "$$existing" != "$(INSTALL_MODE)" ]; then \ - echo "ERROR: Namespace $(NAMESPACE) already installed using '$$existing'."; \ - echo "Refusing to install with '$(INSTALL_MODE)'."; \ - echo "Cleanup that namespace and remove $(INSTALL_LOCK) to switch methods."; \ - exit 2; \ - fi; \ -else \ - echo "$(INSTALL_MODE)" > "$(INSTALL_LOCK)"; \ -fi; -endef - .PHONY: install install-helm install-plain-manifests-file install-config-dir -install: $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) +install: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done -install-helm: $(CS)/$(NAMESPACE)/install-helm +install-helm: $(CS)/$(NAMESPACE)/helm/install.done -install-plain-manifests-file: $(CS)/$(NAMESPACE)/install-plain-manifests-file +install-plain-manifests-file: $(CS)/$(NAMESPACE)/plain-manifests-file/install.done -install-config-dir: $(CS)/$(NAMESPACE)/install-config-dir +install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.done CONFIG_DIR_INPUTS := $(shell find config -type f) ##@ Deployments of all manifests needed to run -$(CS)/$(NAMESPACE)/install-helm: INSTALL_MODE := helm -$(CS)/$(NAMESPACE)/install-helm: $(CLEANUP_E2E_DONE) $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/helm/install.done: INSTALL_MODE := helm +$(CS)/$(NAMESPACE)/helm/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ - $(ASSERT_INSTALL_METHOD) \ helm_args=( \ upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ --kube-context $(CTX) \ @@ -436,10 +419,9 @@ $(CS)/$(NAMESPACE)/install-helm: $(CLEANUP_E2E_DONE) $(HELM_SYNC_OUTPUTS) $(CS)/ helm "$${helm_args[@]}"; \ touch $@ -$(CS)/$(NAMESPACE)/install-config-dir: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/install-config-dir: $(CLEANUP_E2E_DONE) $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/config-dir/install.done: INSTALL_MODE := config-dir +$(CS)/$(NAMESPACE)/config-dir/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ - $(ASSERT_INSTALL_METHOD) \ tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ cp -R config "$$tmpdir/config"; \ @@ -448,12 +430,11 @@ $(CS)/$(NAMESPACE)/install-config-dir: $(CLEANUP_E2E_DONE) $(MANIFEST_OUTPUTS) $ (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) | $(KUBECTL) --context "$(CTX)" apply -f - touch $@ -$(CS)/$(NAMESPACE)/install-plain-manifests-file: INSTALL_MODE := plain-manifests-file -$(CS)/$(NAMESPACE)/install-plain-manifests-file: $(CLEANUP_E2E_DONE) dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: INSTALL_MODE := plain-manifests-file +$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed @set -euo pipefail; \ ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ - $(ASSERT_INSTALL_METHOD) \ tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ cp -R dist "$$tmpdir/dist"; \ @@ -485,7 +466,7 @@ cleanup-port-forwards: ## Stop all port-forwards @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true @echo "βœ… Port-forwards stopped" -$(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/install-$(INSTALL_MODE) +$(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done mkdir -p $(@D) kubectl --context $(CTX) label --overwrite ns $(NAMESPACE) \ $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) \ @@ -494,8 +475,7 @@ $(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/install-$( @echo "βœ… Added $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) label to namespace $(NAMESPACE)" touch $@ -CLEANUP_E2E_DONE := $(CS)/$(NAMESPACE)/cleanup-e2e.$(INSTALL_MODE).done -$(CLEANUP_E2E_DONE): Makefile $(CS)/ready +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done: Makefile $(CS)/ready @set -euo pipefail; \ ctx="$(CTX)"; \ ns_list="$$(kubectl --context "$$ctx" get ns -l $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) -o jsonpath='{.items[*].metadata.name}' || true)"; \ diff --git a/test/e2e/kind/start-cluster-k3d.sh b/test/e2e/kind/start-cluster-k3d.sh new file mode 100644 index 000000000..3647f0edf --- /dev/null +++ b/test/e2e/kind/start-cluster-k3d.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Script to create k3d cluster with audit webhook support for Docker-outside-Docker devcontainers +# Keeps the "HOST_PROJECT_PATH must be mountable by the Docker daemon" logic from the kind version. +# Lets k3d pick the API port. Rewrites kubeconfig to host.docker.internal: when available. + +set -euo pipefail + +CLUSTER_NAME="${K3D_CLUSTER:-gitops-reverser-test-e2e}" +AUDIT_DIR_REL="test/e2e/kind/audit" +K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" + +# These filenames match your previous kind cluster-template.yaml +AUDIT_POLICY_FILE="policy.yaml" +AUDIT_WEBHOOK_CONFIG_FILE="webhook-config.yaml" + +docker_can_mount_repo() { + local candidate="$1" + docker run --rm -v "${candidate}:/hostproj:ro" busybox:1.36.1 sh -c \ + "test -f /hostproj/${AUDIT_DIR_REL}/${AUDIT_POLICY_FILE} && test -f /hostproj/${AUDIT_DIR_REL}/${AUDIT_WEBHOOK_CONFIG_FILE}" \ + >/dev/null 2>&1 +} + +resolve_host_project_path() { + local repo_pwd + repo_pwd="$(pwd -P)" + + local candidates=() + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + candidates+=("${HOST_PROJECT_PATH}") + fi + candidates+=("${repo_pwd}") + + local candidate + for candidate in "${candidates[@]}"; do + if docker_can_mount_repo "$candidate"; then + echo "$candidate" + return 0 + fi + done + + echo "❌ ERROR: Unable to determine a usable HOST_PROJECT_PATH for k3d volume mounts." >&2 + echo "Tried:" >&2 + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + echo " - HOST_PROJECT_PATH=${HOST_PROJECT_PATH}" >&2 + else + echo " - HOST_PROJECT_PATH=" >&2 + fi + echo " - pwd=${repo_pwd}" >&2 + echo "" >&2 + echo "Fix: set HOST_PROJECT_PATH to the path visible to the Docker daemon that contains this repo." >&2 + echo "Examples:" >&2 + echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${repo_pwd}'" >&2 + echo " - Docker-outside-of-Docker (daemon on your host): export HOST_PROJECT_PATH=''" >&2 + return 1 +} + +cluster_exists() { + # Use JSON output to avoid formatting differences across versions + k3d cluster list -o json 2>/dev/null | grep -q "\"name\":\"${CLUSTER_NAME}\"" +} + +create_cluster() { + local host_project_path="$1" + local audit_host_dir="${host_project_path}/${AUDIT_DIR_REL}" + + echo "πŸš€ Creating k3d cluster '${CLUSTER_NAME}' with audit webhook support..." + echo "πŸ”§ Using HOST_PROJECT_PATH: ${host_project_path}" + echo "πŸ”§ Mounting audit dir: ${audit_host_dir} -> /etc/kubernetes/audit" + + # NOTE: no --api-port: let k3d pick an available port. + # + # Mount audit dir into the server container and pass kube-apiserver audit flags + # via k3s "--kube-apiserver-arg=..." options. + k3d cluster create "${CLUSTER_NAME}" \ + --kubeconfig-update-default \ + --kubeconfig-switch-context \ + -v "${audit_host_dir}:/etc/kubernetes/audit@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-policy-file=/etc/kubernetes/audit/${AUDIT_POLICY_FILE}@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-config-file=/etc/kubernetes/audit/${AUDIT_WEBHOOK_CONFIG_FILE}@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-batch-max-wait=1s@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-batch-max-size=10@server:0" \ + 2>&1 | tee "${K3D_CREATE_LOG_FILE}" +} + +merge_kubeconfig() { + # Ensure kubeconfig is present and current-context points at this cluster + k3d kubeconfig merge "${CLUSTER_NAME}" \ + --kubeconfig-switch-context \ + --kubeconfig-update-default >/dev/null +} + +rewrite_kubeconfig_for_devcontainer() { + local cluster_entry server host port + + cluster_entry="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" + server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" + + # Parse https://HOST:PORT + if [[ "$server" =~ ^https://([^:/]+):([0-9]+)$ ]]; then + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + else + echo "ℹ️ kubeconfig server is '${server}' (couldn't parse host/port; no rewrite)" + return 0 + fi + + # In devcontainers we want to talk to the docker host. + if getent hosts host.docker.internal >/dev/null 2>&1; then + # Only rewrite if it's something that won't work inside the container + if [[ "$host" == "0.0.0.0" || "$host" == "127.0.0.1" || "$host" == "localhost" ]]; then + echo "πŸ” Rewriting kubeconfig server endpoint to host.docker.internal:${port}..." + kubectl config set-cluster "$cluster_entry" \ + --server="https://host.docker.internal:${port}" \ + --tls-server-name=localhost >/dev/null + echo "βœ… kubeconfig endpoint updated for devcontainer networking" + else + echo "ℹ️ kubeconfig server host is '${host}' (no rewrite needed)" + fi + else + echo "ℹ️ host.docker.internal not resolvable; keeping server as ${server}" + fi +} + +main() { + if cluster_exists; then + echo "♻️ Reusing existing k3d cluster '${CLUSTER_NAME}' (no delete/recreate)" + else + export HOST_PROJECT_PATH + HOST_PROJECT_PATH="$(resolve_host_project_path)" + create_cluster "${HOST_PROJECT_PATH}" + fi + + echo "πŸ“‹ Configuring kubeconfig for cluster '${CLUSTER_NAME}'..." + merge_kubeconfig + rewrite_kubeconfig_for_devcontainer + + echo "βœ… Cluster setup complete!" + echo "Try:" + echo " kubectl get nodes" +} + +main "$@" From 92df8f8295d473ef1b4c0b932509b34499146a2f Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 12:12:46 +0000 Subject: [PATCH 14/30] chore: Trying to run on k3d instead of kind --- .devcontainer/Dockerfile | 8 ++-- Makefile | 60 ++++++++++++------------- config/service.yaml | 2 +- test/e2e/kind/audit/webhook-config.yaml | 6 +-- test/e2e/kind/start-cluster-k3d.sh | 4 +- test/e2e/scripts/setup-port-forwards.sh | 21 ++++++--- 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d80dd56c8..cd874890d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -37,7 +37,7 @@ ENV KUBECTL_VERSION=v1.35.0 \ KUBEBUILDER_VERSION=4.11.1 \ GOLANGCI_LINT_VERSION=v2.10.1 \ HELM_VERSION=v4.0.0 \ - KIND_VERSION=v0.31.0 + K3D_VERSION=v5.8.3 # Install kubectl RUN curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ @@ -63,9 +63,9 @@ RUN curl -fsSL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz \ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ | sh -s -- -b /usr/local/bin ${GOLANGCI_LINT_VERSION} -# Install Kind (needed in CI for e2e cluster creation, and in dev for local clusters) -RUN curl -Lo /usr/local/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ - && chmod +x /usr/local/bin/kind +# Install k3d (faster and kind alternative, also more stable with our docker-outside-docker setup) +RUN curl -fsSL -o /usr/local/bin/k3d "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64" \ + && chmod +x /usr/local/bin/k3d # Set working directory WORKDIR /workspaces diff --git a/Makefile b/Makefile index 24ad8e81c..ddec38ee8 100644 --- a/Makefile +++ b/Makefile @@ -90,10 +90,10 @@ vet: ## Run go vet against code. test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(shell pwd)/bin -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Kind cluster names per E2E test type (kept separate to avoid cross-test contamination). +# E2E cluster names per test type (kept separate to avoid cross-test contamination). KIND_CLUSTER_E2E ?= gitops-reverser-test-e2e -KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-test-e2e-quickstart-helm -KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-test-e2e-quickstart-manifest +KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-qs-helm +KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-qs-manifest # KIND_CLUSTER is used by cleanup-cluster; defaults to the main e2e cluster. KIND_CLUSTER ?= $(KIND_CLUSTER_E2E) E2E_LOCAL_IMAGE ?= gitops-reverser:e2e-local @@ -106,10 +106,10 @@ export PROJECT_IMAGE E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. -# Defaults to the main e2e cluster; override with CTX=kind- to reuse stamp targets for other clusters. -CTX ?= kind-$(KIND_CLUSTER_E2E) -# Derive the Kind cluster name by stripping the "kind-" prefix from CTX. -CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(CTX)) +# Defaults to the main e2e cluster; override with CTX=k3d- to reuse stamp targets for other clusters. +CTX ?= k3d-$(KIND_CLUSTER_E2E) +# Derive the cluster name by stripping a known context prefix from CTX. +CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(patsubst k3d-%,%,$(CTX))) CS := .stamps/cluster/$(CTX) IS := .stamps/image GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum @@ -122,30 +122,30 @@ NAMESPACE ?= gitops-reverser E2E_NAMESPACE_LABEL_KEY ?= e2e E2E_NAMESPACE_LABEL_VALUE ?= true -KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) -KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) +KUBECONTEXT_QS_HELM := k3d-$(KIND_CLUSTER_QUICKSTART_HELM) +KUBECONTEXT_QS_MANIFEST := k3d-$(KIND_CLUSTER_QUICKSTART_MANIFEST) HELM_CHART_SOURCE ?= charts/gitops-reverser .PHONY: cleanup-cluster -cleanup-cluster: ## Tear down the Kind cluster used for e2e tests and remove its stamps - @if $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER)$$"; then \ - echo "🧹 Deleting Kind cluster '$(KIND_CLUSTER)'"; \ - $(KIND) delete cluster --name $(KIND_CLUSTER); \ +cleanup-cluster: ## Tear down the E2E cluster used for tests and remove its stamps + @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(KIND_CLUSTER)$$"; then \ + echo "🧹 Deleting k3d cluster '$(KIND_CLUSTER)'"; \ + $(K3D) cluster delete $(KIND_CLUSTER); \ else \ - echo "ℹ️ Kind cluster '$(KIND_CLUSTER)' does not exist; skipping cleanup"; \ + echo "ℹ️ k3d cluster '$(KIND_CLUSTER)' does not exist; skipping cleanup"; \ fi - rm -rf .stamps/cluster/kind-$(KIND_CLUSTER) + rm -rf .stamps/cluster/k3d-$(KIND_CLUSTER) .PHONY: cleanup-e2e-clusters -cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps +cleanup-e2e-clusters: ## Tear down all E2E clusters and their stamps @for cluster in "$(KIND_CLUSTER_E2E)" "$(KIND_CLUSTER_QUICKSTART_HELM)" "$(KIND_CLUSTER_QUICKSTART_MANIFEST)"; do \ - if $(KIND) get clusters 2>/dev/null | grep -q "^$$cluster$$"; then \ - echo "🧹 Deleting Kind cluster '$$cluster'"; \ - $(KIND) delete cluster --name "$$cluster"; \ + if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$$cluster$$"; then \ + echo "🧹 Deleting k3d cluster '$$cluster'"; \ + $(K3D) cluster delete "$$cluster"; \ else \ - echo "ℹ️ Kind cluster '$$cluster' does not exist; skipping cleanup"; \ + echo "ℹ️ k3d cluster '$$cluster' does not exist; skipping cleanup"; \ fi; \ - rm -rf .stamps/cluster/kind-$$cluster; \ + rm -rf .stamps/cluster/k3d-$$cluster; \ done @@ -235,7 +235,7 @@ dist/install.yaml: $(HELM_SYNC_OUTPUTS) $(CHART_INPUTS) ## Generate consolidated ## Tool Binaries - all pre-installed in devcontainer KUBECTL ?= kubectl -KIND ?= kind +K3D ?= k3d HELM ?= helm KUSTOMIZE ?= kustomize CONTROLLER_GEN ?= controller-gen @@ -258,11 +258,11 @@ setup-envtest: ## Setup envtest binaries for unit tests exit 1; \ } -##@ E2E Stamp Targets (cluster-parameterized; pass CTX=kind- to target a different cluster) +##@ E2E Stamp Targets (cluster-parameterized; pass CTX=k3d- to target a different cluster) -$(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml +$(CS)/ready: test/e2e/kind/start-cluster-k3d.sh mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster.sh + K3D_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster-k3d.sh kubectl --context $(CTX) get ns >/dev/null touch $@ @@ -346,7 +346,7 @@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready mkdir -p $(CS); \ img_id="$$( $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(PROJECT_IMAGE) )"; \ echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CLUSTER_FROM_CTX)"; \ - $(KIND) load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX); \ + $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_FROM_CTX); \ echo "$$img_id" > "$@" $(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done $(CS)/image.loaded @@ -371,10 +371,10 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh -E2E_TEST_INPUTS := $(CS)/$(NAMESPACE)/age-key.txt $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) +E2E_TEST_INPUTS := $(CS)/$(NAMESPACE)/age-key.applied $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) - CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) K3D_CLUSTER=$(CLUSTER_FROM_CTX) NAMESPACE=$(NAMESPACE) \ E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ @@ -447,14 +447,14 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTA .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ + CTX=$(KUBECONTEXT_QS_HELM) KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) K3D_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_HELM)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ + CTX=$(KUBECONTEXT_QS_MANIFEST) KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) K3D_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework diff --git a/config/service.yaml b/config/service.yaml index 7c78d243c..a6669d063 100644 --- a/config/service.yaml +++ b/config/service.yaml @@ -7,7 +7,7 @@ metadata: name: gitops-reverser-service namespace: sut spec: - clusterIP: 10.96.200.200 # This is required because kube-apiserver starts before CoreDNS (so we use a fixed address) + clusterIP: 10.43.200.200 # k3d/k3s service CIDR; kube-apiserver needs a fixed address before CoreDNS is available ports: - name: admission port: 9443 diff --git a/test/e2e/kind/audit/webhook-config.yaml b/test/e2e/kind/audit/webhook-config.yaml index 4d3b4f2df..ac9f361d0 100644 --- a/test/e2e/kind/audit/webhook-config.yaml +++ b/test/e2e/kind/audit/webhook-config.yaml @@ -1,4 +1,4 @@ -# Kubernetes audit webhook configuration for Kind cluster +# Kubernetes audit webhook configuration for k3d cluster # This configures the kube-apiserver to send audit events to the gitops-reverser webhook apiVersion: v1 kind: Config @@ -8,8 +8,8 @@ clusters: cluster: # IMPORTANT: Use fixed ClusterIP instead of DNS name # kube-apiserver starts before CoreDNS, so DNS resolution fails at startup - # The ClusterIP is set in config/default/audit_webhook_service_fixed_ip_patch.yaml - server: https://10.96.200.200:9444/audit-webhook/kind-e2e + # The ClusterIP is set in config/service.yaml + server: https://10.43.200.200:9444/audit-webhook/kind-e2e # Skip TLS verification for testing (webhook uses self-signed cert from cert-manager) insecure-skip-tls-verify: true contexts: diff --git a/test/e2e/kind/start-cluster-k3d.sh b/test/e2e/kind/start-cluster-k3d.sh index 3647f0edf..92a4e69d2 100644 --- a/test/e2e/kind/start-cluster-k3d.sh +++ b/test/e2e/kind/start-cluster-k3d.sh @@ -25,10 +25,10 @@ resolve_host_project_path() { repo_pwd="$(pwd -P)" local candidates=() + candidates+=("${repo_pwd}") if [ -n "${HOST_PROJECT_PATH:-}" ]; then candidates+=("${HOST_PROJECT_PATH}") fi - candidates+=("${repo_pwd}") local candidate for candidate in "${candidates[@]}"; do @@ -86,7 +86,7 @@ merge_kubeconfig() { # Ensure kubeconfig is present and current-context points at this cluster k3d kubeconfig merge "${CLUSTER_NAME}" \ --kubeconfig-switch-context \ - --kubeconfig-update-default >/dev/null + --kubeconfig-merge-default >/dev/null } rewrite_kubeconfig_for_devcontainer() { diff --git a/test/e2e/scripts/setup-port-forwards.sh b/test/e2e/scripts/setup-port-forwards.sh index a886ad142..a4bde5373 100755 --- a/test/e2e/scripts/setup-port-forwards.sh +++ b/test/e2e/scripts/setup-port-forwards.sh @@ -16,8 +16,19 @@ if [[ -z "${KUBE_CONTEXT}" ]]; then exit 1 fi -# Fast path: keep existing healthy forwards instead of restarting. -if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ +has_expected_forward_processes() { + local gitea_pf + local prom_pf + + gitea_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/${GITEA_SERVICE:-gitea-http}( |.* )13000:13000" | grep -v grep || true)" + prom_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/prometheus-operated( |.* )19090:9090" | grep -v grep || true)" + + [[ -n "${gitea_pf}" && -n "${prom_pf}" ]] +} + +# Fast path: keep existing healthy forwards only if they belong to this context. +if has_expected_forward_processes && \ + curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then echo "βœ… Existing port-forwards are healthy for context ${KUBE_CONTEXT}; skipping restart" exit 0 @@ -25,8 +36,8 @@ fi # Cleanup old port-forwards echo "🧹 Cleaning up old port-forwards..." -pkill -f "kubectl port-forward.*19090" || true -pkill -f "kubectl port-forward.*13000" || true +pkill -f "kubectl.*port-forward.*19090" || true +pkill -f "kubectl.*port-forward.*13000" || true sleep 1 echo "πŸ”Œ Setting up port-forwards for e2e testing..." @@ -65,7 +76,7 @@ setup_port_forward() { local remote_port="$5" # Check if already running - if ps aux | grep -E "kubectl.*port-forward.*${service}.*${local_port}:${remote_port}" | grep -v grep >/dev/null 2>&1; then + if ps aux | grep -E "kubectl.*--context[= ]${KUBE_CONTEXT}.*port-forward.*${service}.*${local_port}:${remote_port}" | grep -v grep >/dev/null 2>&1; then echo "βœ… ${service_name} port-forward already running" return 0 fi From d7ca9cb1b449d043c84e9306409e6521286491af Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 12:22:28 +0000 Subject: [PATCH 15/30] chore: That will help --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e72aaa4c0..0a77cd88f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: helm version golangci-lint version controller-gen --version - kind version + k3d version docker --version echo 'βœ… All CI container tools verified' " @@ -123,7 +123,7 @@ jobs: kustomize version helm version golangci-lint version - kind version + k3d version docker --version echo 'βœ… All dev container tools verified' " From 92ab85f3e384a19e7ad32a1813e92f5774ee56b8 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 12:27:44 +0000 Subject: [PATCH 16/30] fix: preventing warning on k3d creation DoD style --- test/e2e/kind/start-cluster-k3d.sh | 49 ++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/test/e2e/kind/start-cluster-k3d.sh b/test/e2e/kind/start-cluster-k3d.sh index 92a4e69d2..0a7c4d96d 100644 --- a/test/e2e/kind/start-cluster-k3d.sh +++ b/test/e2e/kind/start-cluster-k3d.sh @@ -8,6 +8,7 @@ set -euo pipefail CLUSTER_NAME="${K3D_CLUSTER:-gitops-reverser-test-e2e}" AUDIT_DIR_REL="test/e2e/kind/audit" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" +REPO_PWD="$(pwd -P)" # These filenames match your previous kind cluster-template.yaml AUDIT_POLICY_FILE="policy.yaml" @@ -21,11 +22,8 @@ docker_can_mount_repo() { } resolve_host_project_path() { - local repo_pwd - repo_pwd="$(pwd -P)" - local candidates=() - candidates+=("${repo_pwd}") + candidates+=("${REPO_PWD}") if [ -n "${HOST_PROJECT_PATH:-}" ]; then candidates+=("${HOST_PROJECT_PATH}") fi @@ -45,11 +43,11 @@ resolve_host_project_path() { else echo " - HOST_PROJECT_PATH=" >&2 fi - echo " - pwd=${repo_pwd}" >&2 + echo " - pwd=${REPO_PWD}" >&2 echo "" >&2 echo "Fix: set HOST_PROJECT_PATH to the path visible to the Docker daemon that contains this repo." >&2 echo "Examples:" >&2 - echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${repo_pwd}'" >&2 + echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${REPO_PWD}'" >&2 echo " - Docker-outside-of-Docker (daemon on your host): export HOST_PROJECT_PATH=''" >&2 return 1 } @@ -59,6 +57,44 @@ cluster_exists() { k3d cluster list -o json 2>/dev/null | grep -q "\"name\":\"${CLUSTER_NAME}\"" } +ensure_k3d_stat_compat_path() { + local host_project_path="$1" + + if [ "${host_project_path}" = "${REPO_PWD}" ]; then + return 0 + fi + + # k3d validates -v SOURCE with a local stat() before creating containers. + # In Docker-outside-of-Docker setups, HOST_PROJECT_PATH can exist on the host + # daemon but not inside this devcontainer, which causes a noisy warning. + # Creating a local symlink keeps k3d's preflight happy without changing the + # real bind source path passed to Docker. + # Example of the warning that we prevent: WARN[0000] failed to stat file/directory '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/kind/audit' volume mount '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/kind/audit:/etc/kubernetes/audit': please make sure it exists + local parent_dir + parent_dir="$(dirname "${host_project_path}")" + mkdir -p "${parent_dir}" 2>/dev/null || { + echo "⚠️ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 + return 0 + } + + if [ -L "${host_project_path}" ]; then + ln -sfn "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { + echo "⚠️ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } + return 0 + fi + + if [ -e "${host_project_path}" ]; then + return 0 + fi + + ln -s "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { + echo "⚠️ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } +} + create_cluster() { local host_project_path="$1" local audit_host_dir="${host_project_path}/${AUDIT_DIR_REL}" @@ -127,6 +163,7 @@ main() { else export HOST_PROJECT_PATH HOST_PROJECT_PATH="$(resolve_host_project_path)" + ensure_k3d_stat_compat_path "${HOST_PROJECT_PATH}" create_cluster "${HOST_PROJECT_PATH}" fi From 0987302bf48dc83e89d2c889ca9400a832a04fcf Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 12:40:18 +0000 Subject: [PATCH 17/30] chore: completing migrating to k3d --- test/e2e/cluster/README.md | 77 +++++++ test/e2e/{kind => cluster}/audit/policy.yaml | 0 .../audit/webhook-config.yaml | 0 .../start-cluster.sh} | 38 +++- test/e2e/kind/README.md | 155 --------------- test/e2e/kind/cluster-template.yaml | 33 --- test/e2e/kind/start-cluster.sh | 188 ------------------ 7 files changed, 109 insertions(+), 382 deletions(-) create mode 100644 test/e2e/cluster/README.md rename test/e2e/{kind => cluster}/audit/policy.yaml (100%) rename test/e2e/{kind => cluster}/audit/webhook-config.yaml (100%) rename test/e2e/{kind/start-cluster-k3d.sh => cluster/start-cluster.sh} (82%) delete mode 100644 test/e2e/kind/README.md delete mode 100644 test/e2e/kind/cluster-template.yaml delete mode 100755 test/e2e/kind/start-cluster.sh diff --git a/test/e2e/cluster/README.md b/test/e2e/cluster/README.md new file mode 100644 index 000000000..16d3c061f --- /dev/null +++ b/test/e2e/cluster/README.md @@ -0,0 +1,77 @@ +# E2E Cluster (k3d) with Audit Webhook + +This directory contains the remaining e2e cluster assets after the Kind-specific files were removed. +The e2e environment now uses k3d with kube-apiserver audit webhook flags configured at cluster create time. + +## Current Files + +- [`start-cluster.sh`](start-cluster.sh): creates/reuses a k3d cluster and configures kubeconfig for devcontainer usage +- [`audit/policy.yaml`](audit/policy.yaml): Kubernetes audit policy used by kube-apiserver +- [`audit/webhook-config.yaml`](audit/webhook-config.yaml): webhook target used by kube-apiserver audit backend + +## How It Works + +`start-cluster.sh` mounts the local audit directory into the k3d server node and sets: + +- `audit-policy-file=/etc/kubernetes/audit/policy.yaml` +- `audit-webhook-config-file=/etc/kubernetes/audit/webhook-config.yaml` +- `audit-webhook-batch-max-wait=1s` +- `audit-webhook-batch-max-size=10` + +The webhook URL in [`audit/webhook-config.yaml`](audit/webhook-config.yaml) targets: + +`https://10.43.200.200:9444/audit-webhook/kind-e2e` + +Notes: +- `insecure-skip-tls-verify: true` is intentional for local e2e +- `kind-e2e` in the path is the cluster ID label used by tests/metrics + +## Usage + +Run the full e2e suite: + +```bash +make test-e2e +``` + +Run quickstart e2e variants: + +```bash +make test-e2e-quickstart-manifest +make test-e2e-quickstart-helm +``` + +## Verification + +Confirm cluster is up: + +```bash +kubectl get nodes +``` + +Confirm audit files are mounted in the k3d server node: + +```bash +docker exec k3d-gitops-reverser-test-e2e-server-0 ls -l /etc/kubernetes/audit +``` + +Confirm audit metrics are incrementing: + +```promql +gitopsreverser_audit_events_received_total +``` + +## Troubleshooting + +`failed to stat file/directory ... volume mount ...`: this can appear in Docker-outside-of-Docker setups when the host path is valid for the Docker daemon but not visible inside the devcontainer. `start-cluster.sh` includes compatibility symlink logic for this. + +No audit events: + +1. Check webhook config in the mounted directory. +2. Check the operator service and deployment in `gitops-reverser`. +3. Check controller logs for `audit-handler` entries. + +## Reference + +- [Kubernetes Audit Documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/) +- [Audit Handler Implementation](../../../internal/webhook/audit_handler.go) diff --git a/test/e2e/kind/audit/policy.yaml b/test/e2e/cluster/audit/policy.yaml similarity index 100% rename from test/e2e/kind/audit/policy.yaml rename to test/e2e/cluster/audit/policy.yaml diff --git a/test/e2e/kind/audit/webhook-config.yaml b/test/e2e/cluster/audit/webhook-config.yaml similarity index 100% rename from test/e2e/kind/audit/webhook-config.yaml rename to test/e2e/cluster/audit/webhook-config.yaml diff --git a/test/e2e/kind/start-cluster-k3d.sh b/test/e2e/cluster/start-cluster.sh similarity index 82% rename from test/e2e/kind/start-cluster-k3d.sh rename to test/e2e/cluster/start-cluster.sh index 0a7c4d96d..25ec68c9b 100644 --- a/test/e2e/kind/start-cluster-k3d.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -59,11 +59,16 @@ cluster_exists() { ensure_k3d_stat_compat_path() { local host_project_path="$1" + local can_sudo=false if [ "${host_project_path}" = "${REPO_PWD}" ]; then return 0 fi + if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then + can_sudo=true + fi + # k3d validates -v SOURCE with a local stat() before creating containers. # In Docker-outside-of-Docker setups, HOST_PROJECT_PATH can exist on the host # daemon but not inside this devcontainer, which causes a noisy warning. @@ -73,14 +78,28 @@ ensure_k3d_stat_compat_path() { local parent_dir parent_dir="$(dirname "${host_project_path}")" mkdir -p "${parent_dir}" 2>/dev/null || { - echo "⚠️ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 - return 0 + if [ "${can_sudo}" = true ]; then + sudo -n mkdir -p "${parent_dir}" >/dev/null 2>&1 || { + echo "⚠️ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 + return 0 + } + else + echo "⚠️ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 + return 0 + fi } if [ -L "${host_project_path}" ]; then ln -sfn "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { - echo "⚠️ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 - return 0 + if [ "${can_sudo}" = true ]; then + sudo -n ln -sfn "${REPO_PWD}" "${host_project_path}" >/dev/null 2>&1 || { + echo "⚠️ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } + else + echo "⚠️ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + fi } return 0 fi @@ -90,8 +109,15 @@ ensure_k3d_stat_compat_path() { fi ln -s "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { - echo "⚠️ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 - return 0 + if [ "${can_sudo}" = true ]; then + sudo -n ln -s "${REPO_PWD}" "${host_project_path}" >/dev/null 2>&1 || { + echo "⚠️ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } + else + echo "⚠️ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + fi } } diff --git a/test/e2e/kind/README.md b/test/e2e/kind/README.md deleted file mode 100644 index d63c922d3..000000000 --- a/test/e2e/kind/README.md +++ /dev/null @@ -1,155 +0,0 @@ -# Kind Cluster with Audit Webhook Support - -This directory contains configuration files to set up a Kind cluster with Kubernetes audit webhook support for e2e testing. - -## Overview - -The gitops-reverser operator exposes an experimental audit webhook endpoint at `/audit-webhook/{clusterID}` that receives audit events from the Kubernetes API server. This setup configures Kind to send audit events to this endpoint for testing and metrics collection. - -## Files - -- **[`cluster.yaml`](cluster.yaml)**: Kind cluster configuration with audit webhook support -- **[`audit/policy.yaml`](audit/policy.yaml)**: Kubernetes audit policy (filters noise, captures intent) -- **[`audit/webhook-config.yaml`](audit/webhook-config.yaml)**: Audit webhook configuration (points to operator service) - -## Configuration Details - -### Cluster Configuration - -The [`cluster.yaml`](cluster.yaml:1) mounts the audit policy and webhook configuration files into the Kind control plane node and configures the kube-apiserver with: - -- `audit-policy-file`: Filters which events are audited -- `audit-webhook-config-file`: Configures where to send audit events -- `audit-webhook-batch-max-wait`: Maximum wait time before sending a batch (5s) -- `audit-webhook-batch-max-size`: Maximum events in a batch (100) - -### Webhook Configuration - -The [`webhook-config.yaml`](audit/webhook-config.yaml:1) configures the kube-apiserver to send audit events to: - -``` -https://10.96.200.200:443/audit-webhook/kind-e2e -``` - -**Important**: Uses `insecure-skip-tls-verify: true` because the webhook service uses a self-signed certificate from cert-manager. - -### Audit Policy - -The [`policy.yaml`](audit/policy.yaml:1) is copied from [`docs/audit-setup/cluster/audit/policy.yaml`](../../../docs/audit-setup/cluster/audit/policy.yaml:1) and filters: - -- **Noise**: Drops status updates, heartbeats, ephemeral checks -- **Security**: Filters secrets and sensitive operations -- **Intent**: Captures create/update/patch/delete operations on resources - -## Usage - -### Creating a Cluster - -The Makefile automatically uses this configuration when creating Kind clusters: - -```bash -make setup-cluster -``` - -This will create a cluster named `gitops-reverser-test-e2e` with audit webhook support. - -### Running E2E Tests - -The e2e test suite includes a test that verifies audit events are received: - -```bash -make test-e2e -``` - -Look for the test: **"should receive audit webhook events from kube-apiserver"** - -### Quick Verification - -Use the provided verification script to check if audit webhook is properly configured: - -```bash -./test/e2e/kind/verify-audit.sh -``` - -This script checks: -- Audit configuration files are mounted in the control plane -- kube-apiserver is configured with audit parameters -- Operator and webhook service are deployed -- Basic audit logging is working - -### Verifying Audit Events - -After deploying the operator, you can verify audit events are being received: - -1. **Check metrics** (via Prometheus at `http://localhost:19090`): - ```promql - gitopsreverser_audit_events_received_total - ``` - -2. **Check controller logs**: - ```bash - kubectl logs -n gitops-reverser-system -l control-plane=controller-manager --tail=100 | grep "audit" - ``` - -3. **Create a test resource** to trigger audit events: - ```bash - kubectl create configmap audit-test -n gitops-reverser-system --from-literal=test=value - ``` - -## Metrics - -The audit webhook tracks metrics with labels: - -- `gvr`: Group/Version/Resource (e.g., `apps/v1/deployments`) -- `action`: Verb (e.g., `create`, `update`, `delete`) -- `user`: Username who performed the action -- `processed`: Whether the event was processed (filters status updates) - -## Troubleshooting - -### Audit Events Not Received - -1. **Check kube-apiserver logs**: - ```bash - docker exec gitops-reverser-test-e2e-control-plane cat /var/log/kubernetes/kube-apiserver-audit.log - ``` - -2. **Verify audit webhook service exists**: - ```bash - kubectl get svc -n gitops-reverser-system gitops-reverser-service - ``` - -3. **Check if kube-apiserver can reach the webhook**: - ```bash - kubectl logs -n kube-system -l component=kube-apiserver - ``` - -### File Mounting Issues - -If audit configuration files aren't found, ensure paths in [`cluster.yaml`](cluster.yaml:7-11) are correct relative to the project root: - -```yaml -hostPath: ./test/e2e/kind/audit/policy.yaml -hostPath: ./test/e2e/kind/audit/webhook-config.yaml -``` - -## Differences from K3s Setup - -The K3s setup in [`docs/audit-setup/cluster/`](../../../docs/audit-setup/cluster/) uses: - -- Fixed ClusterIP: `10.43.200.200` (K3s default) -- Service name: Generic reference -- Config location: `/etc/rancher/k3s/audit/` - -The Kind setup uses: - -- Fixed ClusterIP: `10.96.200.200` -- Path-based cluster identity: `/audit-webhook/kind-e2e` -- Config location: `/etc/kubernetes/` (Kind standard) - -## References - -- [Kubernetes Audit Documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/) -- [Kind Extra Mounts](https://kind.sigs.k8s.io/docs/user/configuration/#extra-mounts) -- [Kubeadm Config Patches](https://kind.sigs.k8s.io/docs/user/configuration/#kubeadm-config-patches) -- [Audit Handler Implementation](../../../internal/webhook/audit_handler.go:1) diff --git a/test/e2e/kind/cluster-template.yaml b/test/e2e/kind/cluster-template.yaml deleted file mode 100644 index 2b9619bce..000000000 --- a/test/e2e/kind/cluster-template.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# kind cluster configuration template with audit webhook support -# This template uses ${HOST_PROJECT_PATH} which gets replaced at runtime -# with the actual host path by the start-cluster.sh script -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: - # Bind the API server on all host interfaces so devcontainer clients can - # reach it via host.docker.internal. - apiServerAddress: "0.0.0.0" -nodes: -- role: control-plane - # Mount the entire audit directory using $HOST_PROJECT_PATH. - extraMounts: - - hostPath: ${HOST_PROJECT_PATH}/test/e2e/kind/audit - containerPath: /etc/kubernetes/audit - readOnly: false - - # Configure kube-apiserver with audit webhook - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - audit-policy-file: /etc/kubernetes/audit/policy.yaml - audit-webhook-config-file: /etc/kubernetes/audit/webhook-config.yaml - audit-webhook-batch-max-wait: "1s" - audit-webhook-batch-max-size: "10" - extraVolumes: - - name: audit-config - hostPath: /etc/kubernetes/audit - mountPath: /etc/kubernetes/audit - readOnly: true - pathType: DirectoryOrCreate diff --git a/test/e2e/kind/start-cluster.sh b/test/e2e/kind/start-cluster.sh deleted file mode 100755 index 1160d3baf..000000000 --- a/test/e2e/kind/start-cluster.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/bin/bash -# Script to create Kind cluster with proper host path substitution for Docker-in-Docker - -set -euo pipefail - -CLUSTER_NAME="${KIND_CLUSTER:-gitops-reverser-test-e2e}" -TEMPLATE_FILE="test/e2e/kind/cluster-template.yaml" -CONFIG_FILE="test/e2e/kind/cluster.ignore.yaml" -KIND_CREATE_LOG_FILE="${TMPDIR:-/tmp}/kind-create-${CLUSTER_NAME}.log" -POD_SUBNET="${KIND_POD_SUBNET:-10.244.0.0/16}" - -docker_can_mount_repo() { - local candidate="$1" - docker run --rm -v "${candidate}:/hostproj:ro" busybox:1.36.1 sh -c \ - 'test -f /hostproj/test/e2e/kind/audit/policy.yaml' >/dev/null 2>&1 -} - -resolve_host_project_path() { - local repo_pwd - repo_pwd="$(pwd -P)" - - local candidates=() - if [ -n "${HOST_PROJECT_PATH:-}" ]; then - candidates+=("${HOST_PROJECT_PATH}") - fi - candidates+=("${repo_pwd}") - - local candidate - for candidate in "${candidates[@]}"; do - if docker_can_mount_repo "$candidate"; then - echo "$candidate" - return 0 - fi - done - - echo "❌ ERROR: Unable to determine a usable HOST_PROJECT_PATH for Kind extraMounts." >&2 - echo "Tried:" >&2 - if [ -n "${HOST_PROJECT_PATH:-}" ]; then - echo " - HOST_PROJECT_PATH=${HOST_PROJECT_PATH}" >&2 - else - echo " - HOST_PROJECT_PATH=" >&2 - fi - echo " - pwd=${repo_pwd}" >&2 - echo "" >&2 - echo "Fix: set HOST_PROJECT_PATH to the path visible to the Docker daemon that contains this repo." >&2 - echo "Examples:" >&2 - echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${repo_pwd}'" >&2 - echo " - Docker-outside-of-Docker (daemon on your host): export HOST_PROJECT_PATH=''" >&2 - return 1 -} - -create_primary_cluster() { - kind create cluster --name "$CLUSTER_NAME" --config "$CONFIG_FILE" --wait 5m --retain 2>&1 | tee "$KIND_CREATE_LOG_FILE" -} - -# Known Kind bootstrap race in DOOD setups: https://github.com/kubernetes-sigs/kind/issues/2867 -is_known_kind_bootstrap_flake() { - grep -Eq \ - "failed to apply overlay network|failed to remove control plane taint|failed to remove control plane load balancer label|failed to download openapi|couldn't get current server API group list|The connection to the server .*:6443 was refused" \ - "$KIND_CREATE_LOG_FILE" -} - -create_cluster_with_retries() { - local retries="${KIND_CREATE_RETRIES:-2}" - local attempt=0 - - while true; do - attempt=$((attempt + 1)) - if create_primary_cluster; then - return 0 - fi - - if is_known_kind_bootstrap_flake && [ "$attempt" -le "$retries" ]; then - echo "⚠️ Kind bootstrap flake detected; retrying create (attempt ${attempt}/${retries})" - kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true - sleep 2 - continue - fi - - return 1 - done -} - -run_dood_self_heal() { - local control_plane_container="${CLUSTER_NAME}-control-plane" - local container_found=false - for _ in $(seq 1 30); do - if docker inspect "$control_plane_container" >/dev/null 2>&1; then - container_found=true - break - fi - sleep 1 - done - [ "$container_found" = "true" ] || return 1 - - local ready=false - for _ in $(seq 1 120); do - if docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw=/readyz >/dev/null 2>&1; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != "true" ]; then - docker exec "$control_plane_container" crictl ps -a || true - return 1 - fi - - local applied=false - for _ in $(seq 1 40); do - if docker exec "$control_plane_container" sh -lc \ - "sed 's|{{ \\.PodSubnet }}|${POD_SUBNET}|g' /kind/manifests/default-cni.yaml > /tmp/default-cni-rendered.yaml && \ - kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /tmp/default-cni-rendered.yaml && \ - kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /kind/manifests/default-storage.yaml"; then - applied=true - break - fi - sleep 2 - done - [ "$applied" = "true" ] || return 1 - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - taint nodes --all node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - label nodes --all node.kubernetes.io/exclude-from-external-load-balancers- >/dev/null 2>&1 || true - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - -n kube-system rollout status daemonset/kindnet --timeout=300s - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - wait --for=condition=Ready nodes --all --timeout=300s -} - -if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then - echo "♻️ Reusing existing Kind cluster '$CLUSTER_NAME' (no delete/recreate)" -else - export HOST_PROJECT_PATH - HOST_PROJECT_PATH="$(resolve_host_project_path)" - - echo "πŸ”§ Using HOST_PROJECT_PATH: $HOST_PROJECT_PATH" - echo "πŸ“ Generating Kind cluster configuration from template..." - envsubst < "$TEMPLATE_FILE" > "$CONFIG_FILE" - echo "βœ… Generated configuration:" - cat "$CONFIG_FILE" - echo "" - - echo "πŸš€ Creating Kind cluster '$CLUSTER_NAME' with audit webhook support..." - if create_cluster_with_retries; then - echo "βœ… Kind cluster created successfully" - else - if is_known_kind_bootstrap_flake; then - echo "⚠️ Kind bootstrap flake detected; attempting self-heal" - else - echo "⚠️ Kind cluster creation failed; attempting self-heal" - fi - - if run_dood_self_heal; then - echo "βœ… Kind cluster self-healed successfully" - else - echo "❌ Kind cluster creation failed." - echo "πŸ“„ See log: $KIND_CREATE_LOG_FILE" - exit 1 - fi - fi -fi - -echo "πŸ“‹ Configuring kubeconfig for cluster '$CLUSTER_NAME'..." -kind export kubeconfig --name "$CLUSTER_NAME" - -current_cluster_name="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" -current_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" - -if [[ "$current_server" =~ ^https://(127\.0\.0\.1|localhost|0\.0\.0\.0):([0-9]+)$ ]]; then - apiserver_port="${BASH_REMATCH[2]}" - if getent hosts host.docker.internal >/dev/null 2>&1; then - echo "πŸ” Rewriting kubeconfig server endpoint to host.docker.internal:${apiserver_port}..." - kubectl config set-cluster "$current_cluster_name" \ - --server="https://host.docker.internal:${apiserver_port}" \ - --tls-server-name=localhost >/dev/null - echo "βœ… kubeconfig endpoint updated for devcontainer networking" - else - echo "ℹ️ host.docker.internal not resolvable; keeping server as ${current_server} (--network host or native environment)" - fi -else - echo "ℹ️ kubeconfig server is '$current_server' (no rewrite needed)" -fi - -echo "βœ… Cluster setup complete!" From 10f7092ddb7c0b420bd0579db6581b3f4c4df733 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 12:43:50 +0000 Subject: [PATCH 18/30] fix: Getting all paths right --- .../SETUP_CLUSTER_TROUBLESHOOTING.md | 8 +++--- Makefile | 4 +-- docs/audit-setup/cluster/readme.md | 2 +- docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md | 2 +- docs/ci/FINDINGS.md | 2 +- ..._DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md | 6 ++-- docs/ci/KUBECTL_TLS_DEBUG_REPORT.md | 4 +-- docs/make-e2e-deps.md | 4 +-- docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md | 28 +++++++++---------- test/e2e/cluster/start-cluster.sh | 4 +-- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md b/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md index 84977e37e..74d2fe2df 100644 --- a/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md +++ b/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md @@ -10,19 +10,19 @@ Get "https://172.19.0.2:6443/livez?timeout=10s": dial tcp 172.19.0.2:6443: conne ## Root cause (current setup) -`test/e2e/kind/start-cluster.sh` generates `test/e2e/kind/cluster.ignore.yaml` from `HOST_PROJECT_PATH`. +`test/e2e/cluster/start-cluster.sh` generates `test/e2e/cluster/cluster.ignore.yaml` from `HOST_PROJECT_PATH`. In the current devcontainer config, `HOST_PROJECT_PATH` is set from `${localWorkspaceFolder}`. That produced: ``` -hostPath: /home/simon/git/gitops-reverser2/test/e2e/kind/audit +hostPath: /home/simon/git/gitops-reverser2/test/e2e/cluster/audit ``` But that mounted directory exists and is empty in the container, while the real audit files are under: ``` -/workspaces/gitops-reverser2/test/e2e/kind/audit +/workspaces/gitops-reverser2/test/e2e/cluster/audit ``` Because the mount source is wrong/empty, kube-apiserver cannot read: @@ -56,7 +56,7 @@ Your current config mixes modes and path assumptions, so Kind mount path resolut Before running `make setup-cluster`, verify generated config points to a path with files: ```bash -cat test/e2e/kind/cluster.ignore.yaml +cat test/e2e/cluster/cluster.ignore.yaml ls -la ``` diff --git a/Makefile b/Makefile index ddec38ee8..481cfa9e2 100644 --- a/Makefile +++ b/Makefile @@ -260,9 +260,9 @@ setup-envtest: ## Setup envtest binaries for unit tests ##@ E2E Stamp Targets (cluster-parameterized; pass CTX=k3d- to target a different cluster) -$(CS)/ready: test/e2e/kind/start-cluster-k3d.sh +$(CS)/ready: test/e2e/cluster/start-cluster.sh mkdir -p $(CS) - K3D_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster-k3d.sh + K3D_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/cluster/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ diff --git a/docs/audit-setup/cluster/readme.md b/docs/audit-setup/cluster/readme.md index 34ad9489a..06b18f0cb 100644 --- a/docs/audit-setup/cluster/readme.md +++ b/docs/audit-setup/cluster/readme.md @@ -14,7 +14,7 @@ kube-apiserver-arg: Do the upgrade in order on your contorl plane nodes: -Also copy in the [policy file](/test/e2e/kind/audit/policy.yaml) +Also copy in the [policy file](/test/e2e/cluster/audit/policy.yaml) sudo systemctl restart k3s # Wait 30 seconds diff --git a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md index 51bf90b07..56b1068de 100644 --- a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md +++ b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md @@ -105,7 +105,7 @@ Outcome: - if `PROJECT_IMAGE` is set: no prep - else: call Make for cluster/image prep (IDE fallback) -### Kind cluster bootstrap (`test/e2e/kind/start-cluster.sh`) +### Kind cluster bootstrap (`test/e2e/cluster/start-cluster.sh`) - Reuses existing Kind cluster if present (no delete/recreate in script). - Creates cluster only when missing. - Still exports/re-writes kubeconfig endpoint for devcontainer networking. diff --git a/docs/ci/FINDINGS.md b/docs/ci/FINDINGS.md index 2df67b0ce..773bc1c38 100644 --- a/docs/ci/FINDINGS.md +++ b/docs/ci/FINDINGS.md @@ -54,7 +54,7 @@ The current working model is: - `--add-host=host.docker.internal:host-gateway` - Kind cluster config sets: - `networking.apiServerAddress: "0.0.0.0"` -- `test/e2e/kind/start-cluster.sh` rewrites kubeconfig server endpoints from +- `test/e2e/cluster/start-cluster.sh` rewrites kubeconfig server endpoints from `127.0.0.1|localhost|0.0.0.0` to `host.docker.internal:` and sets `tls-server-name=localhost` diff --git a/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md index a689f5815..0d990fe4f 100644 --- a/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md +++ b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md @@ -44,7 +44,7 @@ This matches known upstream issue patterns for Kind in DOOD-style environments: ## Implemented Interim Workaround (DOOD) -`test/e2e/kind/start-cluster.sh` now includes a compact self-heal path for known DOOD bootstrap flakes: +`test/e2e/cluster/start-cluster.sh` now includes a compact self-heal path for known DOOD bootstrap flakes: create with `--retain`, wait for API readiness, apply Kind default CNI/storage with `--validate=false`, then wait for `kindnet` and node readiness. @@ -78,8 +78,8 @@ environment, removing host-socket coupling for local cluster lifecycle. ### Phase 2: Kind Config Alignment -1. Keep `test/e2e/kind/cluster-template.yaml` mount source aligned with container-visible path. -2. Review `test/e2e/kind/start-cluster.sh` kubeconfig rewrite logic: +1. Keep `test/e2e/cluster/cluster-template.yaml` mount source aligned with container-visible path. +2. Review `test/e2e/cluster/start-cluster.sh` kubeconfig rewrite logic: - Current rewrite to `host.docker.internal` is DOOD-oriented - In DinD, direct endpoint usage is preferred unless a concrete networking reason remains 3. Validate audit mounts from inside Kind control-plane: diff --git a/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md b/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md index d5e07a99b..bf4b1d95c 100644 --- a/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md +++ b/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md @@ -43,7 +43,7 @@ In `.devcontainer/devcontainer.json`: ### B) Kind API server bind address -In `test/e2e/kind/cluster-template.yaml`: +In `test/e2e/cluster/cluster-template.yaml`: ```yaml networking: @@ -54,7 +54,7 @@ This ensures host publish is reachable from devcontainer via `host.docker.intern ### C) Kubeconfig rewrite in cluster setup script -In `test/e2e/kind/start-cluster.sh`, after `kind export kubeconfig`: +In `test/e2e/cluster/start-cluster.sh`, after `kind export kubeconfig`: - detect kubeconfig server host in `{127.0.0.1, localhost, 0.0.0.0}` - rewrite to `https://host.docker.internal:` diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md index c59940be5..1e1d96b81 100644 --- a/docs/make-e2e-deps.md +++ b/docs/make-e2e-deps.md @@ -123,9 +123,9 @@ What this buys you in practice: ### 1 β€” Cluster ready ```make -$(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml +$(CS)/ready: test/e2e/cluster/start-cluster.sh test/e2e/cluster/cluster-template.yaml mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster.sh + KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/cluster/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ ``` diff --git a/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md b/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md index 1d2db3700..26c985c08 100644 --- a/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md +++ b/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md @@ -25,7 +25,7 @@ Even with files correctly mounted: ### Evidence ```bash # Inside devcontainer: -$ ls test/e2e/kind/audit/policy.yaml +$ ls test/e2e/cluster/audit/policy.yaml -rw-r--r-- 1 vscode vscode 1858 policy.yaml # βœ… File exists # From Docker on host: @@ -54,7 +54,7 @@ This injects the physical host path (e.g., `/home/user/projects/gitops-reverser2 #### 2. Created Template Configuration -**File:** [`test/e2e/kind/cluster-template.yaml`](../test/e2e/kind/cluster-template.yaml) +**File:** [`test/e2e/cluster/cluster-template.yaml`](../test/e2e/cluster/cluster-template.yaml) ```yaml kind: Cluster @@ -62,7 +62,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraMounts: - - hostPath: ${HOST_PROJECT_PATH}/test/e2e/kind/audit # ← Mount to node + - hostPath: ${HOST_PROJECT_PATH}/test/e2e/cluster/audit # ← Mount to node containerPath: /etc/kubernetes/audit readOnly: false @@ -85,7 +85,7 @@ nodes: #### 3. Created Cluster Startup Script -**File:** [`test/e2e/kind/start-cluster.sh`](../test/e2e/kind/start-cluster.sh) +**File:** [`test/e2e/cluster/start-cluster.sh`](../test/e2e/cluster/start-cluster.sh) The script: 1. Checks if `HOST_PROJECT_PATH` environment variable is set @@ -103,7 +103,7 @@ setup-cluster: ## Set up a Kind cluster for e2e tests if it does not exist @if ! command -v $(KIND) >/dev/null 2>&1; then \ echo "Kind is not installed - skipping..."; \ else \ - KIND_CLUSTER=$(KIND_CLUSTER) bash test/e2e/kind/start-cluster.sh; \ + KIND_CLUSTER=$(KIND_CLUSTER) bash test/e2e/cluster/start-cluster.sh; \ fi ``` @@ -144,7 +144,7 @@ patches: #### 3. Updated Audit Webhook Config -**File:** [`test/e2e/kind/audit/webhook-config.yaml`](../test/e2e/kind/audit/webhook-config.yaml) +**File:** [`test/e2e/cluster/audit/webhook-config.yaml`](../test/e2e/cluster/audit/webhook-config.yaml) ```yaml clusters: @@ -176,15 +176,15 @@ steps: # Generate cluster.yaml from template with HOST_PROJECT_PATH substitution - name: Generate Kind cluster config from template run: | - envsubst < test/e2e/kind/cluster-template.yaml > test/e2e/kind/cluster.yaml - cat test/e2e/kind/cluster.yaml + envsubst < test/e2e/cluster/cluster-template.yaml > test/e2e/cluster/cluster.yaml + cat test/e2e/cluster/cluster.yaml # Use helm/kind-action with our custom config - name: Set up Kind cluster with audit webhook support uses: helm/kind-action@v1.13.0 with: cluster_name: gitops-reverser-test-e2e - config: test/e2e/kind/cluster.yaml # Our generated config + config: test/e2e/cluster/cluster.yaml # Our generated config wait: 5m ``` @@ -350,8 +350,8 @@ The test **"should receive audit webhook events from kube-apiserver"** should pa # Old directories created by previous failed attempts still exist on host # Delete them manually: docker run --rm -v /:/host busybox rm -rf \ - /host/workspaces/gitops-reverser2/test/e2e/kind/audit/policy.yaml \ - /host/workspaces/gitops-reverser2/test/e2e/kind/audit/webhook-config.yaml + /host/workspaces/gitops-reverser2/test/e2e/cluster/audit/policy.yaml \ + /host/workspaces/gitops-reverser2/test/e2e/cluster/audit/webhook-config.yaml # Then try creating cluster again kind delete cluster --name gitops-reverser-test-e2e @@ -377,11 +377,11 @@ docker exec gitops-reverser-test-e2e-control-plane crictl logs $(docker exec git ## Files Modified - [`.devcontainer/devcontainer.json`](../.devcontainer/devcontainer.json) - Added `HOST_PROJECT_PATH` env var -- [`test/e2e/kind/cluster-template.yaml`](../test/e2e/kind/cluster-template.yaml) - Template with placeholder -- [`test/e2e/kind/start-cluster.sh`](../test/e2e/kind/start-cluster.sh) - Script to substitute and create cluster +- [`test/e2e/cluster/cluster-template.yaml`](../test/e2e/cluster/cluster-template.yaml) - Template with placeholder +- [`test/e2e/cluster/start-cluster.sh`](../test/e2e/cluster/start-cluster.sh) - Script to substitute and create cluster - [`Makefile`](../Makefile) - Updated `setup-cluster` target - [`test/e2e/e2e_test.go`](../test/e2e/e2e_test.go) - Added audit webhook test -- [`test/e2e/kind/README.md`](../test/e2e/kind/README.md) - Documentation +- [`test/e2e/cluster/README.md`](../test/e2e/cluster/README.md) - Documentation ## Verification: It Should Work Now! diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index 25ec68c9b..df769f3dc 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -6,7 +6,7 @@ set -euo pipefail CLUSTER_NAME="${K3D_CLUSTER:-gitops-reverser-test-e2e}" -AUDIT_DIR_REL="test/e2e/kind/audit" +AUDIT_DIR_REL="test/e2e/cluster/audit" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" @@ -74,7 +74,7 @@ ensure_k3d_stat_compat_path() { # daemon but not inside this devcontainer, which causes a noisy warning. # Creating a local symlink keeps k3d's preflight happy without changing the # real bind source path passed to Docker. - # Example of the warning that we prevent: WARN[0000] failed to stat file/directory '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/kind/audit' volume mount '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/kind/audit:/etc/kubernetes/audit': please make sure it exists + # Example of the warning that we prevent: WARN[0000] failed to stat file/directory '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/cluster/audit' volume mount '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/cluster/audit:/etc/kubernetes/audit': please make sure it exists local parent_dir parent_dir="$(dirname "${host_project_path}")" mkdir -p "${parent_dir}" 2>/dev/null || { From 73e6e3b4539031ac29c33a2105b8a00988444433 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 13:10:12 +0000 Subject: [PATCH 19/30] chore: Let's see if it's just slower or that we have a difference --- test/e2e/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 05e444c72..b9e9d8656 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1012,7 +1012,7 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), "Git log should contain DELETE operation") } - Eventually(verifyFileDeleted).Should(Succeed()) + Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) By("cleaning up test resources") cleanupWatchRule(watchRuleName, namespace) From 55bc5ee7a33e5cfa9fc480d5ae4d37f738031b2e Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 13:29:54 +0000 Subject: [PATCH 20/30] chore: Does this help then? --- test/e2e/e2e_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index b9e9d8656..c272c0b45 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1012,7 +1012,7 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), "Git log should contain DELETE operation") } - Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) + Eventually(verifyFileDeleted).Should(Succeed()) By("cleaning up test resources") cleanupWatchRule(watchRuleName, namespace) @@ -1598,7 +1598,7 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), "Git log should contain DELETE operation") } - Eventually(verifyFileDeleted).Should(Succeed()) + Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) By("cleaning up test resources") cleanupClusterWatchRule(clusterWatchRuleName) From 25d4d9608513b14ee20cf7ed63f2f81405aab436 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 13:51:52 +0000 Subject: [PATCH 21/30] chore: Now it's behaving better with local source changes --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 481cfa9e2..4b87af5ad 100644 --- a/Makefile +++ b/Makefile @@ -97,12 +97,14 @@ KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-qs-manifest # KIND_CLUSTER is used by cleanup-cluster; defaults to the main e2e cluster. KIND_CLUSTER ?= $(KIND_CLUSTER_E2E) E2E_LOCAL_IMAGE ?= gitops-reverser:e2e-local -# Capture whether PROJECT_IMAGE was provided (CI sets it; local runs typically don't). -PROJECT_IMAGE_PROVIDED ?= $(strip $(PROJECT_IMAGE)) -export PROJECT_IMAGE_PROVIDED -# PROJECT_IMAGE is the single source of truth after this point. -PROJECT_IMAGE ?= $(E2E_LOCAL_IMAGE) +# Normalize PROJECT_IMAGE once, then consume only normalized variables below. +# - If PROJECT_IMAGE is explicitly provided (CI / caller), use it and skip local source-triggered build dependency. +# - Otherwise default to E2E_LOCAL_IMAGE and treat it as local-build managed. +PROJECT_IMAGE_INPUT := $(strip $(PROJECT_IMAGE)) +PROJECT_IMAGE := $(if $(PROJECT_IMAGE_INPUT),$(PROJECT_IMAGE_INPUT),$(E2E_LOCAL_IMAGE)) +PROJECT_IMAGE_PROVIDED := $(if $(PROJECT_IMAGE_INPUT),true,) export PROJECT_IMAGE +export PROJECT_IMAGE_PROVIDED E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. From 5f9c572244b0502a5d75af6219d75ebc646f5fa0 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 16:33:33 +0000 Subject: [PATCH 22/30] chore: cleaning up seperate clusters (we now can 'clean' the cluster and prevent constant recreation). --- Makefile | 55 +-- docs/make-e2e-deps.md | 378 ------------------ docs/make-e2e-further.md | 163 -------- ...file-e2e-image-and-deploy-refactor-plan.md | 159 -------- docs/makefile-e2e-overview-and-plan.md | 133 ------ test/e2e/cluster/start-cluster.sh | 2 +- test/e2e/e2e_suite_test.go | 25 +- test/utils/utils.go | 14 - 8 files changed, 28 insertions(+), 901 deletions(-) delete mode 100644 docs/make-e2e-deps.md delete mode 100644 docs/make-e2e-further.md delete mode 100644 docs/makefile-e2e-image-and-deploy-refactor-plan.md delete mode 100644 docs/makefile-e2e-overview-and-plan.md diff --git a/Makefile b/Makefile index 4b87af5ad..63257b1cf 100644 --- a/Makefile +++ b/Makefile @@ -90,12 +90,6 @@ vet: ## Run go vet against code. test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(shell pwd)/bin -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# E2E cluster names per test type (kept separate to avoid cross-test contamination). -KIND_CLUSTER_E2E ?= gitops-reverser-test-e2e -KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-qs-helm -KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-qs-manifest -# KIND_CLUSTER is used by cleanup-cluster; defaults to the main e2e cluster. -KIND_CLUSTER ?= $(KIND_CLUSTER_E2E) E2E_LOCAL_IMAGE ?= gitops-reverser:e2e-local # Normalize PROJECT_IMAGE once, then consume only normalized variables below. # - If PROJECT_IMAGE is explicitly provided (CI / caller), use it and skip local source-triggered build dependency. @@ -108,10 +102,10 @@ export PROJECT_IMAGE_PROVIDED E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. -# Defaults to the main e2e cluster; override with CTX=k3d- to reuse stamp targets for other clusters. -CTX ?= k3d-$(KIND_CLUSTER_E2E) +# Defaults to the main e2e cluster; override with CTX= to reuse stamp targets for other clusters. +CTX ?= k3d-gitops-reverser-test-e2e # Derive the cluster name by stripping a known context prefix from CTX. -CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(patsubst k3d-%,%,$(CTX))) +CLUSTER_NAME ?= $(patsubst kind-%,%,$(patsubst k3d-%,%,$(CTX))) CS := .stamps/cluster/$(CTX) IS := .stamps/image GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum @@ -123,33 +117,17 @@ INSTALL_NAME ?= gitops-reverser NAMESPACE ?= gitops-reverser E2E_NAMESPACE_LABEL_KEY ?= e2e E2E_NAMESPACE_LABEL_VALUE ?= true - -KUBECONTEXT_QS_HELM := k3d-$(KIND_CLUSTER_QUICKSTART_HELM) -KUBECONTEXT_QS_MANIFEST := k3d-$(KIND_CLUSTER_QUICKSTART_MANIFEST) HELM_CHART_SOURCE ?= charts/gitops-reverser .PHONY: cleanup-cluster cleanup-cluster: ## Tear down the E2E cluster used for tests and remove its stamps - @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(KIND_CLUSTER)$$"; then \ - echo "🧹 Deleting k3d cluster '$(KIND_CLUSTER)'"; \ - $(K3D) cluster delete $(KIND_CLUSTER); \ + @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(CLUSTER_NAME)$$"; then \ + echo "🧹 Deleting k3d cluster '$(CLUSTER_NAME)'"; \ + $(K3D) cluster delete $(CLUSTER_NAME); \ else \ - echo "ℹ️ k3d cluster '$(KIND_CLUSTER)' does not exist; skipping cleanup"; \ + echo "ℹ️ k3d cluster '$(CLUSTER_NAME)' does not exist; skipping cleanup"; \ fi - rm -rf .stamps/cluster/k3d-$(KIND_CLUSTER) - -.PHONY: cleanup-e2e-clusters -cleanup-e2e-clusters: ## Tear down all E2E clusters and their stamps - @for cluster in "$(KIND_CLUSTER_E2E)" "$(KIND_CLUSTER_QUICKSTART_HELM)" "$(KIND_CLUSTER_QUICKSTART_MANIFEST)"; do \ - if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$$cluster$$"; then \ - echo "🧹 Deleting k3d cluster '$$cluster'"; \ - $(K3D) cluster delete "$$cluster"; \ - else \ - echo "ℹ️ k3d cluster '$$cluster' does not exist; skipping cleanup"; \ - fi; \ - rm -rf .stamps/cluster/k3d-$$cluster; \ - done - + rm -rf .stamps/cluster/$(CTX) # Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. .PHONY: prepare-e2e @@ -166,7 +144,6 @@ $(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install mkdir -p $(@D) touch $@ - # Keep `make test-e2e` as the classic entrypoint. .PHONY: test-e2e test-e2e: $(CS)/e2e.passed @@ -264,7 +241,7 @@ setup-envtest: ## Setup envtest binaries for unit tests $(CS)/ready: test/e2e/cluster/start-cluster.sh mkdir -p $(CS) - K3D_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/cluster/start-cluster.sh + CLUSTER_NAME=$(CLUSTER_NAME) bash test/e2e/cluster/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ @@ -347,8 +324,8 @@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready @set -euo pipefail; \ mkdir -p $(CS); \ img_id="$$( $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(PROJECT_IMAGE) )"; \ - echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CLUSTER_FROM_CTX)"; \ - $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_FROM_CTX); \ + echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CTX)"; \ + $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_NAME); \ echo "$$img_id" > "$@" $(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done $(CS)/image.loaded @@ -376,7 +353,7 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p E2E_TEST_INPUTS := $(CS)/$(NAMESPACE)/age-key.applied $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) - CTX=$(CTX) KIND_CLUSTER=$(CLUSTER_FROM_CTX) K3D_CLUSTER=$(CLUSTER_FROM_CTX) NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) INSTALL_MODE=$(INSTALL_MODE) NAMESPACE=$(NAMESPACE) \ E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ @@ -449,16 +426,16 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTA .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - CTX=$(KUBECONTEXT_QS_HELM) KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) K3D_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ - E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_HELM)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - CTX=$(KUBECONTEXT_QS_MANIFEST) KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) K3D_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ + CTX=$(CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ - E2E_AGE_KEY_FILE=.stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: cleanup-port-forwards diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md deleted file mode 100644 index 1e1d96b81..000000000 --- a/docs/make-e2e-deps.md +++ /dev/null @@ -1,378 +0,0 @@ -# E2E test dependencies with Make stamps - -For the roadmap/implementation plan, see `docs/makefile-e2e-overview-and-plan.md`. - -## What the old setup did (replaced) - -`make test-e2e` used to run these prerequisites unconditionally on every invocation: - -``` -setup-cluster β†’ cleanup-webhook β†’ setup-e2e β†’ check-cert-manager β†’ e2e-deploy β†’ setup-port-forwards -``` - -Every single step was `.PHONY`. Make couldn't skip any of them. On a warm devcontainer with a running cluster that hadn't changed at all, you still paid for image rebuild, Helm reinstall, kustomize apply, and full rollout waits. - ---- - -## The fix: stamp files per cluster - -Each cluster-side step is represented as a local file. Make's timestamp logic handles the "did this need to rerun?" question automatically. - -The stamp namespace: - -``` -.stamps/ - image/ - controller.id ← image ID written after docker build - cluster/ - kind-/ - ready ← cluster created and kubectl works - cert-manager.installed - gitea.installed - prometheus.installed - image.loaded ← image loaded into this specific cluster - crds.applied - controller.deployed - portforward.running - e2e.passed ← test result stamp -``` - -Each stamp is written **only after a real verification step** (rollout status, kubectl wait, etc.). If any prerequisite file is newer than the stamp, Make reruns that step β€” and only that step. - -When `make cleanup-cluster` runs, it also `rm -rf .stamps/cluster/kind-$(KIND_CLUSTER)` so the next run starts fresh. - ---- - -## CTX-parameterized stamp targets - -All stamp targets are generic β€” they use a single `CTX` variable instead of hardcoded cluster names. This eliminates duplication: one set of rules serves all clusters. - -### Variables - -```make -KIND_CLUSTER_E2E ?= gitops-reverser-test-e2e -KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-test-e2e-quickstart-helm -KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-test-e2e-quickstart-manifest - -# CTX: kubeconfig context for the cluster being operated on. -# Defaults to the main e2e cluster; override with CTX=kind- to reuse stamp targets for other clusters. -CTX ?= kind-$(KIND_CLUSTER_E2E) -# Derive the Kind cluster name by stripping the "kind-" prefix from CTX. -CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(CTX)) -CS := .stamps/cluster/$(CTX) # cluster stamp directory -IS := .stamps/image # image stamp directory - -GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum - -KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) -KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) -``` - -> **Note**: `GO_SOURCES` covers `cmd/` and `internal/` β€” there is no `pkg/` directory in this project. - -To target a different cluster with any stamp target, pass `CTX` on the command line: - -```sh -make CTX=kind-my-other-cluster .stamps/cluster/kind-my-other-cluster/cert-manager.installed -``` - ---- - -## Dependency graph - -``` -make test-e2e (phony) - └─ $(CS)/e2e.passed - β”œβ”€ test/e2e/** (any test change reruns tests) - └─ go test ./test/e2e - └─ BeforeSuite calls Make: - make CTX= INSTALL_NAME= NAMESPACE= \ - $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure - -$(CS)/$(NAMESPACE)/prepare-e2e.ready (stamp) - β”œβ”€ $(CS)/sut.namespace.cleaned - β”œβ”€ $(CS)/controller.deployed - β”œβ”€ $(CS)/portforward.running - └─ $(CS)/age-key.applied -``` - -Notes: - -- The Makefile stays seed-agnostic. The Go suite derives seed-suffixed `INSTALL_NAME` and `NAMESPACE` and passes - them into Make so install-time cluster-scoped names can be unique per run. -- `portforward-ensure` always runs a health check + (re)starts port-forwards as needed for the current test - process. The stamp target `$(CS)/portforward.running` is still used as a cached gate for other Make targets. - -What this buys you in practice: - -| Change you make | Steps that actually rerun | -|---|---| -| Edit a Go source file | image rebuild β†’ load into kind β†’ controller redeploy β†’ tests | -| Edit a test file only | tests only | -| Edit a CRD manifest | CRD apply β†’ controller redeploy β†’ tests | -| Edit RBAC/manager config | controller redeploy β†’ tests | -| Edit `gitea-values.yaml` | gitea reinstall β†’ port-forwards restart β†’ tests | -| Edit `ensure-prometheus-operator.sh` | prometheus reinstall β†’ port-forwards restart β†’ tests | -| Change `CERT_MANAGER_VERSION` | cert-manager reinstall β†’ controller redeploy β†’ tests | -| Change nothing | everything skipped β€” only the test binary is checked | - ---- - -## Stamp targets, step by step - -### 1 β€” Cluster ready - -```make -$(CS)/ready: test/e2e/cluster/start-cluster.sh test/e2e/cluster/cluster-template.yaml - mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/cluster/start-cluster.sh - kubectl --context $(CTX) get ns >/dev/null - touch $@ -``` - -### 2 β€” cert-manager - -```make -$(CS)/cert-manager.installed: $(CS)/ready - mkdir -p $(CS) - kubectl --context $(CTX) apply -f $(CERT_MANAGER_MANIFEST_URL) | grep -v unchanged - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-webhook --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - touch $@ -``` - -### 3 β€” Gitea - -```make -$(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml - mkdir -p $(CS) - $(HELM) repo add gitea-charts https://dl.gitea.com/charts/ 2>/dev/null || true - $(HELM) repo update gitea-charts - kubectl --context $(CTX) create namespace $(GITEA_NAMESPACE) --dry-run=client -o yaml \ - | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install gitea gitea-charts/gitea \ - --namespace $(GITEA_NAMESPACE) \ - --version $(GITEA_CHART_VERSION) \ - --values test/e2e/gitea-values.yaml - kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/gitea --timeout=300s - touch $@ -``` - -### 4 β€” Prometheus Operator - -```make -$(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-operator.sh - mkdir -p $(CS) - KUBECONTEXT=$(CTX) bash test/e2e/scripts/ensure-prometheus-operator.sh - touch $@ -``` - -### 5 β€” Image build - -For local images (no registry push), `RepoDigests` is empty. Use the image ID instead β€” it changes whenever the build output changes. - -```make -$(IS)/controller.id: $(GO_SOURCES) Dockerfile - mkdir -p $(IS) - $(CONTAINER_TOOL) build -t $(E2E_LOCAL_IMAGE) . - $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(E2E_LOCAL_IMAGE) > $@ -``` - -### 6 β€” Image loaded into cluster - -A separate stamp for loading the image into a specific cluster. This is used by both the main e2e flow (via `controller.deployed`) and the quickstart targets. - -```make -$(CS)/image.loaded: $(IS)/controller.id $(CS)/ready - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) - touch $@ -``` - -### 7 β€” CRDs applied - -```make -$(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) - mkdir -p $(CS) - kubectl --context $(CTX) apply -k config/crd - kubectl --context $(CTX) wait --for=condition=Established crd --all --timeout=120s - touch $@ -``` - -### 8 β€” Controller deployed - -`DEPLOY_INPUTS` includes `prometheus.installed` because the controller's ServiceMonitor depends on the Prometheus Operator CRDs being present. - -```make -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(IS)/controller.id \ - $(shell find config -type f -not -path 'config/crd/*') - -$(CS)/controller.deployed: $(DEPLOY_INPUTS) - mkdir -p $(CS) - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true - cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_LOCAL_IMAGE) - $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - - kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s - touch $@ -``` - -### 9 β€” Port-forwards - -`setup-port-forwards.sh` uses bare `kubectl` (no `--context` flag), so the current kubeconfig context must be set first with `kubectl config use-context $(CTX)`. - -This target stays a **file target, not `.PHONY`**. The file target skips the recipe when the stamp is newer than all prerequisites. The health-check-first pattern inside the recipe handles the case where forwards are already alive from a previous run but the stamp needed refreshing. - -```make -$(CS)/portforward.running: $(CS)/controller.deployed $(CS)/gitea.installed $(CS)/prometheus.installed - mkdir -p $(CS) - if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ - curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then \ - echo "port-forwards already healthy, skipping restart"; \ - touch $@; exit 0; \ - fi - kubectl config use-context $(CTX) - bash test/e2e/scripts/setup-port-forwards.sh - curl -fsS http://localhost:13000/api/healthz >/dev/null || { echo "Gitea health check failed"; exit 1; } - curl -fsS http://localhost:19090/-/healthy >/dev/null || { echo "Prometheus health check failed"; exit 1; } - touch $@ -``` - -The three execution paths this covers: - -| Situation | What happens | -|---|---| -| Stamp newer than all prerequisites | Make skips the recipe entirely β€” zero curl calls | -| Stamp stale, but forwards still alive (e.g. controller redeployed, forwards survived) | Fast path: two curls pass β†’ touch stamp β†’ done | -| Stamp stale and forwards are dead | Slow path: restart script β†’ two curls verify β†’ touch stamp | - -### 10 β€” E2E tests with result stamp - -```make -$(CS)/e2e.passed: $(CS)/portforward.running $(shell find test/e2e -name '*.go') - mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v - touch $@ - -.PHONY: test-e2e -test-e2e: $(CS)/e2e.passed -``` - -### 11 β€” Cluster teardown invalidates all stamps - -```make -.PHONY: cleanup-cluster -cleanup-cluster: - @if $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER)$$"; then \ - $(KIND) delete cluster --name $(KIND_CLUSTER); \ - fi - rm -rf .stamps/cluster/kind-$(KIND_CLUSTER) -``` - -`KIND_CLUSTER` defaults to `$(KIND_CLUSTER_E2E)` but can be overridden: `make cleanup-cluster KIND_CLUSTER=my-other-cluster`. - ---- - -## Quickstart tests - -Quickstart tests (`test-e2e-quickstart-helm`, `test-e2e-quickstart-manifest`) validate the user-facing install paths (Helm chart and generated `dist/install.yaml`). They use dedicated clusters to keep them isolated from the main e2e suite. - -### Design: always start with a clean cluster - -Quickstart tests **always delete and recreate their cluster** at the start of every run. This avoids a class of failures -where Helm tries to install on top of resources that were previously applied with `kubectl apply` (server-side apply), -which can surface as `metadata.managedFields must be nil` errors. A clean cluster is the simplest reliable guarantee. - -### CTX override pattern - -Quickstart targets reuse the generic stamp targets by setting `CTX` before running `go test` (the Go `BeforeSuite` -invokes `make prepare-e2e`, which uses stamp targets under `.stamps/cluster/$(CTX)/...`): - -```sh -CTX=kind-gitops-reverser-test-e2e-quickstart-helm go test ./test/e2e/ ... -``` - -The stamp rules re-evaluate `CS` and `CLUSTER_FROM_CTX` with the overridden `CTX`, so all infra is isolated per cluster. - -### Helm quickstart target - -```make -.PHONY: test-e2e-quickstart-helm -test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - CTX=$(KUBECONTEXT_QS_HELM) INSTALL_MODE=helm \ - E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework -``` - -### Manifest quickstart target - -```make -.PHONY: test-e2e-quickstart-manifest -test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - CTX=$(KUBECONTEXT_QS_MANIFEST) INSTALL_MODE=plain-manifests-file \ - E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework -``` - -The manifest target implicitly validates the generated `dist/install.yaml` via the `plain-manifests-file` install mode. - -### Quickstart smoke lives in Go - -Quickstart smoke assertions run as a Go E2E test (`quickstart-framework` label). Cluster provisioning, controller install, -and port-forward setup is owned by `make prepare-e2e` (invoked from `BeforeSuite`). - ---- - -## Targeted test runs with Ginkgo focus flags - -The entire e2e suite lives in one `Describe("Manager", ...)` block. You can run subsets without touching the test file: - -```make -.PHONY: test-e2e-gitprovider -test-e2e-gitprovider: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="GitProvider" - -.PHONY: test-e2e-watchrule -test-e2e-watchrule: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="WatchRule" - -.PHONY: test-e2e-encryption -test-e2e-encryption: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="encrypt|SOPS|age" -``` - -These are `.PHONY` (always rerun the test binary) but still benefit from stamp-based infra: they won't rebuild the image or reinstall Gitea unless something actually changed. - ---- - -## What stamps cannot protect you from - -Stamps assume the world doesn't change behind Make's back: - -- **Manual `kubectl delete` of a CRD or deployment**: the stamp still exists. Fix: `make cleanup-cluster` to reset stamps. -- **Kind cluster restart / node failure**: `$(CS)/ready` still exists. The recipe guards against this with a live `kubectl get ns` call. -- **Port-forward dies between test runs**: the `.portforward.running` stamp is stale. The curl health-check at the start of the recipe catches this. -- **cert-manager version bump without touching `CERT_MANAGER_VERSION`**: the stamp won't invalidate. Encoding the version in the stamp path (`.../cert-manager-$(CERT_MANAGER_VERSION).installed`) would prevent this, but is not currently implemented. - ---- - -## Summary - -| What | Current (after stamps) | -|---|---| -| `make test-e2e` with nothing changed | Runs tests only | -| Go code change | Rebuilds image β†’ redeploys β†’ tests | -| Test file change only | Tests only | -| CRD change | Reapplies CRDs β†’ redeploys β†’ tests | -| Gitea config change | Reinstalls Gitea β†’ tests | -| `make test-e2e-gitprovider` | Runs GitProvider tests only (infra skipped if unchanged) | -| `make test-e2e-quickstart-helm` | Deletes cluster β†’ fresh cluster β†’ cert-manager + gitea + image β†’ quickstart helm test | -| `make test-e2e-quickstart-manifest` | Deletes cluster β†’ fresh cluster β†’ cert-manager + gitea + image β†’ quickstart manifest test | diff --git a/docs/make-e2e-further.md b/docs/make-e2e-further.md deleted file mode 100644 index fdfe97d56..000000000 --- a/docs/make-e2e-further.md +++ /dev/null @@ -1,163 +0,0 @@ -# E2E: move suite setup into Make stamps (clean plan) - -## Goal - -Make `go test ./test/e2e` run **only test logic**, not cluster provisioning or stable suite setup. - -Principle: -- If a step is deterministic and its completion can be captured by a file mtime, it belongs in a Make stamp. -- If it must be unique per run (e.g., unique repo name), it stays in `BeforeAll`. - -## Target outcome - -After this change, `BeforeAll` does only: -- Gitea per-run repo setup (unique repo + checkout dir) -- Prometheus client init + availability check (pure Go + port-forward) - -Everything else becomes stamp-based and runs *outside* the test binary. - -## Changes (implementation checklist) - -## Proposed dependency graph (parallel-friendly) - -This graph is the target Make stamp topology to enable `make -j` parallelism while keeping dependencies correct. - -```mermaid -flowchart TD - GO[Go sources + Dockerfile] --> IMG[IS/controller.id] - - READY[CS/ready] --> CRDS[CS/crds.applied] - READY --> CM[CS/cert-manager.installed] - READY --> PROM[CS/prometheus.installed] - READY --> GITEA[CS/gitea.installed] - READY --> SUTNS[CS/sut.namespace.ready] - - SUTNS --> AGE[CS/age-key.applied] - - IMG --> CTRL[CS/controller.deployed] - CRDS --> CTRL - CM --> CTRL - PROM --> CTRL - SUTNS --> CTRL - - CTRL --> PF[CS/portforward.running] - GITEA --> PF - PROM --> PF - - PF --> E2E[CS/e2e.passed] - AGE --> E2E - TESTSRC[test/e2e/**/*.go] --> E2E -``` - -Parallelism notes: -- `IS/controller.id` can run in parallel with `CS/ready`. -- After `CS/ready`, `CS/crds.applied`, `CS/cert-manager.installed`, `CS/prometheus.installed`, `CS/gitea.installed`, - and `CS/sut.namespace.ready` can run in parallel. -- `CS/age-key.applied` only needs the `sut` namespace; it should not wait for controller rollout. - -### 1) Replace suite Age key generation with a stamp + Go tool - -Why: -- The suite key does not need to change per run. -- The Secret YAML should be materialized as a real file (auditable, diffable). -- The test should not generate keys or apply Secrets. - -Actions: -1. Add a small Go tool: - - Path: `test/e2e/tools/gen-age-key/main.go` - - Output files: - - `--key-file` writes an `age` identity in `age-keygen`-compatible format. - - `--secret-file` writes a Kubernetes Secret manifest containing `stringData.identity.agekey`. -2. Add a cluster-scoped stamp: - -```make -# Where tests read the private key from (overridable). -E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt - -$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml - mkdir -p $(CS) - kubectl --context $(CTX) apply -f config/namespace.yaml - kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted - touch $@ - -$(CS)/age-key.applied: $(CS)/sut.namespace.ready test/e2e/tools/gen-age-key/main.go - mkdir -p $(CS) - go run ./test/e2e/tools/gen-age-key \ - --key-file $(CS)/age-key.txt \ - --secret-file $(CS)/age-key-secret.yaml \ - --namespace sut \ - --secret-name sops-age-key - kubectl --context $(CTX) apply -f $(CS)/age-key-secret.yaml - touch $@ -``` - -3. Stop copying keys around by making the test configurable: - - In `test/e2e/e2e_test.go`, replace the hardcoded key path with: - - `E2E_AGE_KEY_FILE` env var (default `/tmp/e2e-age-key.txt`). - - In `Makefile` set `E2E_AGE_KEY_FILE=$(CS)/age-key.txt` for the test run. - -### 2) Move namespace creation/labeling out of `BeforeAll` - -Namespace creation already happens via Kustomize (`config/namespace.yaml`). - -Make it explicit and parallel-friendly by adding a namespace stamp that runs immediately after cluster readiness: -- `$(CS)/sut.namespace.ready` (create + label) - -Then: -- remove namespace creation + label commands from `BeforeAll` -- ensure `controller.deployed` depends on `$(CS)/sut.namespace.ready` (so the label is applied before pods are created) - -### 3) Remove certificate Secret polling from `BeforeAll` - -Delete `waitForCertificateSecrets()` usage and the helper entirely. - -Rationale: -- The controller `Deployment` mounts `admission-server-cert` and `audit-server-cert` as Secret volumes. -- Pods cannot become Ready (and `kubectl rollout status` cannot succeed) until those Secrets exist. - -### 4) Run preventive CRD cleanup only when the suite actually runs - -Do not put CRD cleanup in `controller.deployed` (it won’t re-run when the stamp is fresh). - -Instead, run it in the `$(CS)/e2e.passed` recipe immediately before `go test`: -- `kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true` - -This guarantees: -- Cleanup happens exactly when the test suite runs. -- It does not rely on stamp invalidation. - -### 5) Wire the new stamps into the existing chain - -Keep `portforward.running` focused on port-forward readiness. - -Preferred wiring: -- Add `$(CS)/age-key.applied` (and `$(CS)/namespace.configured` if used) as dependencies of `$(CS)/e2e.passed`. - -Example: - -```make -$(CS)/e2e.passed: $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') - mkdir -p $(CS) - kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true - KIND_CLUSTER=$(CLUSTER_FROM_CTX) \ - E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ - go test ./test/e2e/ -v -ginkgo.v - touch $@ -``` - -## Code edits summary - -- `Makefile`: add `age-key.applied` (and optionally `namespace.configured`) stamps; wire into `e2e.passed`; export `E2E_AGE_KEY_FILE`. -- `test/e2e/tools/gen-age-key/main.go`: new tool to write key file + Secret YAML. -- `test/e2e/e2e_test.go`: remove suite-side namespace/label, age secret setup, and cert-secret wait; read key path from env. -- `test/e2e/helpers.go`: delete `setupSOPSAgeSecret` and `waitForCertificateSecrets` (and related imports). - -## Validation - -Run: -- `make lint` -- `make test` -- `docker info` (must work before e2e) -- `make test-e2e` -- `make test-e2e-quickstart-manifest` -- `make test-e2e-quickstart-helm` diff --git a/docs/makefile-e2e-image-and-deploy-refactor-plan.md b/docs/makefile-e2e-image-and-deploy-refactor-plan.md deleted file mode 100644 index ac91fbd22..000000000 --- a/docs/makefile-e2e-image-and-deploy-refactor-plan.md +++ /dev/null @@ -1,159 +0,0 @@ -# Makefile E2E Image + Deployment Refactor Plan - -This plan refactors two E2E Makefile abstractions that are currently spread across multiple targets and scripts: - -1) **Image resolution**: β€œwhat image should the tests use?” (local dev build vs CI-provided image) -2) **Deployment readiness**: β€œafter *any* install method, the controller is running with the desired image” - -It is intended to complement: - -- `docs/makefile-e2e-overview-and-plan.md` (roadmap) -- `docs/e2e-shared-infra-parallel-plan.md` (shared-infra ownership contract) - -## Problems to solve (current state) - -- Image logic is duplicated: - - `E2E_IMAGE := $(if $(PROJECT_IMAGE),...)` hides whether we’re using a CI-provided image or a local build. - - Multiple targets check `PROJECT_IMAGE` and branch differently (main e2e vs quickstart flows). -- `$(CS)/$(NAMESPACE)/controller.deployed` is not a β€œdeployment contract” yet: - - It applies `config/` (install-method-specific). - - It patches a deployment **by name** (`deploy/gitops-reverser`). - - It implicitly assumes config-dir installation behavior. - -## Target contracts (end-state) - -### Contract A β€” `PROJECT_IMAGE` becomes the single source of truth - -- If CI pre-builds an image, it passes `PROJECT_IMAGE=` into `make test-e2e` / quickstart targets. -- If `PROJECT_IMAGE` is **not** provided, Make chooses a default local image reference and builds it. -- Only one place in the Makefile β€œdecides” and β€œbranches” based on whether `PROJECT_IMAGE` was provided. -- After that point, all recipes can assume `PROJECT_IMAGE` is non-empty. - -Practical consequences: - -- No more `E2E_IMAGE` indirection (or it becomes an alias of `PROJECT_IMAGE`). -- No more scattered `if [ -z "$(PROJECT_IMAGE)" ] ...` checks outside the image-resolve/build step. - -### Contract B β€” `controller.deployed` becomes install-method-agnostic - -`$(CS)/$(NAMESPACE)/controller.deployed` means: - -- β€œThere is exactly one Deployment in `$(NAMESPACE)`” -- β€œThat Deployment is configured to run `$(PROJECT_IMAGE)`” -- β€œThat Deployment has `imagePullPolicy` set to the desired value for E2E (expected: `IfNotPresent`)” -- β€œThat Deployment has completed rollout” - -Constraints: - -- It finds the Deployment dynamically (not by a hardcoded name). -- It fails fast if it finds **0** deployments or **>1** deployments (and prints what it found). -- It does **not** apply `config/` or otherwise β€œinstall” anything. Installation is a separate concern. - -## Proposed Makefile structure - -### 1) Resolve/build the image once - -Add one stamp target that is the only place allowed to branch on β€œwas `PROJECT_IMAGE` provided?”: - -- `$(IS)/project-image.ready` - -Recipe behavior: - -- If `PROJECT_IMAGE` was provided: - - Ensure the image exists in the local container runtime (`docker pull` or `docker image inspect` + pull-on-miss). - - Write the stamp. -- If `PROJECT_IMAGE` was not provided: - - Set `PROJECT_IMAGE` to the default local tag (e.g. `gitops-reverser:e2e-local`). - - Build it (reusing `$(IS)/controller.id` for cache correctness and optional annotations). - - Write the stamp. - -Makefile variable pattern to support this cleanly: - -- Capture β€œprovided or not” *before* defaulting: - - `PROJECT_IMAGE_PROVIDED := $(strip $(PROJECT_IMAGE))` -- Then default and export: - - `PROJECT_IMAGE ?= $(E2E_LOCAL_IMAGE)` - - `export PROJECT_IMAGE` - -Only `$(IS)/project-image.ready` should read `PROJECT_IMAGE_PROVIDED`. - -### 2) Always load the resolved image into Kind (all E2E clusters) - -Update `$(CS)/image.loaded` to: - -- depend on `$(IS)/project-image.ready` and `$(CS)/ready` -- run `kind load docker-image $(PROJECT_IMAGE) --name $(CLUSTER_FROM_CTX)` - -Rationale: - -- Kind nodes should not need registry credentials for CI images. -- The rest of the system can rely on `imagePullPolicy=IfNotPresent` and fast startup. - -### 3) Separate install from β€œimage + rollout” deployment contract - -Make `install-$(INSTALL_MODE)` responsible only for β€œresources exist” (Helm / dist manifests / config-dir). - -Make `$(CS)/$(NAMESPACE)/controller.deployed` responsible only for: - -1. Assert exactly one Deployment in namespace -2. Patch its container image to `$(PROJECT_IMAGE)` -3. Patch its `imagePullPolicy` (expected: `IfNotPresent`) -4. Wait for rollout readiness - -Container selection strategy (no deployment-name assumptions): - -- Prefer a single container whose image contains `gitops-reverser` (expected unique match). -- If that’s ambiguous, fail with a diagnostic dump of container names/images. - -## Dependency graph impact (high level) - -- `$(CS)/$(NAMESPACE)/prepare-e2e.ready` should depend on: - - `$(CS)/$(NAMESPACE)/namespace.cleaned` - - `$(CS)/$(NAMESPACE)/install-$(INSTALL_MODE)` - - `$(CS)/image.loaded` - - `$(CS)/$(NAMESPACE)/controller.deployed` - - `$(CS)/portforward.running` - - `$(CS)/$(NAMESPACE)/age-key.applied` - -Quickstart targets should also rely on the same image resolution + loading primitives (no custom branching). - -## Phased implementation plan (small PRs) - -### PR5a β€” Normalize `PROJECT_IMAGE` (image resolution) - -- Introduce `PROJECT_IMAGE_PROVIDED`, default `PROJECT_IMAGE`, and `export PROJECT_IMAGE`. -- Add `$(IS)/project-image.ready` and move all branching there. -- Update `$(CS)/image.loaded` to load `$(PROJECT_IMAGE)` (not `$(E2E_LOCAL_IMAGE)`). -- Remove `E2E_IMAGE` (or make it a strict alias), and remove `PROJECT_IMAGE` checks from quickstart targets. - -Acceptance: - -- `make test-e2e` and quickstart targets have exactly one β€œdecision point” for CI vs local image. -- Both local and CI runs result in `$(PROJECT_IMAGE)` existing locally and being loadable into Kind. - -### PR5b β€” Redefine `controller.deployed` as the install-agnostic contract - -- Stop applying `config/` in `$(CS)/$(NAMESPACE)/controller.deployed`. -- Implement β€œexactly one Deployment in namespace” discovery + failure modes. -- Patch image + pull policy + rollout wait without assuming a deployment name. - -Acceptance: - -- `controller.deployed` works after `install-config-dir`, `install-helm`, and `install-plain-manifests-file`. -- If the namespace contains 0 or >1 deployments, `controller.deployed` fails with a clear diagnostic. - -### PR5c β€” Align quickstart smoke harness with the new contract - -- Make quickstart Make targets perform install via `install-$(mode)` + `controller.deployed` (instead of patching in - a shell harness). -- Keep quickstart smoke assertions in Go (the `quickstart-framework` E2E spec) so install method changes don't require - shell logic updates. - -Acceptance: - -- Quickstart smoke tests no longer patch deployments directly and no longer depend on a fixed deployment name. - -## Non-goals (for this refactor) - -- Changing the Go E2E run-context ownership model from `docs/e2e-shared-infra-parallel-plan.md`. -- Expanding the suite Γ— install-method matrix beyond what already exists. diff --git a/docs/makefile-e2e-overview-and-plan.md b/docs/makefile-e2e-overview-and-plan.md deleted file mode 100644 index 82cc9b6c4..000000000 --- a/docs/makefile-e2e-overview-and-plan.md +++ /dev/null @@ -1,133 +0,0 @@ -# Makefile E2E Overview and Implementation Plan - -This is the high-level roadmap for the Makefile-driven E2E work. For the detailed stamp dependency graph, see -`docs/make-e2e-deps.md`. - -## Goals (from Makefile + suite comments) - -- Keep `make test-e2e` as the classic developer entrypoint. -- Use a **single seed** to scope a test run (namespaces + stamps), shared by Make and Go. -- Make the E2E model explicit as a **matrix**: `suite` Γ— `install-method`. -- Keep Go focused on scenario logic/assertions; Make owns cluster orchestration (Kind/Helm/kubectl/image/stamps). - -## Current reality (as of Feb 27, 2026) - -- There is a stamp-based infra model under `.stamps/cluster//...` and `.stamps/image/...`. -- The full-suite Go code shells out to Make in `BeforeSuite`: - - runs `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp; no tests inside) - - runs `portforward-ensure` to guarantee live port-forwards for the test process - - sets `kubectl config use-context $(CTX)` for helpers that don’t pass `--context` -- `make test-e2e` runs `go test ./test/e2e/...` and the Go suite owns infra prep via the Make targets above. -- Quickstart manifest uses `dist/install.yaml` (no missing Make targets). - -## Design contract (end-state) - -### 1) One β€œprepare” target that Go calls - -Go should call exactly one Make target in `BeforeSuite`: - -- `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp; no `go test` inside) - -That stamp is responsible for: - -- cleaning + recreating the controller namespace (`sut` for now) -- running the selected installer (`INSTALL_MODE=...`) -- ensuring the controller is deployed with the correct image -- ensuring shared test dependencies are ready (port-forwards, age key, etc.) - -Then Go runs tests normally and only owns scenario logic/assertions. - -### 2) Seed β†’ run namespace (shared by Make + Go) - -Inputs/outputs: - -- input: Ginkgo random seed (internal to the Go suite) -- output: seed-suffixed `INSTALL_NAME` + `NAMESPACE` passed into Make from Go - - rationale: the install name must include the seed because quickstart installs include cluster-scoped resources - that would otherwise collide by name - -The full suite should start using the seed-suffixed run namespace for test-created resources so concurrent runs -don’t collide. Keep `sut` as the controller namespace initially (parameterize later). - -### 3) Makefile matrix: suite Γ— install-method - -Keep the matrix intentionally small: - -- `suite=full` with `INSTALL_MODE=config-dir` (internal/dev install path) -- `suite=quickstart` with `INSTALL_MODE=helm|plain-manifests-file` (user-facing install paths) - -Stable entrypoints: - -- `make test-e2e` (alias for full + config-dir) -- `make test-e2e-quickstart-helm` -- `make test-e2e-quickstart-manifest` - -Optional later (only if it proves valuable): `suite=full` with `INSTALL_MODE=helm|plain-manifests-file`. - -## Implementation plan (small PRs with clear outcomes) - -### PR1 β€” Fix-first correctness (done) - -- Convert the `// ...` β€œnotes” in `Makefile` into Make comments (`# ...`) so they are not treated as recipe lines. -- Re-wire `make test-e2e` to actually run the full E2E suite again (and keep the alias stable). -- Fix `test-e2e-quickstart-manifest` to depend on `dist/install.yaml` (or rename/add the intended target). -- Make `$(CS)/portforward.running` a real stamp target (remove `.PHONY`) and add a health-check fast-path. - -Acceptance: - -- `make test-e2e` runs the suite end-to-end on a prepared cluster. -- Quickstart helm+manifest targets work locally without undefined targets. - -### PR2 β€” Implement `prepare-e2e.ready` as the single orchestration contract (done) - -- Implement `$(CS)/$(NAMESPACE)/prepare-e2e.ready` (stamp) as the β€œone target Go calls”. -- Update Go `BeforeSuite` to call `prepare-e2e.ready` (and `portforward-ensure`) and set kubectl context. - -Acceptance: - -- `go test ./test/e2e/...` works when invoked from CI/Make and always provisions infra via `prepare-e2e.ready`. - -### PR3 β€” Run scoping via seed - -- Use the Go suite’s seed as the single run identifier and pass only: - - `INSTALL_NAME=-` - - `NAMESPACE=-` -- Decide which resources should be seed-scoped: - - quickstart installs: must be seed-scoped (cluster-scoped resource name collisions) - - full suite: start with seed-scoped stamps only, then migrate test-created resources next - -Acceptance: - -- Two runs with different seeds can share a cluster without namespace collisions. - -### PR4 β€” Reduce shell quickstart assertions (Go parity) - -- (Done) Port quickstart smoke assertions into Go (commit progression, encryption checks, invalid-credential status). -- (Done) Retire the shell quickstart harness once parity is reached. - -Acceptance: - -- Quickstart smoke tests’ β€œreal assertions” live in Go, not shell. - -### PR5 β€” Simplify image resolution + make deployment install-agnostic - -This refactor makes `PROJECT_IMAGE` the single source of truth and turns `$(CS)/$(NAMESPACE)/controller.deployed` -into a true β€œdeployment contract” that is independent of the chosen install method. - -Plan: `docs/makefile-e2e-image-and-deploy-refactor-plan.md`. - -## Notes / non-goals (for now) - -- Don’t expand the suite/install matrix until the `prepare-e2e.ready` contract and run scoping are solid. -- Avoid moving Kubernetes orchestration into Go; prefer Make stamps and verified gates. - -## Why `manifests` and `helm-sync` are `.PHONY` - -They are command-style β€œentrypoint” targets, not real files. - -- They don’t produce a file named `manifests` or `helm-sync`; the real outputs are file targets like - `config/crd/bases/*.yaml` and `charts/gitops-reverser/crds/*.yaml`. -- Marking them `.PHONY` avoids accidental β€œup to date” behavior if a file named `manifests` or `helm-sync` - exists in the repo (or gets created by tooling). -- Incremental behavior still comes from the file targets: `.PHONY` only forces Make to *check* the graph when - you run `make manifests` / `make helm-sync`. diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index df769f3dc..0bb34c4a3 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -5,7 +5,7 @@ set -euo pipefail -CLUSTER_NAME="${K3D_CLUSTER:-gitops-reverser-test-e2e}" +CLUSTER_NAME="${CLUSTER_NAME:-gitops-reverser-test-e2e}" AUDIT_DIR_REL="test/e2e/cluster/audit" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b41c7da96..c97bf6f17 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -59,12 +59,14 @@ var _ = AfterSuite(func() { func ensureE2EPrepared() { ctx := resolveE2EContext() setE2EKubectlContext(ctx) - seed := ginkgoRandomSeed() ns := resolveE2ENamespace() - installName := resolveE2EInstallName(seed) + installMode, err := resolveE2EInstallMode() + Expect(err).NotTo(HaveOccurred(), "INSTALL_MODE environment variable must be set for e2e runs") + installName := resolveE2EInstallName(ns) cmd := makeCommand( fmt.Sprintf("CTX=%s", ctx), fmt.Sprintf("INSTALL_NAME=%s", installName), + fmt.Sprintf("INSTALL_MODE=%s", installMode), fmt.Sprintf("NAMESPACE=%s", ns), "prepare-e2e", ) @@ -106,16 +108,18 @@ func makeCommand(args ...string) *exec.Cmd { return cmd } -func resolveE2EInstallName(seed int64) string { +func resolveE2EInstallName(namespace string) string { if value := strings.TrimSpace(os.Getenv("INSTALL_NAME")); value != "" { return value } - return fmt.Sprintf("gitops-reverser-%d", seed) + return namespace } -func ginkgoRandomSeed() int64 { - suiteConfig, _ := GinkgoConfiguration() - return suiteConfig.RandomSeed +func resolveE2EInstallMode() (string, error) { + if value := strings.TrimSpace(os.Getenv("INSTALL_MODE")); value != "" { + return value, nil + } + return "", fmt.Errorf("INSTALL_MODE environment variable must be set") } func resolveE2EContext() string { @@ -123,13 +127,6 @@ func resolveE2EContext() string { return value } - if cluster := strings.TrimSpace(os.Getenv("KIND_CLUSTER")); cluster != "" { - if strings.HasPrefix(cluster, "kind-") { - return cluster - } - return fmt.Sprintf("kind-%s", cluster) - } - output, err := kubectlRun("config", "current-context") if err == nil { if value := strings.TrimSpace(output); value != "" { diff --git a/test/utils/utils.go b/test/utils/utils.go index 057d1a957..8630ce957 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -22,7 +22,6 @@ package utils import ( "bufio" "bytes" - "context" "fmt" "os" "os/exec" @@ -51,19 +50,6 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } -// LoadImageToKindClusterWithName loads a local docker image to the kind cluster. -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - ctx := context.Background() - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.CommandContext(ctx, "kind", kindOptions...) - _, err := Run(cmd) - return err -} - // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string { From fbeaddbd84101ca91aed96e75e8b4fb5437f0990 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 17:08:19 +0000 Subject: [PATCH 23/30] fix: The order in which the dependencies is (perhaps to?) important --- .claude/settings.local.json | 3 +- Makefile | 27 ++++++++---- test/e2e/e2e_test.go | 2 +- test/e2e/tools/gen-age-key/main.go | 71 +++++++++++++++++++++++------- 4 files changed, 75 insertions(+), 28 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index a15d65dea..50613e4e6 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -17,7 +17,8 @@ "Bash(grep -r \"make\\\\|Make\" /workspaces/gitops-reverser2/test/e2e/*.go)", "Bash(make .stamps/image/controller.id)", "Bash(docker version)", - "Bash(docker buildx version)" + "Bash(docker buildx version)", + "Bash(go build ./test/e2e/tools/gen-age-key/...)" ] } } diff --git a/Makefile b/Makefile index 63257b1cf..26fa64e51 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ $(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install $(CS)/$(NAMESPACE)/label-namespace.ready \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ - $(CS)/$(NAMESPACE)/age-key.applied + $(CS)/$(NAMESPACE)/sops-secret.applied mkdir -p $(@D) touch $@ @@ -282,14 +282,23 @@ $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-opera kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ -$(CS)/$(NAMESPACE)/age-key.applied: Makefile $(CS)/$(NAMESPACE)/label-namespace.ready test/e2e/tools/gen-age-key/main.go +# Step 1: Generate age key file β€” no cluster/namespace dependency; safe to run before installation. +$(CS)/age-key.txt: Makefile test/e2e/tools/gen-age-key/main.go mkdir -p $(@D) go run ./test/e2e/tools/gen-age-key \ - --key-file $(CS)/$(NAMESPACE)/age-key.txt \ - --secret-file $(CS)/$(NAMESPACE)/age-key-secret.yaml \ + --key-file $@ + +# Step 2: Derive Kubernetes Secret manifest from the key file β€” still no cluster dependency. +$(CS)/$(NAMESPACE)/sops-secret.yaml: $(CS)/age-key.txt Makefile + go run ./test/e2e/tools/gen-age-key \ + --key-file $(CS)/age-key.txt \ + --secret-file $@ \ --namespace $(NAMESPACE) \ --secret-name sops-age-key - kubectl --context $(CTX) apply -f $(CS)/$(NAMESPACE)/age-key-secret.yaml + +# Step 3: Apply the secret into the namespace β€” requires the namespace to already exist. +$(CS)/$(NAMESPACE)/sops-secret.applied: Makefile $(CS)/$(NAMESPACE)/sops-secret.yaml $(CS)/$(NAMESPACE)/label-namespace.ready + kubectl --context $(CTX) apply -f $(CS)/$(NAMESPACE)/sops-secret.yaml touch $@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile @@ -350,11 +359,11 @@ portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure p mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh -E2E_TEST_INPUTS := $(CS)/$(NAMESPACE)/age-key.applied $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) +E2E_TEST_INPUTS := $(CS)/age-key.txt $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) $(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) CTX=$(CTX) INSTALL_MODE=$(INSTALL_MODE) NAMESPACE=$(NAMESPACE) \ - E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ @@ -428,14 +437,14 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTA test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster CTX=$(CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ - E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster CTX=$(CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ - E2E_AGE_KEY_FILE=$(CS)/$(NAMESPACE)/age-key.txt \ + E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: cleanup-port-forwards diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c272c0b45..dfd64336d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -135,7 +135,7 @@ var _ = Describe("Manager", Ordered, func() { }) // Optimize timeouts for faster test execution - SetDefaultEventuallyTimeout(20 * time.Second) + SetDefaultEventuallyTimeout(30 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) Context("Manager", func() { diff --git a/test/e2e/tools/gen-age-key/main.go b/test/e2e/tools/gen-age-key/main.go index fa83b9495..b043399f7 100644 --- a/test/e2e/tools/gen-age-key/main.go +++ b/test/e2e/tools/gen-age-key/main.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "filippo.io/age" @@ -40,29 +41,49 @@ func main() { var namespace string var secretName string - flag.StringVar(&keyFile, "key-file", "", "path to write age identity file") - flag.StringVar(&secretFile, "secret-file", "", "path to write Kubernetes Secret manifest") - flag.StringVar(&namespace, "namespace", "", "Secret namespace") - flag.StringVar(&secretName, "secret-name", "", "Secret name") + flag.StringVar(&keyFile, "key-file", "", "path to write (or read, if it already exists) age identity file") + flag.StringVar(&secretFile, "secret-file", "", "path to write Kubernetes Secret manifest (optional; requires --namespace and --secret-name)") + flag.StringVar(&namespace, "namespace", "", "Secret namespace (required with --secret-file)") + flag.StringVar(&secretName, "secret-name", "", "Secret name (required with --secret-file)") flag.Parse() - if keyFile == "" || secretFile == "" || namespace == "" || secretName == "" { - exitf("all flags are required: --key-file --secret-file --namespace --secret-name") + if keyFile == "" { + exitf("--key-file is required") + } + if secretFile != "" && (namespace == "" || secretName == "") { + exitf("--namespace and --secret-name are required when --secret-file is provided") } - identity, err := age.GenerateX25519Identity() - if err != nil { - exitf("generate age identity: %v", err) + // If the key file already exists, reuse it; otherwise generate a fresh identity. + var identity *age.X25519Identity + if _, err := os.Stat(keyFile); err == nil { + parsed, err := parseIdentityFile(keyFile) + if err != nil { + exitf("parse existing key file: %v", err) + } + identity = parsed + } else if os.IsNotExist(err) { + generated, err := age.GenerateX25519Identity() + if err != nil { + exitf("generate age identity: %v", err) + } + identity = generated + + keyContent := fmt.Sprintf( + "# created: %s\n# public key: %s\n%s\n", + time.Now().UTC().Format(time.RFC3339), + identity.Recipient(), + identity.String(), + ) + if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { + exitf("write key file: %v", err) + } + } else { + exitf("stat key file: %v", err) } - keyContent := fmt.Sprintf( - "# created: %s\n# public key: %s\n%s\n", - time.Now().UTC().Format(time.RFC3339), - identity.Recipient(), - identity.String(), - ) - if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { - exitf("write key file: %v", err) + if secretFile == "" { + return } secretContent := fmt.Sprintf(`apiVersion: v1 @@ -80,6 +101,22 @@ stringData: } } +// parseIdentityFile reads an age identity file, locates the AGE-SECRET-KEY line, +// and returns the parsed X25519Identity. +func parseIdentityFile(path string) (*age.X25519Identity, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "AGE-SECRET-KEY-") { + return age.ParseX25519Identity(line) + } + } + return nil, fmt.Errorf("no AGE-SECRET-KEY line found in %s", path) +} + func writeFile(path string, content []byte, mode os.FileMode) error { parent := filepath.Dir(path) if parent != "" && parent != "." { From 23c7a5250b8b0032940034dcd20d3ed0a995770d Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Mon, 2 Mar 2026 17:13:56 +0000 Subject: [PATCH 24/30] fix: Let's get the linter happy again --- test/e2e/e2e_suite_test.go | 3 +- test/e2e/tools/gen-age-key/main.go | 60 ++++++++++++++++-------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index c97bf6f17..75d8b9417 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -19,6 +19,7 @@ limitations under the License. package e2e import ( + "errors" "fmt" "os" "os/exec" @@ -119,7 +120,7 @@ func resolveE2EInstallMode() (string, error) { if value := strings.TrimSpace(os.Getenv("INSTALL_MODE")); value != "" { return value, nil } - return "", fmt.Errorf("INSTALL_MODE environment variable must be set") + return "", errors.New("INSTALL_MODE environment variable must be set") } func resolveE2EContext() string { diff --git a/test/e2e/tools/gen-age-key/main.go b/test/e2e/tools/gen-age-key/main.go index b043399f7..a0279498b 100644 --- a/test/e2e/tools/gen-age-key/main.go +++ b/test/e2e/tools/gen-age-key/main.go @@ -42,7 +42,8 @@ func main() { var secretName string flag.StringVar(&keyFile, "key-file", "", "path to write (or read, if it already exists) age identity file") - flag.StringVar(&secretFile, "secret-file", "", "path to write Kubernetes Secret manifest (optional; requires --namespace and --secret-name)") + flag.StringVar(&secretFile, "secret-file", "", + "path to write Kubernetes Secret manifest (optional; requires --namespace and --secret-name)") flag.StringVar(&namespace, "namespace", "", "Secret namespace (required with --secret-file)") flag.StringVar(&secretName, "secret-name", "", "Secret name (required with --secret-file)") flag.Parse() @@ -54,32 +55,9 @@ func main() { exitf("--namespace and --secret-name are required when --secret-file is provided") } - // If the key file already exists, reuse it; otherwise generate a fresh identity. - var identity *age.X25519Identity - if _, err := os.Stat(keyFile); err == nil { - parsed, err := parseIdentityFile(keyFile) - if err != nil { - exitf("parse existing key file: %v", err) - } - identity = parsed - } else if os.IsNotExist(err) { - generated, err := age.GenerateX25519Identity() - if err != nil { - exitf("generate age identity: %v", err) - } - identity = generated - - keyContent := fmt.Sprintf( - "# created: %s\n# public key: %s\n%s\n", - time.Now().UTC().Format(time.RFC3339), - identity.Recipient(), - identity.String(), - ) - if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { - exitf("write key file: %v", err) - } - } else { - exitf("stat key file: %v", err) + identity, err := loadOrGenerateIdentity(keyFile) + if err != nil { + exitf("%v", err) } if secretFile == "" { @@ -101,6 +79,34 @@ stringData: } } +// loadOrGenerateIdentity returns the X25519Identity from keyFile if it already exists, +// or generates a fresh one and writes it to keyFile. +func loadOrGenerateIdentity(keyFile string) (*age.X25519Identity, error) { + _, statErr := os.Stat(keyFile) + if statErr != nil && !os.IsNotExist(statErr) { + return nil, fmt.Errorf("stat key file: %w", statErr) + } + if statErr == nil { + return parseIdentityFile(keyFile) + } + + // File does not exist β€” generate a fresh identity. + identity, err := age.GenerateX25519Identity() + if err != nil { + return nil, fmt.Errorf("generate age identity: %w", err) + } + keyContent := fmt.Sprintf( + "# created: %s\n# public key: %s\n%s\n", + time.Now().UTC().Format(time.RFC3339), + identity.Recipient(), + identity.String(), + ) + if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { + return nil, fmt.Errorf("write key file: %w", err) + } + return identity, nil +} + // parseIdentityFile reads an age identity file, locates the AGE-SECRET-KEY line, // and returns the parsed X25519Identity. func parseIdentityFile(path string) (*age.X25519Identity, error) { From dc250ba27cddf59b4a608a0976e1ec1f7ae28f99 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 10:16:20 +0000 Subject: [PATCH 25/30] chore: Cleaning up, let's do a one Makefile approach first --- .claude/settings.local.json | 19 +- .devcontainer/Dockerfile | 3 +- .github/workflows/ci.yml | 5 + Makefile | 88 ++++---- checkmake.ini | 2 + docs/e2e/namespace-label-cleanup-analysis.md | 210 +++++++++++++++++++ docs/e2e/notes-whats-going-wrnog.txt | 30 +++ test/e2e/scripts/setup-gitea.sh | 3 +- 8 files changed, 314 insertions(+), 46 deletions(-) create mode 100644 checkmake.ini create mode 100644 docs/e2e/namespace-label-cleanup-analysis.md create mode 100644 docs/e2e/notes-whats-going-wrnog.txt diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 50613e4e6..d0e26f573 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -18,7 +18,24 @@ "Bash(make .stamps/image/controller.id)", "Bash(docker version)", "Bash(docker buildx version)", - "Bash(go build ./test/e2e/tools/gen-age-key/...)" + "Bash(go build ./test/e2e/tools/gen-age-key/...)", + "Bash(make cleanup-cluster)", + "Bash(make -n prepare-e2e INSTALL_MODE=helm)", + "Bash(checkmake Makefile)", + "Bash(docker info)", + "Bash(make fmt vet)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e get clusterrole,clusterrolebinding -l app.kubernetes.io/part-of=gitops-reverser --no-headers)", + "Read(//workspaces/gitops-reverser2/**)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e get crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai -o jsonpath='{range .items[*]}{.metadata.name}: managedFields manager={.metadata.managedFields[0].manager}{\"\"\\\\n\"\"}{end}')", + "Bash(helm upgrade --help)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete ns gitops-reverser --ignore-not-found=true)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e wait --for=delete ns/gitops-reverser --timeout=120s)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai --ignore-not-found=true)", + "Bash(grep -l \"managedFields\" charts/gitops-reverser/crds/*.yaml)", + "Bash(grep -l \"managedFields\" config/crd/bases/*.yaml)", + "Bash(helm install gitops-reverser charts/gitops-reverser --kube-context k3d-gitops-reverser-test-e2e --namespace gitops-reverser --create-namespace --debug)", + "Bash(helm version)" ] } } diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index cd874890d..be1940537 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -85,7 +85,8 @@ RUN chgrp -R godev /go && \ # Install Go tools used by the project (using @version doesn't need go.mod) # These run AFTER permissions are set to ensure proper group ownership RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 \ - && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \ + && go install github.com/mrtazz/checkmake@latest # Initialize golangci-lint cache by running it once on an empty directory # This downloads linter dependencies without needing source code diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a77cd88f..fd279d4cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,7 @@ jobs: controller-gen --version k3d version docker --version + checkmake --version echo 'βœ… All CI container tools verified' " @@ -125,6 +126,7 @@ jobs: golangci-lint version k3d version docker --version + checkmake --version echo 'βœ… All dev container tools verified' " @@ -193,6 +195,9 @@ jobs: - name: Configure Git safe directory (for now needed as workarround https://github.com/actions/checkout/issues/2031) run: git config --global --add safe.directory /__w/gitops-reverser/gitops-reverser + - name: checkmake Makefile + run: checkmake Makefile + - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: diff --git a/Makefile b/Makefile index 26fa64e51..31a99f2e4 100644 --- a/Makefile +++ b/Makefile @@ -51,13 +51,14 @@ MANIFEST_OUTPUTS := $(CRD_BASE_OUTPUTS) \ config/webhook/manifests.yaml MANIFEST_INPUTS := $(shell find api internal cmd -type f -name '*.go' \ ! -name '*_test.go' ! -name 'zz_generated.deepcopy.go') +CONTROLLER_GEN_ARGS := rbac:roleName=gitops-reverser crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: manifests manifests: $(MANIFEST_OUTPUTS) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(MANIFEST_OUTPUTS) &: $(MANIFEST_INPUTS) @rm -f config/crd/bases/*.yaml \ config/rbac/role.yaml \ config/webhook/manifests.yaml - $(CONTROLLER_GEN) rbac:roleName=gitops-reverser crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) $(CONTROLLER_GEN_ARGS) HELM_CRD_OUTPUTS := $(patsubst config/crd/bases/%,charts/gitops-reverser/crds/%,$(CRD_BASE_OUTPUTS)) HELM_SYNC_OUTPUTS := $(HELM_CRD_OUTPUTS) \ @@ -73,6 +74,7 @@ GENERATE_INPUTS := $(shell find api -type f -name '*.go' ! -name '*_test.go' \ ! -name 'zz_generated.deepcopy.go') hack/boilerplate.go.txt GENERATE_OUTPUT := api/v1alpha1/zz_generated.deepcopy.go +.PHONY: generate generate: $(GENERATE_OUTPUT) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(GENERATE_OUTPUT): $(GENERATE_INPUTS) @@ -137,7 +139,6 @@ prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). $(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done \ - $(CS)/$(NAMESPACE)/label-namespace.ready \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ $(CS)/$(NAMESPACE)/sops-secret.applied @@ -226,7 +227,12 @@ ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk - # Gitea E2E Configuration GITEA_NAMESPACE ?= gitea-e2e -GITEA_CHART_VERSION ?= 12.5.0 # https://gitea.com/gitea/helm-gitea +GITEA_RELEASE_NAME ?= gitea +GITEA_HELM_REPO_NAME ?= gitea-charts +GITEA_HELM_REPO_URL ?= https://dl.gitea.com/charts/ +GITEA_CHART_NAME ?= gitea +GITEA_CHART_VERSION ?= 12.5.0 # https://gitea.com/gitea/helm-gitea +GITEA_WAIT_TIMEOUT ?= 300s .PHONY: setup-envtest setup-envtest: ## Setup envtest binaries for unit tests @@ -258,16 +264,17 @@ $(CS)/cert-manager.installed: $(CS)/ready $(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml mkdir -p $(CS) - $(HELM) repo add gitea-charts https://dl.gitea.com/charts/ 2>/dev/null || true - $(HELM) repo update gitea-charts + $(HELM) repo add $(GITEA_HELM_REPO_NAME) $(GITEA_HELM_REPO_URL) 2>/dev/null || true + $(HELM) repo update $(GITEA_HELM_REPO_NAME) kubectl --context $(CTX) create namespace $(GITEA_NAMESPACE) --dry-run=client -o yaml \ | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install gitea gitea-charts/gitea \ + $(HELM) --kube-context $(CTX) upgrade --install $(GITEA_RELEASE_NAME) \ + $(GITEA_HELM_REPO_NAME)/$(GITEA_CHART_NAME) \ --namespace $(GITEA_NAMESPACE) \ --version $(GITEA_CHART_VERSION) \ --values test/e2e/gitea-values.yaml - kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/gitea --timeout=300s - touch $@ + kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/$(GITEA_RELEASE_NAME) --timeout=$(GITEA_WAIT_TIMEOUT) + echo $(GITEA_CHART_VERSION) > $@ PROMETHEUS_SETUP_MANIFESTS := $(shell find test/e2e/setup/prometheus -type f -name '*.yaml') $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-operator.sh $(PROMETHEUS_SETUP_MANIFESTS) @@ -282,6 +289,11 @@ $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-opera kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ +# Aggregate stamp for external E2E services required by tests. +$(CS)/services.ready: $(CS)/cert-manager.installed $(CS)/prometheus.installed $(CS)/gitea.installed + mkdir -p $(CS) + touch $@ + # Step 1: Generate age key file β€” no cluster/namespace dependency; safe to run before installation. $(CS)/age-key.txt: Makefile test/e2e/tools/gen-age-key/main.go mkdir -p $(@D) @@ -297,7 +309,7 @@ $(CS)/$(NAMESPACE)/sops-secret.yaml: $(CS)/age-key.txt Makefile --secret-name sops-age-key # Step 3: Apply the secret into the namespace β€” requires the namespace to already exist. -$(CS)/$(NAMESPACE)/sops-secret.applied: Makefile $(CS)/$(NAMESPACE)/sops-secret.yaml $(CS)/$(NAMESPACE)/label-namespace.ready +$(CS)/$(NAMESPACE)/sops-secret.applied: $(CS)/$(NAMESPACE)/sops-secret.yaml kubectl --context $(CTX) apply -f $(CS)/$(NAMESPACE)/sops-secret.yaml touch $@ @@ -309,10 +321,7 @@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile -t $(E2E_LOCAL_IMAGE) . $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(E2E_LOCAL_IMAGE) > $@ -ifeq ($(PROJECT_IMAGE_PROVIDED),) -$(IS)/project-image.ready: $(IS)/controller.id -endif -$(IS)/project-image.ready: Makefile +$(IS)/project-image.ready: $(if $(PROJECT_IMAGE_PROVIDED),,$(IS)/controller.id) mkdir -p $(IS) @set -euo pipefail; \ if [ -n "$(PROJECT_IMAGE_PROVIDED)" ]; then \ @@ -355,7 +364,7 @@ $(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/insta touch "$@" .PHONY: portforward-ensure -portforward-ensure: $(CS)/gitea.installed $(CS)/prometheus.installed ## Ensure port-forwards are running (always checks) +portforward-ensure: $(CS)/services.ready ## Ensure port-forwards are running (always checks) mkdir -p $(CS) E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh @@ -373,7 +382,7 @@ CONTROLLER_DEPLOY_SELECTOR ?= app.kubernetes.io/part-of=gitops-reverser # INSTALL_MODE, INSTALL_NAME, NAMESPACE defaults are defined near the E2E variables section above. VALID_INSTALL_MODES := config-dir helm plain-manifests-file ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) - $(error INSTALL_MODE must be one of: $(VALID_INSTALL_MODES)) + $(error INSTALL_MODE must be one of [$(VALID_INSTALL_MODES)]) endif .PHONY: install install-helm install-plain-manifests-file install-config-dir @@ -388,8 +397,7 @@ install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.done CONFIG_DIR_INPUTS := $(shell find config -type f) ##@ Deployments of all manifests needed to run -$(CS)/$(NAMESPACE)/helm/install.done: INSTALL_MODE := helm -$(CS)/$(NAMESPACE)/helm/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done $(HELM_SYNC_OUTPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/helm/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/helm/cleanup-e2e-installs.done $(HELM_SYNC_OUTPUTS) @set -euo pipefail; \ helm_args=( \ upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ @@ -407,8 +415,7 @@ $(CS)/$(NAMESPACE)/helm/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup helm "$${helm_args[@]}"; \ touch $@ -$(CS)/$(NAMESPACE)/config-dir/install.done: INSTALL_MODE := config-dir -$(CS)/$(NAMESPACE)/config-dir/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/config-dir/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/config-dir/cleanup-e2e-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) @set -euo pipefail; \ tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ @@ -418,8 +425,7 @@ $(CS)/$(NAMESPACE)/config-dir/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/c (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) | $(KUBECTL) --context "$(CTX)" apply -f - touch $@ -$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: INSTALL_MODE := plain-manifests-file -$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done dist/install.yaml $(CS)/cert-manager.installed $(CS)/prometheus.installed +$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/plain-manifests-file/cleanup-e2e-installs.done dist/install.yaml @set -euo pipefail; \ ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ @@ -434,14 +440,14 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/$(NAMESPACE)/$(INSTA touch $@ .PHONY: test-e2e-quickstart-helm -test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster +test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) CTX=$(CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest -test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster +test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) CTX=$(CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ @@ -454,31 +460,29 @@ cleanup-port-forwards: ## Stop all port-forwards @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true @echo "βœ… Port-forwards stopped" -$(CS)/$(NAMESPACE)/label-namespace.ready: Makefile $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done - mkdir -p $(@D) - kubectl --context $(CTX) label --overwrite ns $(NAMESPACE) \ - $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) \ - app.kubernetes.io/name=gitops-reverser control-plane=gitops-reverser \ - pod-security.kubernetes.io/enforce=restricted - @echo "βœ… Added $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) label to namespace $(NAMESPACE)" - touch $@ - $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done: Makefile $(CS)/ready @set -euo pipefail; \ ctx="$(CTX)"; \ - ns_list="$$(kubectl --context "$$ctx" get ns -l $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) -o jsonpath='{.items[*].metadata.name}' || true)"; \ - if [ -z "$$ns_list" ]; then \ - echo "ℹ️ No e2e=true namespaces in '$$ctx'"; \ + ns="$(NAMESPACE)"; \ + if kubectl --context "$$ctx" get ns "$$ns" >/dev/null 2>&1; then \ + echo "🧹 Deleting namespace '$$ns' in '$$ctx'"; \ + kubectl --context "$$ctx" delete ns "$$ns" --ignore-not-found=true; \ + kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ + rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ + rm -f ".stamps/cluster/$$ctx/image.loaded" 2>/dev/null || true; \ else \ - echo "🧹 Deleting $(E2E_NAMESPACE_LABEL_KEY)=$(E2E_NAMESPACE_LABEL_VALUE) namespaces in '$$ctx': $$ns_list"; \ - kubectl --context "$$ctx" delete ns $$ns_list --ignore-not-found=true; \ - for ns in $$ns_list; do \ - kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ - rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ - done; \ + echo "ℹ️ Namespace '$$ns' does not exist in '$$ctx'"; \ fi; \ - # Clear single-instance blockers (safe always) + # Clear cluster-scoped resources that survive namespace deletion (safe always) kubectl --context "$$ctx" delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true || true; \ kubectl --context "$$ctx" delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true || true; \ + kubectl --context "$$ctx" delete clusterrole,clusterrolebinding -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true || true; \ + kubectl --context "$$ctx" delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai --ignore-not-found=true || true; \ mkdir -p "$(@D)"; \ touch "$@" + +##@ Cleanup + +.PHONY: clean +clean: ## Remove build artifacts (binaries, coverage, generated dist files, build stamps) + rm -rf bin/ cover.out dist/ .stamps/ diff --git a/checkmake.ini b/checkmake.ini new file mode 100644 index 000000000..275c70cd4 --- /dev/null +++ b/checkmake.ini @@ -0,0 +1,2 @@ +[maxbodylength] +maxBodyLength = 20 diff --git a/docs/e2e/namespace-label-cleanup-analysis.md b/docs/e2e/namespace-label-cleanup-analysis.md new file mode 100644 index 000000000..e7425db6f --- /dev/null +++ b/docs/e2e/namespace-label-cleanup-analysis.md @@ -0,0 +1,210 @@ +# E2E Namespace Label Cleanup β€” Investigation & Analysis + +## The Problem + +Running the following sequence fails intermittently: + +```bash +make cleanup-cluster +make test-e2e # INSTALL_MODE=config-dir (default) +make test-e2e-quickstart-helm # INSTALL_MODE=helm +``` + +The failure: when switching from `config-dir` to `helm`, the cleanup step cannot find the +`gitops-reverser` namespace to delete it. The namespace has no `e2e=true` label, so the label +selector in `cleanup-e2e-installs.done` finds nothing and skips deletion. Helm then tries to +install into an already-provisioned namespace, causing a conflict. + +--- + +## Makefile Architecture (Stamp Files) + +The Makefile uses stamp files (empty files whose existence/mtime tracks build state) to express +a dependency graph. The relevant chain for switching install modes: + +``` +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done + └─ deletes all "e2e=true" namespaces in the cluster; removes their stamp dirs +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done + └─ installs controller into namespace (creates namespace if needed) +$(CS)/$(NAMESPACE)/label-namespace.ready + └─ kubectl label --overwrite ns $(NAMESPACE) e2e=true ... +$(CS)/$(NAMESPACE)/sops-secret.applied +$(CS)/$(NAMESPACE)/controller.deployed +$(CS)/$(NAMESPACE)/prepare-e2e.ready +``` + +Where `$(CS) = .stamps/cluster/$(CTX)` and `$(INSTALL_MODE)` is one of `helm`, `config-dir`, +or `plain-manifests-file`. + +### Why this should work + +`label-namespace.ready` depends on `$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done`. When +`INSTALL_MODE` changes (e.g. config-dir β†’ helm): + +- `helm/install.done` does **not** exist +- `label-namespace.ready` is therefore **out-of-date** (its prerequisite is missing) +- GNU Make rebuilds it **after** `helm/install.done` is built +- `helm/install.done` depends on `helm/cleanup-e2e-installs.done` (which runs first) + +So the intended order is: **cleanup β†’ install β†’ label**. + +--- + +## Investigation Findings + +### 1. GNU Make correctly handles deleted prerequisites + +When `cleanup-e2e-installs.done` runs, it executes: +```bash +rm -rf ".stamps/cluster/$ctx/$ns" +``` +This deletes **all** stamps under `gitops-reverser/`, including `label-namespace.ready` and +`sops-secret.yaml`. These files were considered up-to-date at **plan time** (before any recipes +ran) but are gone when their recipe would execute. + +**Empirically verified**: GNU Make evaluates the dependency graph once at the start, marks +targets for rebuild, then executes in order. When a prerequisite is deleted mid-build by another +recipe, Make correctly re-creates it β€” it does not skip it just because its own prerequisites +haven't changed since plan time. + +### 2. Dry-run output shows correct ordering + +Running `make -n prepare-e2e INSTALL_MODE=helm` (with config-dir stamps present) produces: + +``` +# helm/cleanup-e2e-installs.done ← deletes gitops-reverser namespace + stamps +# helm/install.done ← helm installs, creates fresh namespace +# label-namespace.ready ← kubectl label e2e=true applied +# sops-secret.applied ← secret applied into namespace +# prepare-e2e.ready +``` + +The logical order is correct per the dry-run. + +### 3. `sops-secret.yaml` is NOT in the dry-run plan + +At plan time, `sops-secret.yaml` is considered up-to-date (its deps `age-key.txt` and `Makefile` +are unchanged). But the cleanup deletes it. GNU Make re-creates it correctly because +`sops-secret.applied` depends on it and is in the rebuild plan. + +Additionally, `gen-age-key` calls `os.MkdirAll` before writing, so it can recreate the file +even after the stamp directory was removed. + +### 4. The `config/namespace.yaml` does not include `e2e=true` + +```yaml +# config/namespace.yaml +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + control-plane: gitops-reverser + pod-security.kubernetes.io/enforce: restricted + name: sut # replaced by kustomize to the actual NAMESPACE +``` + +When `config-dir/install.done` runs `kubectl apply` (via kustomize), it sets +`kubectl.kubernetes.io/last-applied-configuration` on the namespace **without** `e2e=true`. + +The `e2e=true` label is only applied by the **subsequent** `label-namespace.ready` step via +`kubectl label --overwrite`. Client-side apply preserves externally-added labels (they are not in +the `last-applied-configuration` so `kubectl apply` does not remove them). However, this relies +on `label-namespace.ready` always running **after** any `kubectl apply` that touches the namespace. + +### 5. `setup-gitea.sh` creates the namespace without `e2e=true` + +In `test/e2e/scripts/setup-gitea.sh`, the `setup_credentials()` function does: +```bash +TARGET_NAMESPACE=${TARGET_NAMESPACE:-${SUT_NAMESPACE:-${QUICKSTART_NAMESPACE:-${NAMESPACE:-sut}}}} +kubectl create namespace "$TARGET_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - +``` + +During `make test-e2e` (INSTALL_MODE=config-dir, NAMESPACE=gitops-reverser): +- `NAMESPACE=gitops-reverser` is in the environment +- `SUT_NAMESPACE` and `QUICKSTART_NAMESPACE` are **not** set +- So `TARGET_NAMESPACE=gitops-reverser` +- `kubectl apply` is called **without `--context`**, using the default kubeconfig context + +This `kubectl apply` on the namespace spec has **no `e2e=true` label**. Whether this strips +the label depends on exact timing (before or after `label-namespace.ready` ran) and on +server-side vs client-side apply semantics. + +During `make test-e2e-quickstart-helm`, the quickstart framework sets `SUT_NAMESPACE=sut` in +the subprocess environment before calling `setup-gitea.sh`, so `TARGET_NAMESPACE=sut` β€” the +`gitops-reverser` namespace is not touched here. + +### 6. Potential server-side apply conflict + +If the namespace was first created by `kubectl apply` (kustomize) and later `helm install +--create-namespace` tries to manage the same namespace object via server-side apply, there +can be field ownership conflicts. Helm uses SSA which might overwrite or drop labels managed +by other field managers. + +--- + +## Unresolved Root Cause + +Despite the above investigation, the **exact mechanism** causing `e2e=true` to be absent when +`helm/cleanup-e2e-installs.done` runs has not been definitively identified without a live +execution trace. The strongest candidates are: + +| Hypothesis | Likelihood | How to verify | +|---|---|---| +| `setup-gitea.sh` applies namespace without `e2e=true` before `label-namespace.ready` runs | Medium | Add `--context` to every kubectl in setup-gitea.sh; check timing in logs | +| Helm SSA strips labels when reinstalling without deletion first | Medium | Check if cleanup actually deleted the namespace before helm install | +| `label-namespace.ready` stamp survives but Kubernetes label was somehow reverted | Low | Add a check in `prepare-e2e.ready` recipe that verifies the label | +| INSTALL_MODE variable not correctly propagated to child make | Low | Print INSTALL_MODE in cleanup recipe | + +--- + +## The User's Pragmatic Notes + +From `notes-whats-going-wrnog.txt`: + +> Perhaps I should go a bit smarter about this: or more pragmatic. I could also have the same +> namespace, always, for now. And just make sure that I have a task that always first deletes the +> old namespace that would also prevent me from sitting in this rabbit hole. + +This is a sound alternative. Instead of relying on the `e2e=true` label to find-and-delete the +namespace, the cleanup target could unconditionally delete the **known** namespace (`gitops-reverser`) +regardless of labels. The label mechanism adds indirection that is fragile when cluster state and +stamp state diverge. + +Other notes the user captured: +- Move Gitea installation to a higher (cluster-level) stamp, similar to how `age-key.txt` moved + to cluster level β€” Gitea is cluster-wide infrastructure, not per-namespace +- Move Gitea credential creation (`setup-gitea.sh` secret creation) to a higher level stamp so it + runs once, not per test invocation +- Restructure e2e tests for better parallelism (long-term goal) + +--- + +## Recommended Next Step + +Replace the label-based namespace discovery with a **direct, unconditional namespace delete** in +`cleanup-e2e-installs.done`. The current target already knows `$(NAMESPACE)` β€” use it directly: + +```makefile +# Instead of: find namespaces by e2e=true label, then delete +# Do: unconditionally delete the known namespace +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done: Makefile $(CS)/ready + @set -euo pipefail; \ + ctx="$(CTX)"; ns="$(NAMESPACE)"; \ + if kubectl --context "$$ctx" get ns "$$ns" >/dev/null 2>&1; then \ + echo "🧹 Deleting namespace '$$ns' in '$$ctx'"; \ + kubectl --context "$$ctx" delete ns "$$ns" --ignore-not-found=true; \ + kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ + rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ + else \ + echo "ℹ️ Namespace '$$ns' does not exist in '$$ctx'"; \ + fi; \ + ... +``` + +**Trade-off**: loses the ability to detect "orphaned" namespaces from a previous run with a +*different* `NAMESPACE` value. But since `NAMESPACE` defaults to `gitops-reverser` and that +rarely changes, this is an acceptable trade-off for reliability. + +The `e2e=true` label can be kept for informational / debugging purposes without being load-bearing +for the cleanup logic. diff --git a/docs/e2e/notes-whats-going-wrnog.txt b/docs/e2e/notes-whats-going-wrnog.txt new file mode 100644 index 000000000..ff94ff888 --- /dev/null +++ b/docs/e2e/notes-whats-going-wrnog.txt @@ -0,0 +1,30 @@ +Ok I tried to find the solution but failed. Somehow the label is dissapearing and we don't know anymore what to delete. + +Perhaps I should go a bit smarter about this: or more pragmatic. I could also have the same namespace, always, for now. And just make sure that I have a task that always first deltes the old namespace that would also prevent me from sitting in this rabbit hole. + +Let's celebrate the good things that come from this. + +Let's move the gitea installation a layer higher: which will also be a timesaver + +Let's move the gitea creation of credentials a level higher, let's be more clear on what to expect. + +LEt's finally get that structure in the end2end tests, so that we can run things in parallel (and also write tests that mimic what could happen during a well visited talk!) + + +--- + +What I could wish: + +1. I want a flexible way to verify that my code is behaving like it should. +2. I like speed, so I don't want to wait on useless stuff +3. My controller is complex: it requires an audit webhook in the kubernets cluster +4. I had the dream to run multiple tests in parallel: because of 3 I can't at the moment (could use something to distribute the events but thats to much for now) + * Yesterday I realised that what I'm trying now is to complex, it's not needed to install in multiple namespaces. I can't keep the installations +5. An important thing is: I keep the running application, so if the test fails I can investigate the state afterwards +6. I have automated image updating and I have nice Makefile targets that will 'reinstall' when needed (e.g. config/ folder changes) +7. I can be flexible in how I install, so I can also switch to helm and see if it behaves the same. + + +What I learned: +* Makefile is file based, especially the dependency tree is very cool but also fragile if you are managing Kubernetes clusters with it. +* Idea: what could GitOps do in this? Details do matter in this. diff --git a/test/e2e/scripts/setup-gitea.sh b/test/e2e/scripts/setup-gitea.sh index 37184f559..8e9233f82 100755 --- a/test/e2e/scripts/setup-gitea.sh +++ b/test/e2e/scripts/setup-gitea.sh @@ -3,7 +3,6 @@ set -euo pipefail # Configuration GITEA_NAMESPACE=${GITEA_NAMESPACE:-gitea-e2e} -GITEA_SERVICE="gitea-http" ADMIN_USER="giteaadmin" ADMIN_PASS="giteapassword123" ORG_NAME="testorg" @@ -31,7 +30,7 @@ fi echo "πŸš€ Setting up Gitea test environment with repository: $REPO_NAME" test_api_connectivity() { - # Port-forward should already be running via Makefile's setup-port-forwards + # Installation and port-forward setup are handled by Makefile stamp targets. echo "πŸ” Testing API connectivity..." for i in {1..30}; do if curl -s -f "$API_URL/version" >/dev/null 2>&1; then From 7882a21821231dbaf0f31f9d4913772b111d8586 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 10:20:40 +0000 Subject: [PATCH 26/30] fix: Always check these things yourself --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index be1940537..70dfe8448 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -86,7 +86,7 @@ RUN chgrp -R godev /go && \ # These run AFTER permissions are set to ensure proper group ownership RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 \ && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \ - && go install github.com/mrtazz/checkmake@latest + && go install github.com/checkmake/checkmake/cmd/checkmake@latest # Initialize golangci-lint cache by running it once on an empty directory # This downloads linter dependencies without needing source code From 73680a32f5b86963b806c98c0e7e4db9d3101c3d Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 11:21:14 +0000 Subject: [PATCH 27/30] fix: Now we can really uninstall, also multiple if that would already be supported --- Makefile | 123 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index 31a99f2e4..9b9448ae4 100644 --- a/Makefile +++ b/Makefile @@ -121,16 +121,6 @@ E2E_NAMESPACE_LABEL_KEY ?= e2e E2E_NAMESPACE_LABEL_VALUE ?= true HELM_CHART_SOURCE ?= charts/gitops-reverser -.PHONY: cleanup-cluster -cleanup-cluster: ## Tear down the E2E cluster used for tests and remove its stamps - @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(CLUSTER_NAME)$$"; then \ - echo "🧹 Deleting k3d cluster '$(CLUSTER_NAME)'"; \ - $(K3D) cluster delete $(CLUSTER_NAME); \ - else \ - echo "ℹ️ k3d cluster '$(CLUSTER_NAME)' does not exist; skipping cleanup"; \ - fi - rm -rf .stamps/cluster/$(CTX) - # Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. .PHONY: prepare-e2e prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare E2E prerequisites for Go tests @@ -138,7 +128,7 @@ prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). -$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done \ +$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ $(CS)/$(NAMESPACE)/sops-secret.applied @@ -346,7 +336,7 @@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_NAME); \ echo "$$img_id" > "$@" -$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done $(CS)/image.loaded +$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml $(CS)/image.loaded @set -euo pipefail; \ ctx="$(CTX)"; ns="$(NAMESPACE)"; img="$(PROJECT_IMAGE)"; c="$(CONTROLLER_CONTAINER)"; sel="$(CONTROLLER_DEPLOY_SELECTOR)"; \ [ -n "$$img" ] || { echo "ERROR: PROJECT_IMAGE must be non-empty" >&2; exit 2; }; \ @@ -386,24 +376,46 @@ ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) endif .PHONY: install install-helm install-plain-manifests-file install-config-dir -install: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done +install: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml -install-helm: $(CS)/$(NAMESPACE)/helm/install.done +install-helm: $(CS)/$(NAMESPACE)/helm/install.yaml -install-plain-manifests-file: $(CS)/$(NAMESPACE)/plain-manifests-file/install.done +install-plain-manifests-file: $(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml -install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.done +install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.yaml -CONFIG_DIR_INPUTS := $(shell find config -type f) + +# Shared cleanup logic β€” inlined at the start of every install recipe so it always +# runs whenever the recipe runs, regardless of stamp freshness. +define DO_CLEANUP_INSTALLS + @set -euo pipefail; \ + ctx="$(CTX)"; \ + stamp_cluster_dir=".stamps/cluster/$$ctx"; \ + echo "🧹 Deleting resources via installed manifests in '$$ctx'"; \ + while IFS= read -r -d '' manifest; do \ + echo " kubectl delete -f $$manifest"; \ + $(KUBECTL) --context "$$ctx" delete -f "$$manifest" --ignore-not-found=true || true; \ + done < <(find "$$stamp_cluster_dir" -name 'install.yaml' -print0 2>/dev/null); \ + $(KUBECTL) --context "$$ctx" delete crd \ + gitproviders.configbutler.ai \ + gittargets.configbutler.ai \ + watchrules.configbutler.ai \ + clusterwatchrules.configbutler.ai \ + --ignore-not-found=true || true; \ + echo "πŸ—‚ Removing namespace stamp dirs"; \ + find "$$stamp_cluster_dir" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +; \ +endef ##@ Deployments of all manifests needed to run -$(CS)/$(NAMESPACE)/helm/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/helm/cleanup-e2e-installs.done $(HELM_SYNC_OUTPUTS) +$(CS)/$(NAMESPACE)/helm/install.yaml: $(CS)/services.ready $(HELM_SYNC_OUTPUTS) + $(DO_CLEANUP_INSTALLS) @set -euo pipefail; \ + mkdir -p "$(@D)"; \ helm_args=( \ upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ --kube-context $(CTX) \ --namespace $(NAMESPACE) \ - --create-namespace \ + --set createNamespace=true \ ); \ if kubectl --context $(CTX) get crd \ gitproviders.configbutler.ai \ @@ -413,20 +425,28 @@ $(CS)/$(NAMESPACE)/helm/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/he helm_args+=(--skip-crds); \ fi; \ helm "$${helm_args[@]}"; \ - touch $@ + $(HELM) --kube-context $(CTX) get manifest $(INSTALL_NAME) \ + --namespace $(NAMESPACE) > "$@" -$(CS)/$(NAMESPACE)/config-dir/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/config-dir/cleanup-e2e-installs.done $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) +CONFIG_DIR_INPUTS := $(shell find config -type f) +$(CS)/$(NAMESPACE)/config-dir/install.yaml: $(CS)/services.ready $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) + $(DO_CLEANUP_INSTALLS) @set -euo pipefail; \ + mkdir -p "$(@D)"; \ tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ cp -R config "$$tmpdir/config"; \ (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)"); \ (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set image gitops-reverser="$(PROJECT_IMAGE)"); \ - (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) | $(KUBECTL) --context "$(CTX)" apply -f - - touch $@ + (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) \ + | tee "$$tmpdir/install.yaml" \ + | $(KUBECTL) --context "$(CTX)" apply -f -; \ + mv "$$tmpdir/install.yaml" "$@" -$(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/services.ready $(CS)/$(NAMESPACE)/plain-manifests-file/cleanup-e2e-installs.done dist/install.yaml +$(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml: $(CS)/services.ready dist/install.yaml + $(DO_CLEANUP_INSTALLS) @set -euo pipefail; \ + mkdir -p "$(@D)"; \ ctx="$(CTX)"; \ ns="$(NAMESPACE)"; \ tmpdir="$$(mktemp -d)"; \ @@ -436,8 +456,9 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.done: $(CS)/services.ready $(CS) cd "$$tmpdir/dist" && \ kustomize edit set namespace "$$ns" >/dev/null && \ kustomize build . \ - ) | kubectl --context "$$ctx" apply -f -; \ - touch $@ + ) | tee "$$tmpdir/install.yaml" \ + | kubectl --context "$$ctx" apply -f -; \ + mv "$$tmpdir/install.yaml" "$@" .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) @@ -453,36 +474,32 @@ test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework -.PHONY: cleanup-port-forwards -cleanup-port-forwards: ## Stop all port-forwards - @echo "πŸ›‘ Stopping port-forwards..." - @-pkill -f "kubectl.*port-forward.*13000" 2>/dev/null || true - @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true - @echo "βœ… Port-forwards stopped" -$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done: Makefile $(CS)/ready - @set -euo pipefail; \ - ctx="$(CTX)"; \ - ns="$(NAMESPACE)"; \ - if kubectl --context "$$ctx" get ns "$$ns" >/dev/null 2>&1; then \ - echo "🧹 Deleting namespace '$$ns' in '$$ctx'"; \ - kubectl --context "$$ctx" delete ns "$$ns" --ignore-not-found=true; \ - kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ - rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ - rm -f ".stamps/cluster/$$ctx/image.loaded" 2>/dev/null || true; \ - else \ - echo "ℹ️ Namespace '$$ns' does not exist in '$$ctx'"; \ - fi; \ - # Clear cluster-scoped resources that survive namespace deletion (safe always) - kubectl --context "$$ctx" delete svc --all-namespaces -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true || true; \ - kubectl --context "$$ctx" delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true || true; \ - kubectl --context "$$ctx" delete clusterrole,clusterrolebinding -l app.kubernetes.io/name=gitops-reverser --ignore-not-found=true || true; \ - kubectl --context "$$ctx" delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai --ignore-not-found=true || true; \ - mkdir -p "$(@D)"; \ - touch "$@" -##@ Cleanup +##@ Cleanup .PHONY: clean clean: ## Remove build artifacts (binaries, coverage, generated dist files, build stamps) rm -rf bin/ cover.out dist/ .stamps/ + +.PHONY: clean-installs +clean-installs: + $(DO_CLEANUP_INSTALLS) + +.PHONY: clean-port-forwards +clean-port-forwards: ## Stop all port-forwards + @echo "πŸ›‘ Stopping port-forwards..." + @-pkill -f "kubectl.*port-forward.*13000" 2>/dev/null || true + @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true + @echo "βœ… Port-forwards stopped" + +.PHONY: clean-cluster +clean-cluster: ## Tear down the E2E cluster used for tests and remove its stamps + @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(CLUSTER_NAME)$$"; then \ + echo "🧹 Deleting k3d cluster '$(CLUSTER_NAME)'"; \ + $(K3D) cluster delete $(CLUSTER_NAME); \ + else \ + echo "ℹ️ k3d cluster '$(CLUSTER_NAME)' does not exist; skipping cleanup"; \ + fi + rm -rf .stamps/cluster/$(CTX) + From a3560c1c327e583a338ec78edbd46c625972f76a Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 11:52:33 +0000 Subject: [PATCH 28/30] fix: Mistake in Makefile syntax --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9b9448ae4..40f9504ab 100644 --- a/Makefile +++ b/Makefile @@ -403,7 +403,7 @@ define DO_CLEANUP_INSTALLS clusterwatchrules.configbutler.ai \ --ignore-not-found=true || true; \ echo "πŸ—‚ Removing namespace stamp dirs"; \ - find "$$stamp_cluster_dir" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +; \ + find "$$stamp_cluster_dir" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +; endef ##@ Deployments of all manifests needed to run From ebc18aa79f7b8f89929f18463fa9329cfb95d8dc Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 13:00:51 +0000 Subject: [PATCH 29/30] fix: Some tweaks to cleaning existing installs, and moving to a seperate file --- Makefile | 22 +-- docs/e2e/gitea-setup-target-split-plan.md | 186 ++++++++++++++++++++++ hack/cleanup-installs.sh | 62 ++++++++ 3 files changed, 250 insertions(+), 20 deletions(-) create mode 100644 docs/e2e/gitea-setup-target-split-plan.md create mode 100755 hack/cleanup-installs.sh diff --git a/Makefile b/Makefile index 40f9504ab..04f1603e1 100644 --- a/Makefile +++ b/Makefile @@ -117,8 +117,6 @@ GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum INSTALL_MODE ?= config-dir INSTALL_NAME ?= gitops-reverser NAMESPACE ?= gitops-reverser -E2E_NAMESPACE_LABEL_KEY ?= e2e -E2E_NAMESPACE_LABEL_VALUE ?= true HELM_CHART_SOURCE ?= charts/gitops-reverser # Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. @@ -384,26 +382,10 @@ install-plain-manifests-file: $(CS)/$(NAMESPACE)/plain-manifests-file/install.ya install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.yaml - # Shared cleanup logic β€” inlined at the start of every install recipe so it always # runs whenever the recipe runs, regardless of stamp freshness. define DO_CLEANUP_INSTALLS - @set -euo pipefail; \ - ctx="$(CTX)"; \ - stamp_cluster_dir=".stamps/cluster/$$ctx"; \ - echo "🧹 Deleting resources via installed manifests in '$$ctx'"; \ - while IFS= read -r -d '' manifest; do \ - echo " kubectl delete -f $$manifest"; \ - $(KUBECTL) --context "$$ctx" delete -f "$$manifest" --ignore-not-found=true || true; \ - done < <(find "$$stamp_cluster_dir" -name 'install.yaml' -print0 2>/dev/null); \ - $(KUBECTL) --context "$$ctx" delete crd \ - gitproviders.configbutler.ai \ - gittargets.configbutler.ai \ - watchrules.configbutler.ai \ - clusterwatchrules.configbutler.ai \ - --ignore-not-found=true || true; \ - echo "πŸ—‚ Removing namespace stamp dirs"; \ - find "$$stamp_cluster_dir" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +; + @CTX="$(CTX)" KUBECTL="$(KUBECTL)" bash hack/cleanup-installs.sh endef ##@ Deployments of all manifests needed to run @@ -415,7 +397,7 @@ $(CS)/$(NAMESPACE)/helm/install.yaml: $(CS)/services.ready $(HELM_SYNC_OUTPUTS) upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ --kube-context $(CTX) \ --namespace $(NAMESPACE) \ - --set createNamespace=true \ + --create-namespace \ ); \ if kubectl --context $(CTX) get crd \ gitproviders.configbutler.ai \ diff --git a/docs/e2e/gitea-setup-target-split-plan.md b/docs/e2e/gitea-setup-target-split-plan.md new file mode 100644 index 000000000..fcc0949fb --- /dev/null +++ b/docs/e2e/gitea-setup-target-split-plan.md @@ -0,0 +1,186 @@ +# E2E Gitea Setup Split Plan (Make Targets + Concrete Artifacts) + +## Overview + +This plan splits `test/e2e/scripts/setup-gitea.sh` into clear setup levels and maps each level to explicit Make +targets that produce concrete artifact files. + +Goal: + +1. Keep shared/bootstrap work one-time per cluster context (`CTX`). +2. Keep one active repo per test run/namespace. +3. Store most run artifacts in `$(CS)/$(NAMESPACE)/repo/`. +4. Keep checkout data in `$(CS)/$(NAMESPACE)/repo//` for easy developer inspection. + +## Current Behavior Summary + +Today `setup-gitea.sh` does all of this in one execution: + +1. Check API readiness through localhost port-forward. +2. Ensure org (`testorg`). +3. Create/reuse token. +4. Generate SSH keypair and configure Gitea user key. +5. Create namespace secrets (`git-creds`, `git-creds-ssh`, `git-creds-invalid`). +6. Create a repo. +7. Clone checkout and configure git. + +Call sites: + +- `test/e2e/e2e_test.go` in `BeforeAll` +- `test/e2e/quickstart_framework_e2e_test.go` in `setupGiteaRepository` + +Suite-level prep (`test/e2e/e2e_suite_test.go`) currently runs `make prepare-e2e`, but repo+secret bootstrap still happens +inside per-spec setup scripts. + +## Proposed Split + +### Level 1: Cluster-scoped bootstrap (once per CTX) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/gitea/bootstrap/` + +Concrete artifact targets: + +1. `$(CS)/gitea/bootstrap/api.ready` +2. `$(CS)/gitea/bootstrap/org-testorg.ready` +3. `$(CS)/gitea/bootstrap/ready` (aggregate stamp only after concrete files exist) + +Why this helps: + +- Shared infra readiness is explicit and isolated from run-scoped data. +- Per-run token/SSH/secret generation can happen cleanly after infra is prepared. + +### Level 2: Run-scoped active repo + credentials (per NAMESPACE) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/` + +Concrete artifact targets: + +1. `$(CS)/$(NAMESPACE)/repo/active-repo.txt` (contains repo name) +2. `$(CS)/$(NAMESPACE)/repo/token.txt` +3. `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa` +4. `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa.pub` +5. `$(CS)/$(NAMESPACE)/repo/ssh/known_hosts` +6. `$(CS)/$(NAMESPACE)/repo/secrets.yaml` (single multi-doc manifest for `git-creds`, `git-creds-ssh`, `git-creds-invalid`) +7. `$(CS)/$(NAMESPACE)/repo/secrets.applied` +8. `$(CS)/$(NAMESPACE)/repo/repo.ready` + +Source inputs: + +- `$(CS)/gitea/bootstrap/org-testorg.ready` +- `$(CS)/gitea/bootstrap/api.ready` + +### Level 3: Checkout (per active repo name) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/` + +Concrete artifact targets: + +1. `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` +2. `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` +3. `$(CS)/$(NAMESPACE)/repo/checkout.ready` (aggregate; uses active repo from `active-repo.txt`) + +Default checkout location for active repo: + +- `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout` +- `CHECKOUT_DIR` may override, but should default to the stamp path above. + +## Script Refactor Shape + +Recommended split while preserving behavior: + +1. `test/e2e/scripts/gitea-bootstrap.sh` +2. `test/e2e/scripts/gitea-run-setup.sh` (active repo + token + ssh + secrets + checkout) +3. Keep `test/e2e/scripts/setup-gitea.sh` as compatibility wrapper (calls bootstrap + run setup). + +This allows incremental migration without breaking existing tests. + +## Makefile Target Contract + +Suggested phony wrappers: + +1. `e2e-gitea-bootstrap` -> `$(CS)/gitea/bootstrap/ready` +2. `e2e-gitea-run-setup` -> `$(CS)/$(NAMESPACE)/repo/repo.ready` and `$(CS)/$(NAMESPACE)/repo/checkout.ready` +3. `e2e-gitea-run-setup` requires `REPO_NAME` and optional `CHECKOUT_DIR` + +Suggested integration: + +1. Keep `prepare-e2e` focused on cluster + controller prep only. +2. Add a second Make call from `ensureE2EPrepared()` in `test/e2e/e2e_suite_test.go`: + - first call: `make ... prepare-e2e` + - second call: `make ... REPO_NAME= e2e-gitea-run-setup` +3. The second call must run after `prepare-e2e`. + +## Producer/Consumer File Map + +| Producer target | Output file | Consumed by | +| --- | --- | --- | +| `$(CS)/gitea/bootstrap/api.ready` | `.stamps/cluster/$(CTX)/gitea/bootstrap/api.ready` | run-setup target | +| `$(CS)/gitea/bootstrap/org-testorg.ready` | `.stamps/cluster/$(CTX)/gitea/bootstrap/org-testorg.ready` | run-setup target | +| `$(CS)/$(NAMESPACE)/repo/active-repo.txt` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/active-repo.txt` | checkout target + suite env export | +| `$(CS)/$(NAMESPACE)/repo/token.txt` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/token.txt` | `secrets.yaml`, checkout clone auth | +| `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/id_rsa` | `secrets.yaml` | +| `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa.pub` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/id_rsa.pub` | Gitea SSH key registration | +| `$(CS)/$(NAMESPACE)/repo/ssh/known_hosts` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/known_hosts` | `secrets.yaml` | +| `$(CS)/$(NAMESPACE)/repo/secrets.yaml` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/secrets.yaml` | `secrets.applied` | +| `$(CS)/$(NAMESPACE)/repo/secrets.applied` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/secrets.applied` | `repo.ready` | +| `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` | e2e/quickstart tests needing checkout location | +| `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` | `checkout.ready` | +| `$(CS)/$(NAMESPACE)/repo/repo.ready` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/repo.ready` | suite execution start gate | +| `$(CS)/$(NAMESPACE)/repo/checkout.ready` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/checkout.ready` | suite execution start gate | + +Recommended environment variables per producer: + +1. Bootstrap level: `CTX`, `GITEA_NAMESPACE`, optional `GITEA_ADMIN_USER`, `GITEA_ADMIN_PASS`. +2. Run setup level: `CTX`, `NAMESPACE`, `REPO_NAME`, optional `CHECKOUT_DIR`, `GITEA_NAMESPACE`. + +## Go Test Wiring Plan + +1. `test/e2e/e2e_suite_test.go` + - keep existing `prepare-e2e` invocation + - add second make invocation after `prepare-e2e`: + - `make ... REPO_NAME= e2e-gitea-run-setup` + - export outputs for test usage: + - `E2E_REPO_NAME` + - `E2E_CHECKOUT_DIR` (read from `checkout.path`) + - `E2E_GIT_SECRET_HTTP`, `E2E_GIT_SECRET_SSH`, `E2E_GIT_SECRET_INVALID` (run-scoped names) +2. `test/e2e/e2e_test.go` + - remove direct `setup-gitea.sh` call + - consume suite-exported repo/secret env vars +3. `test/e2e/quickstart_framework_e2e_test.go` + - remove direct `setup-gitea.sh` call + - consume suite-exported repo/secret env vars + - use same active repo model as suite setup + +## Feasibility Notes + +1. High feasibility: bootstrap remains stable in `prepare-e2e`. +2. High feasibility: single active repo model keeps state simpler under `$(CS)/$(NAMESPACE)/repo/`. +3. Main implementation risk: namespace mismatch (`NAMESPACE` vs `SUT_NAMESPACE` / `QUICKSTART_NAMESPACE`). + - Mitigation: always pass explicit `NAMESPACE` to run-setup Make call in suite. + +## Implementation Phases + +1. Phase 1: Keep `prepare-e2e` behavior; add `e2e-gitea-run-setup` writing run artifacts to `$(CS)/$(NAMESPACE)/repo/`. +2. Phase 2: Add second Make invocation in `e2e_suite_test.go` immediately after `prepare-e2e`. +3. Phase 3: Remove direct `setup-gitea.sh` invocations from `e2e_test.go` and quickstart test file. +4. Phase 4: Keep wrapper for compatibility, then trim wrapper to call new split scripts only. + +## Validation Gate (after implementation) + +Required sequence: + +1. `make fmt` +2. `make generate` +3. `make vet` +4. `make lint` +5. `make test` +6. `docker info` (precheck for e2e) +7. `make test-e2e` +8. `make test-e2e-quickstart-manifest` +9. `make test-e2e-quickstart-helm` diff --git a/hack/cleanup-installs.sh b/hack/cleanup-installs.sh new file mode 100755 index 000000000..bf15cbe49 --- /dev/null +++ b/hack/cleanup-installs.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +ctx="${CTX:-${1:-}}" +kubectl_bin="${KUBECTL:-${2:-kubectl}}" +base=".stamps/cluster/${ctx}" + +if [[ -z "${ctx}" ]]; then + echo "ERROR: CTX is required (set CTX or pass as arg 1)" >&2 + exit 2 +fi + +echo "🧹 Cleanup installs in context '${ctx}' (base: ${base})" + +cleanup_ns() { + local ns="$1" + + echo "🧹 Cleaning namespace '${ns}'" + + if [[ -d "${base}/${ns}" ]]; then + while IFS= read -r -d '' f; do + echo " kubectl delete -f ${f}" + "${kubectl_bin}" --context "${ctx}" delete -f "${f}" --ignore-not-found=true || true + done < <(find "${base}/${ns}" -type f -name "install.yaml" -print0 2>/dev/null || true) + fi + + if "${kubectl_bin}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then + echo " kubectl delete namespace ${ns}" + "${kubectl_bin}" --context "${ctx}" delete namespace "${ns}" --ignore-not-found=true || true + fi + + if [[ -d "${base}/${ns}" ]]; then + echo " rm -rf ${base}/${ns}" + rm -rf "${base:?}/${ns}" || true + fi +} + +# Match: .stamps/cluster////install.yaml +declare -A seen_ns=() +base_regex="${base//./\\.}" +while IFS= read -r -d '' m; do + if [[ "${m}" =~ ^${base_regex}/([^/]+)/([^/]+)/install\.yaml$ ]]; then + ns="${BASH_REMATCH[1]}" + seen_ns["${ns}"]=1 + fi +done < <(find "${base}" -type f -name "install.yaml" -print0 2>/dev/null || true) + +if [[ "${#seen_ns[@]}" -eq 0 ]]; then + echo "🧹 No installs found under ${base}" + exit 0 +fi + +while IFS= read -r ns; do + [[ -n "${ns}" ]] && cleanup_ns "${ns}" +done < <(printf '%s\n' "${!seen_ns[@]}" | sort) + +"${kubectl_bin}" --context "${ctx}" delete crd \ + gitproviders.configbutler.ai \ + gittargets.configbutler.ai \ + watchrules.configbutler.ai \ + clusterwatchrules.configbutler.ai \ + --ignore-not-found=true || true From 0750e8848443caface93c90cf1ca10b01d92e9a7 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Tue, 3 Mar 2026 13:18:06 +0000 Subject: [PATCH 30/30] chore: More errors while switching back and forth --- Makefile | 19 ++++++++++++------- dist/kustomization.yaml | 8 -------- 2 files changed, 12 insertions(+), 15 deletions(-) delete mode 100644 dist/kustomization.yaml diff --git a/Makefile b/Makefile index 04f1603e1..f46721232 100644 --- a/Makefile +++ b/Makefile @@ -433,14 +433,19 @@ $(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml: $(CS)/services.ready dist/ ns="$(NAMESPACE)"; \ tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ - cp -R dist "$$tmpdir/dist"; \ + cp dist/install.yaml "$$tmpdir/install.yaml"; \ + printf '%s\n' \ + 'apiVersion: kustomize.config.k8s.io/v1beta1' \ + 'kind: Kustomization' \ + 'resources:' \ + '- install.yaml' > "$$tmpdir/kustomization.yaml"; \ ( \ - cd "$$tmpdir/dist" && \ - kustomize edit set namespace "$$ns" >/dev/null && \ - kustomize build . \ - ) | tee "$$tmpdir/install.yaml" \ - | kubectl --context "$$ctx" apply -f -; \ - mv "$$tmpdir/install.yaml" "$@" + cd "$$tmpdir" && \ + $(KUSTOMIZE) edit set namespace "$$ns" >/dev/null && \ + $(KUSTOMIZE) build . \ + ) | tee "$$tmpdir/rendered-install.yaml" \ + | $(KUBECTL) --context "$$ctx" apply -f -; \ + mv "$$tmpdir/rendered-install.yaml" "$@" .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) diff --git a/dist/kustomization.yaml b/dist/kustomization.yaml deleted file mode 100644 index 9e7ab5845..000000000 --- a/dist/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Helper to facilitate quick deployment to another ns (currently used for e2e) -# kustomize edit set namespace blaat -# kustomize build . | kubectl apply -f - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- install.yaml -