diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 000000000..d0e26f573 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,41 @@ +{ + "permissions": { + "allow": [ + "Bash(make fmt)", + "Bash(make vet)", + "Bash(make lint)", + "Bash(make test)", + "Bash(make test-e2e)", + "Bash(make --dry-run test-e2e-quickstart-manifest)", + "Bash(make test-e2e-quickstart-manifest)", + "Bash(make test-e2e-quickstart-helm)", + "Bash(kubectl --context kind-gitops-reverser-test-e2e-quickstart-helm get clusterrole,clusterrolebinding,crd)", + "Bash(make --dry-run test-e2e-quickstart-helm)", + "Bash(make --dry-run test-e2e)", + "Bash(grep -n \"setupSOPSAgeSecret\\\\|generateAgeKey\\\\|e2eAgeKeyPath\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(grep -n \"readSOPSAgeKeyFromFile\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(grep -r \"make\\\\|Make\" /workspaces/gitops-reverser2/test/e2e/*.go)", + "Bash(make .stamps/image/controller.id)", + "Bash(docker version)", + "Bash(docker buildx version)", + "Bash(go build ./test/e2e/tools/gen-age-key/...)", + "Bash(make cleanup-cluster)", + "Bash(make -n prepare-e2e INSTALL_MODE=helm)", + "Bash(checkmake Makefile)", + "Bash(docker info)", + "Bash(make fmt vet)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e get clusterrole,clusterrolebinding -l app.kubernetes.io/part-of=gitops-reverser --no-headers)", + "Read(//workspaces/gitops-reverser2/**)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e get crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai -o jsonpath='{range .items[*]}{.metadata.name}: managedFields manager={.metadata.managedFields[0].manager}{\"\"\\\\n\"\"}{end}')", + "Bash(helm upgrade --help)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete ns gitops-reverser --ignore-not-found=true)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e wait --for=delete ns/gitops-reverser --timeout=120s)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration --ignore-not-found=true)", + "Bash(kubectl --context k3d-gitops-reverser-test-e2e delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai --ignore-not-found=true)", + "Bash(grep -l \"managedFields\" charts/gitops-reverser/crds/*.yaml)", + "Bash(grep -l \"managedFields\" config/crd/bases/*.yaml)", + "Bash(helm install gitops-reverser charts/gitops-reverser --kube-context k3d-gitops-reverser-test-e2e --namespace gitops-reverser --create-namespace --debug)", + "Bash(helm version)" + ] + } +} diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b3606c288..70dfe8448 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update \ vim \ less \ jq \ + yq \ sudo \ acl \ rsync \ @@ -36,7 +37,7 @@ ENV KUBECTL_VERSION=v1.35.0 \ KUBEBUILDER_VERSION=4.11.1 \ GOLANGCI_LINT_VERSION=v2.10.1 \ HELM_VERSION=v4.0.0 \ - KIND_VERSION=v0.31.0 + K3D_VERSION=v5.8.3 # Install kubectl RUN curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ @@ -62,9 +63,9 @@ RUN curl -fsSL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz \ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ | sh -s -- -b /usr/local/bin ${GOLANGCI_LINT_VERSION} -# Install Kind (needed in CI for e2e cluster creation, and in dev for local clusters) -RUN curl -Lo /usr/local/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ - && chmod +x /usr/local/bin/kind +# Install k3d (faster and kind alternative, also more stable with our docker-outside-docker setup) +RUN curl -fsSL -o /usr/local/bin/k3d "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64" \ + && chmod +x /usr/local/bin/k3d # Set working directory WORKDIR /workspaces @@ -84,7 +85,8 @@ RUN chgrp -R godev /go && \ # Install Go tools used by the project (using @version doesn't need go.mod) # These run AFTER permissions are set to ensure proper group ownership RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 \ - && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \ + && go install github.com/checkmake/checkmake/cmd/checkmake@latest # Initialize golangci-lint cache by running it once on an empty directory # This downloads linter dependencies without needing source code diff --git a/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md b/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md index 84977e37e..74d2fe2df 100644 --- a/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md +++ b/.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md @@ -10,19 +10,19 @@ Get "https://172.19.0.2:6443/livez?timeout=10s": dial tcp 172.19.0.2:6443: conne ## Root cause (current setup) -`test/e2e/kind/start-cluster.sh` generates `test/e2e/kind/cluster.ignore.yaml` from `HOST_PROJECT_PATH`. +`test/e2e/cluster/start-cluster.sh` generates `test/e2e/cluster/cluster.ignore.yaml` from `HOST_PROJECT_PATH`. In the current devcontainer config, `HOST_PROJECT_PATH` is set from `${localWorkspaceFolder}`. That produced: ``` -hostPath: /home/simon/git/gitops-reverser2/test/e2e/kind/audit +hostPath: /home/simon/git/gitops-reverser2/test/e2e/cluster/audit ``` But that mounted directory exists and is empty in the container, while the real audit files are under: ``` -/workspaces/gitops-reverser2/test/e2e/kind/audit +/workspaces/gitops-reverser2/test/e2e/cluster/audit ``` Because the mount source is wrong/empty, kube-apiserver cannot read: @@ -56,7 +56,7 @@ Your current config mixes modes and path assumptions, so Kind mount path resolut Before running `make setup-cluster`, verify generated config points to a path with files: ```bash -cat test/e2e/kind/cluster.ignore.yaml +cat test/e2e/cluster/cluster.ignore.yaml ls -la ``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7916f427..fd279d4cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,8 +77,9 @@ jobs: helm version golangci-lint version controller-gen --version - kind version + k3d version docker --version + checkmake --version echo 'โœ… All CI container tools verified' " @@ -123,8 +124,9 @@ jobs: kustomize version helm version golangci-lint version - kind version + k3d version docker --version + checkmake --version echo 'โœ… All dev container tools verified' " @@ -161,7 +163,7 @@ jobs: - name: Generate install.yaml from Helm chart run: | - make build-installer + make dist/install.yaml - name: Package Helm chart run: | @@ -193,6 +195,9 @@ jobs: - name: Configure Git safe directory (for now needed as workarround https://github.com/actions/checkout/issues/2031) run: git config --global --add safe.directory /__w/gitops-reverser/gitops-reverser + - name: checkmake Makefile + run: checkmake Makefile + - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: diff --git a/.gitignore b/.gitignore index 0d9190218..4f01e8feb 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,9 @@ __debug* charts/gitops-reverser/crds/*.yaml charts/gitops-reverser/config/*.yaml -# Keep the install.yaml for distribution -dist/install.yaml +# Generated single-file installer artifact (not tracked). +/dist/install.yaml *.o.yaml *.ignore.* -.stamps/* \ No newline at end of file +.stamps/* diff --git a/.stamps/cluster/pf.running b/.stamps/cluster/pf.running deleted file mode 100644 index e69de29bb..000000000 diff --git a/Makefile b/Makefile index 1824919d8..f46721232 100644 --- a/Makefile +++ b/Makefile @@ -41,20 +41,43 @@ help: ## Display this help. ##@ Development +CRD_BASE_OUTPUTS := \ + config/crd/bases/configbutler.ai_clusterwatchrules.yaml \ + config/crd/bases/configbutler.ai_gitproviders.yaml \ + config/crd/bases/configbutler.ai_gittargets.yaml \ + config/crd/bases/configbutler.ai_watchrules.yaml +MANIFEST_OUTPUTS := $(CRD_BASE_OUTPUTS) \ + config/rbac/role.yaml \ + config/webhook/manifests.yaml +MANIFEST_INPUTS := $(shell find api internal cmd -type f -name '*.go' \ + ! -name '*_test.go' ! -name 'zz_generated.deepcopy.go') +CONTROLLER_GEN_ARGS := rbac:roleName=gitops-reverser crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: manifests -manifests: ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - @rm -f config/crd/bases/*.yaml - $(CONTROLLER_GEN) rbac:roleName=gitops-reverser crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - +manifests: $(MANIFEST_OUTPUTS) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +$(MANIFEST_OUTPUTS) &: $(MANIFEST_INPUTS) + @rm -f config/crd/bases/*.yaml \ + config/rbac/role.yaml \ + config/webhook/manifests.yaml + $(CONTROLLER_GEN) $(CONTROLLER_GEN_ARGS) + +HELM_CRD_OUTPUTS := $(patsubst config/crd/bases/%,charts/gitops-reverser/crds/%,$(CRD_BASE_OUTPUTS)) +HELM_SYNC_OUTPUTS := $(HELM_CRD_OUTPUTS) \ + charts/gitops-reverser/config/role.yaml .PHONY: helm-sync -helm-sync: ## Sync CRDs and roles from config/crd/bases to Helm chart crds directory (for packaging) - @rm -f charts/gitops-reverser/crds/*.yaml +helm-sync: $(HELM_SYNC_OUTPUTS) ## Sync CRDs and roles from config/crd/bases to Helm chart crds directory (for packaging) +$(HELM_SYNC_OUTPUTS) &: $(MANIFEST_OUTPUTS) + @mkdir -p charts/gitops-reverser/crds charts/gitops-reverser/config + @rm -f charts/gitops-reverser/crds/*.yaml charts/gitops-reverser/config/*.yaml @cp config/crd/bases/*.yaml charts/gitops-reverser/crds/ - @rm -f charts/gitops-reverser/config/*.yaml - @cp config/rbac/cluster-role.yaml charts/gitops-reverser/config/ + @cp config/rbac/role.yaml charts/gitops-reverser/config/role.yaml +GENERATE_INPUTS := $(shell find api -type f -name '*.go' ! -name '*_test.go' \ + ! -name 'zz_generated.deepcopy.go') hack/boilerplate.go.txt +GENERATE_OUTPUT := api/v1alpha1/zz_generated.deepcopy.go .PHONY: generate -generate: ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. +generate: $(GENERATE_OUTPUT) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + +$(GENERATE_OUTPUT): $(GENERATE_INPUTS) $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt @@ -69,53 +92,50 @@ vet: ## Run go vet against code. test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(shell pwd)/bin -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Kind cluster names per E2E test type (kept separate to avoid cross-test contamination). -KIND_CLUSTER_E2E ?= gitops-reverser-test-e2e -KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-test-e2e-quickstart-helm -KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-test-e2e-quickstart-manifest -# KIND_CLUSTER is used by cleanup-cluster; defaults to the main e2e cluster. -KIND_CLUSTER ?= $(KIND_CLUSTER_E2E) E2E_LOCAL_IMAGE ?= gitops-reverser:e2e-local -# In CI, PROJECT_IMAGE is the pre-built image from the docker-build job; locally we build E2E_LOCAL_IMAGE. -E2E_IMAGE := $(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) +# Normalize PROJECT_IMAGE once, then consume only normalized variables below. +# - If PROJECT_IMAGE is explicitly provided (CI / caller), use it and skip local source-triggered build dependency. +# - Otherwise default to E2E_LOCAL_IMAGE and treat it as local-build managed. +PROJECT_IMAGE_INPUT := $(strip $(PROJECT_IMAGE)) +PROJECT_IMAGE := $(if $(PROJECT_IMAGE_INPUT),$(PROJECT_IMAGE_INPUT),$(E2E_LOCAL_IMAGE)) +PROJECT_IMAGE_PROVIDED := $(if $(PROJECT_IMAGE_INPUT),true,) +export PROJECT_IMAGE +export PROJECT_IMAGE_PROVIDED E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. -# Defaults to the main e2e cluster; override with CTX=kind- to reuse stamp targets for other clusters. -CTX ?= kind-$(KIND_CLUSTER_E2E) -# Derive the Kind cluster name by stripping the "kind-" prefix from CTX. -CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(CTX)) +# Defaults to the main e2e cluster; override with CTX= to reuse stamp targets for other clusters. +CTX ?= k3d-gitops-reverser-test-e2e +# Derive the cluster name by stripping a known context prefix from CTX. +CLUSTER_NAME ?= $(patsubst kind-%,%,$(patsubst k3d-%,%,$(CTX))) CS := .stamps/cluster/$(CTX) IS := .stamps/image GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum -KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) -KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) - -.PHONY: cleanup-cluster -cleanup-cluster: ## Tear down the Kind cluster used for e2e tests and remove its stamps - @if $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER)$$"; then \ - echo "๐Ÿงน Deleting Kind cluster '$(KIND_CLUSTER)'"; \ - $(KIND) delete cluster --name $(KIND_CLUSTER); \ - else \ - echo "โ„น๏ธ Kind cluster '$(KIND_CLUSTER)' does not exist; skipping cleanup"; \ - fi - rm -rf .stamps/cluster/kind-$(KIND_CLUSTER) - -.PHONY: cleanup-e2e-clusters -cleanup-e2e-clusters: ## Tear down all E2E Kind clusters and their stamps - @for cluster in "$(KIND_CLUSTER_E2E)" "$(KIND_CLUSTER_QUICKSTART_HELM)" "$(KIND_CLUSTER_QUICKSTART_MANIFEST)"; do \ - if $(KIND) get clusters 2>/dev/null | grep -q "^$$cluster$$"; then \ - echo "๐Ÿงน Deleting Kind cluster '$$cluster'"; \ - $(KIND) delete cluster --name "$$cluster"; \ - else \ - echo "โ„น๏ธ Kind cluster '$$cluster' does not exist; skipping cleanup"; \ - fi; \ - rm -rf .stamps/cluster/kind-$$cluster; \ - done +# INSTALL_MODE controls installation workflow: helm|plain-manifests-file|config-dir +# These defaults must be defined before any target names that reference $(NAMESPACE). +INSTALL_MODE ?= config-dir +INSTALL_NAME ?= gitops-reverser +NAMESPACE ?= gitops-reverser +HELM_CHART_SOURCE ?= charts/gitops-reverser + +# Called by the Go e2e suite (BeforeSuite) to prepare prerequisites once, including port-forwards + age key. +.PHONY: prepare-e2e +prepare-e2e: $(CS)/$(NAMESPACE)/prepare-e2e.ready portforward-ensure ## Prepare E2E prerequisites for Go tests + +# Called by the full e2e suite. +# For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. +# Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). +$(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml \ + $(CS)/image.loaded \ + $(CS)/$(NAMESPACE)/controller.deployed \ + $(CS)/$(NAMESPACE)/sops-secret.applied + mkdir -p $(@D) + touch $@ +# Keep `make test-e2e` as the classic entrypoint. .PHONY: test-e2e -test-e2e: $(CS)/e2e.passed ## Run end-to-end tests (stamp-based; only reruns what changed) +test-e2e: $(CS)/e2e.passed .PHONY: lint lint: ## Run golangci-lint linter @@ -167,41 +187,23 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - $(CONTAINER_TOOL) buildx rm gitops-reverser-builder rm Dockerfile.cross -.PHONY: build-installer -build-installer: manifests helm-sync ## Generate a consolidated YAML from Helm chart for easy installation. - @echo "๐Ÿ“ฆ Generating install.yaml from Helm chart..." +CHART_INPUTS := charts/gitops-reverser/Chart.yaml \ + charts/gitops-reverser/values.yaml \ + $(shell find charts/gitops-reverser/templates -type f) + +dist/install.yaml: $(HELM_SYNC_OUTPUTS) $(CHART_INPUTS) ## Generate consolidated YAML from Helm chart. @mkdir -p dist - @$(HELM) template gitops-reverser charts/gitops-reverser \ - --namespace gitops-reverser \ + @$(HELM) template $(INSTALL_NAME) charts/gitops-reverser \ + --namespace $(NAMESPACE) \ --set labels.managedBy=kubectl \ --set createNamespace=true \ - --include-crds > dist/install.yaml - @echo "โœ… Generated dist/install.yaml ($(shell wc -l < dist/install.yaml) lines)" - -##@ Deployment - -.PHONY: install -install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=true -f - - -.PHONY: deploy -deploy: manifests ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config && $(KUSTOMIZE) edit set image gitops-reverser=${IMG} - $(KUSTOMIZE) build config | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config | $(KUBECTL) delete --ignore-not-found=true -f - + --include-crds > $@ ##@ Dependencies ## Tool Binaries - all pre-installed in devcontainer KUBECTL ?= kubectl -KIND ?= kind +K3D ?= k3d HELM ?= helm KUSTOMIZE ?= kustomize CONTROLLER_GEN ?= controller-gen @@ -213,7 +215,12 @@ ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk - # Gitea E2E Configuration GITEA_NAMESPACE ?= gitea-e2e -GITEA_CHART_VERSION ?= 12.5.0 # https://gitea.com/gitea/helm-gitea +GITEA_RELEASE_NAME ?= gitea +GITEA_HELM_REPO_NAME ?= gitea-charts +GITEA_HELM_REPO_URL ?= https://dl.gitea.com/charts/ +GITEA_CHART_NAME ?= gitea +GITEA_CHART_VERSION ?= 12.5.0 # https://gitea.com/gitea/helm-gitea +GITEA_WAIT_TIMEOUT ?= 300s .PHONY: setup-envtest setup-envtest: ## Setup envtest binaries for unit tests @@ -224,35 +231,15 @@ setup-envtest: ## Setup envtest binaries for unit tests exit 1; \ } -##@ E2E Stamp Targets (cluster-parameterized; pass CTX=kind- to target a different cluster) +##@ E2E Stamp Targets (cluster-parameterized; pass CTX=k3d- to target a different cluster) -# DEPLOY_INPUTS: all config files except CRDs (tracked separately by crds.applied). -# Only relevant for the main e2e cluster (controller.deployed is not used by quickstart targets). -# When PROJECT_IMAGE is set (CI), skip the local image build stamp. -ifeq ($(PROJECT_IMAGE),) -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(CS)/sut.namespace.ready \ - $(IS)/controller.id \ - $(shell find config -type f -not -path 'config/crd/*') -else -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(CS)/sut.namespace.ready \ - $(shell find config -type f -not -path 'config/crd/*') -endif - -$(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml +$(CS)/ready: test/e2e/cluster/start-cluster.sh mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster.sh + CLUSTER_NAME=$(CLUSTER_NAME) bash test/e2e/cluster/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ -$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml - mkdir -p $(CS) - kubectl --context $(CTX) apply -f config/namespace.yaml - kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted - touch $@ - -CERT_MANAGER_WAIT_TIMEOUT ?= 600s +CERT_MANAGER_WAIT_TIMEOUT ?= 300s CERT_MANAGER_VERSION ?= v1.19.4 CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml $(CS)/cert-manager.installed: $(CS)/ready @@ -261,38 +248,57 @@ $(CS)/cert-manager.installed: $(CS)/ready kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-webhook --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - touch $@ + echo $(CERT_MANAGER_VERSION) > $@ $(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml mkdir -p $(CS) - $(HELM) repo add gitea-charts https://dl.gitea.com/charts/ 2>/dev/null || true - $(HELM) repo update gitea-charts + $(HELM) repo add $(GITEA_HELM_REPO_NAME) $(GITEA_HELM_REPO_URL) 2>/dev/null || true + $(HELM) repo update $(GITEA_HELM_REPO_NAME) kubectl --context $(CTX) create namespace $(GITEA_NAMESPACE) --dry-run=client -o yaml \ | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install gitea gitea-charts/gitea \ + $(HELM) --kube-context $(CTX) upgrade --install $(GITEA_RELEASE_NAME) \ + $(GITEA_HELM_REPO_NAME)/$(GITEA_CHART_NAME) \ --namespace $(GITEA_NAMESPACE) \ --version $(GITEA_CHART_VERSION) \ --values test/e2e/gitea-values.yaml - kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/gitea --timeout=300s - touch $@ + kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/$(GITEA_RELEASE_NAME) --timeout=$(GITEA_WAIT_TIMEOUT) + echo $(GITEA_CHART_VERSION) > $@ PROMETHEUS_SETUP_MANIFESTS := $(shell find test/e2e/setup/prometheus -type f -name '*.yaml') $(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-operator.sh $(PROMETHEUS_SETUP_MANIFESTS) mkdir -p $(CS) KUBECONTEXT=$(CTX) bash test/e2e/scripts/ensure-prometheus-operator.sh + kubectl --context $(CTX) wait --for=condition=Established \ + crd/prometheuses.monitoring.coreos.com \ + crd/servicemonitors.monitoring.coreos.com \ + --timeout=180s kubectl --context $(CTX) apply -n prometheus-operator -f test/e2e/setup/prometheus kubectl --context $(CTX) wait --for=condition=Available prometheus/prometheus-shared-e2e -n prometheus-operator --timeout=180s kubectl --context $(CTX) wait --for=condition=ready pod -l prometheus=prometheus-shared-e2e -n prometheus-operator --timeout=180s touch $@ -$(CS)/age-key.applied: $(CS)/sut.namespace.ready test/e2e/tools/gen-age-key/main.go +# Aggregate stamp for external E2E services required by tests. +$(CS)/services.ready: $(CS)/cert-manager.installed $(CS)/prometheus.installed $(CS)/gitea.installed mkdir -p $(CS) + touch $@ + +# Step 1: Generate age key file โ€” no cluster/namespace dependency; safe to run before installation. +$(CS)/age-key.txt: Makefile test/e2e/tools/gen-age-key/main.go + mkdir -p $(@D) + go run ./test/e2e/tools/gen-age-key \ + --key-file $@ + +# Step 2: Derive Kubernetes Secret manifest from the key file โ€” still no cluster dependency. +$(CS)/$(NAMESPACE)/sops-secret.yaml: $(CS)/age-key.txt Makefile go run ./test/e2e/tools/gen-age-key \ --key-file $(CS)/age-key.txt \ - --secret-file $(CS)/age-key-secret.yaml \ - --namespace sut \ + --secret-file $@ \ + --namespace $(NAMESPACE) \ --secret-name sops-age-key - kubectl --context $(CTX) apply -f $(CS)/age-key-secret.yaml + +# Step 3: Apply the secret into the namespace โ€” requires the namespace to already exist. +$(CS)/$(NAMESPACE)/sops-secret.applied: $(CS)/$(NAMESPACE)/sops-secret.yaml + kubectl --context $(CTX) apply -f $(CS)/$(NAMESPACE)/sops-secret.yaml touch $@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile @@ -303,95 +309,184 @@ $(IS)/controller.id: $(GO_SOURCES) Dockerfile -t $(E2E_LOCAL_IMAGE) . $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(E2E_LOCAL_IMAGE) > $@ +$(IS)/project-image.ready: $(if $(PROJECT_IMAGE_PROVIDED),,$(IS)/controller.id) + mkdir -p $(IS) + @set -euo pipefail; \ + if [ -n "$(PROJECT_IMAGE_PROVIDED)" ]; then \ + if ! $(CONTAINER_TOOL) image inspect "$(PROJECT_IMAGE)" >/dev/null 2>&1; then \ + echo "Pulling PROJECT_IMAGE=$(PROJECT_IMAGE)"; \ + $(CONTAINER_TOOL) pull "$(PROJECT_IMAGE)"; \ + fi; \ + fi; \ + echo "$(PROJECT_IMAGE)" > "$@" + $(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) mkdir -p $(CS) kubectl --context $(CTX) apply -k config/crd kubectl --context $(CTX) wait --for=condition=Established crd --all --timeout=120s touch $@ -$(CS)/image.loaded: $(IS)/controller.id $(CS)/ready - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) - touch $@ - -$(CS)/controller.deployed: $(DEPLOY_INPUTS) - mkdir -p $(CS) - @[ -z "$(PROJECT_IMAGE)" ] || $(CONTAINER_TOOL) pull $(E2E_IMAGE) - $(KIND) load docker-image $(E2E_IMAGE) --name $(CLUSTER_FROM_CTX) - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true - cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_IMAGE) - $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - IMAGE_ID="$$(cat $(IS)/controller.id)"; \ - kubectl --context $(CTX) -n sut patch deploy/gitops-reverser --type=merge \ - -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"configbutler.ai/e2e-image-id\":\"$$IMAGE_ID\"}}}}}"; \ - fi - kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s - touch $@ - -.PHONY: $(CS)/portforward.running -$(CS)/portforward.running: $(CS)/gitea.installed $(CS)/prometheus.installed +$(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready + @set -euo pipefail; \ + mkdir -p $(CS); \ + img_id="$$( $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(PROJECT_IMAGE) )"; \ + echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CTX)"; \ + $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_NAME); \ + echo "$$img_id" > "$@" + +$(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml $(CS)/image.loaded + @set -euo pipefail; \ + ctx="$(CTX)"; ns="$(NAMESPACE)"; img="$(PROJECT_IMAGE)"; c="$(CONTROLLER_CONTAINER)"; sel="$(CONTROLLER_DEPLOY_SELECTOR)"; \ + [ -n "$$img" ] || { echo "ERROR: PROJECT_IMAGE must be non-empty" >&2; exit 2; }; \ + [ -n "$$c" ] || { echo "ERROR: CONTROLLER_CONTAINER must be non-empty" >&2; exit 2; }; \ + count="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" --no-headers 2>/dev/null | wc -l | tr -d ' ')"; \ + [ "$$count" -eq 1 ] || { \ + echo "ERROR: Expected exactly 1 Deployment matching '$$sel' in namespace '$$ns', found $$count" >&2; \ + kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" -o wide || true; \ + exit 1; \ + }; \ + deploy="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$sel" -o jsonpath='{.items[0].metadata.name}')"; \ + echo "Setting deployment/$$deploy container '$$c' to image '$$img'"; \ + kubectl --context "$$ctx" -n "$$ns" set image "deployment/$$deploy" "$$c=$$img" --record=false; \ + kubectl --context "$$ctx" -n "$$ns" rollout status "deployment/$$deploy" --timeout=180s; \ + touch "$@" + +.PHONY: portforward-ensure +portforward-ensure: $(CS)/services.ready ## Ensure port-forwards are running (always checks) mkdir -p $(CS) - if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ - curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then \ - echo "port-forwards already healthy, skipping restart"; \ - touch $@; exit 0; \ - fi - kubectl config use-context $(CTX) - bash test/e2e/scripts/setup-port-forwards.sh - curl -fsS http://localhost:13000/api/healthz >/dev/null || { echo "Gitea health check failed"; exit 1; } - curl -fsS http://localhost:19090/-/healthy >/dev/null || { echo "Prometheus health check failed"; exit 1; } - touch $@ + E2E_KUBECONTEXT=$(CTX) bash test/e2e/scripts/setup-port-forwards.sh -$(CS)/e2e.passed: $(CS)/controller.deployed $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') +E2E_TEST_INPUTS := $(CS)/age-key.txt $(shell find test/e2e -type f \( -name '*.go' -o -name '*.sh' -o -name '*.yaml' -o -name '*.tmpl' \)) +$(CS)/e2e.passed: $(E2E_TEST_INPUTS) Makefile mkdir -p $(CS) - kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_IMAGE) \ + CTX=$(CTX) INSTALL_MODE=$(INSTALL_MODE) NAMESPACE=$(NAMESPACE) \ E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ go test ./test/e2e/ -v -ginkgo.v touch $@ -##@ E2E Quickstart Tests +CONTROLLER_CONTAINER ?= manager +CONTROLLER_DEPLOY_SELECTOR ?= app.kubernetes.io/part-of=gitops-reverser + +# INSTALL_MODE, INSTALL_NAME, NAMESPACE defaults are defined near the E2E variables section above. +VALID_INSTALL_MODES := config-dir helm plain-manifests-file +ifeq (,$(filter $(INSTALL_MODE),$(VALID_INSTALL_MODES))) + $(error INSTALL_MODE must be one of [$(VALID_INSTALL_MODES)]) +endif + +.PHONY: install install-helm install-plain-manifests-file install-config-dir +install: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml + +install-helm: $(CS)/$(NAMESPACE)/helm/install.yaml + +install-plain-manifests-file: $(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml + +install-config-dir: $(CS)/$(NAMESPACE)/config-dir/install.yaml + +# Shared cleanup logic โ€” inlined at the start of every install recipe so it always +# runs whenever the recipe runs, regardless of stamp freshness. +define DO_CLEANUP_INSTALLS + @CTX="$(CTX)" KUBECTL="$(KUBECTL)" bash hack/cleanup-installs.sh +endef + +##@ Deployments of all manifests needed to run +$(CS)/$(NAMESPACE)/helm/install.yaml: $(CS)/services.ready $(HELM_SYNC_OUTPUTS) + $(DO_CLEANUP_INSTALLS) + @set -euo pipefail; \ + mkdir -p "$(@D)"; \ + helm_args=( \ + upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ + --kube-context $(CTX) \ + --namespace $(NAMESPACE) \ + --create-namespace \ + ); \ + if kubectl --context $(CTX) get crd \ + gitproviders.configbutler.ai \ + gittargets.configbutler.ai \ + watchrules.configbutler.ai \ + clusterwatchrules.configbutler.ai >/dev/null 2>&1; then \ + helm_args+=(--skip-crds); \ + fi; \ + helm "$${helm_args[@]}"; \ + $(HELM) --kube-context $(CTX) get manifest $(INSTALL_NAME) \ + --namespace $(NAMESPACE) > "$@" + +CONFIG_DIR_INPUTS := $(shell find config -type f) +$(CS)/$(NAMESPACE)/config-dir/install.yaml: $(CS)/services.ready $(MANIFEST_OUTPUTS) $(CONFIG_DIR_INPUTS) + $(DO_CLEANUP_INSTALLS) + @set -euo pipefail; \ + mkdir -p "$(@D)"; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + cp -R config "$$tmpdir/config"; \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)"); \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set image gitops-reverser="$(PROJECT_IMAGE)"); \ + (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) \ + | tee "$$tmpdir/install.yaml" \ + | $(KUBECTL) --context "$(CTX)" apply -f -; \ + mv "$$tmpdir/install.yaml" "$@" + +$(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml: $(CS)/services.ready dist/install.yaml + $(DO_CLEANUP_INSTALLS) + @set -euo pipefail; \ + mkdir -p "$(@D)"; \ + ctx="$(CTX)"; \ + ns="$(NAMESPACE)"; \ + tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + cp dist/install.yaml "$$tmpdir/install.yaml"; \ + printf '%s\n' \ + 'apiVersion: kustomize.config.k8s.io/v1beta1' \ + 'kind: Kustomization' \ + 'resources:' \ + '- install.yaml' > "$$tmpdir/kustomization.yaml"; \ + ( \ + cd "$$tmpdir" && \ + $(KUSTOMIZE) edit set namespace "$$ns" >/dev/null && \ + $(KUSTOMIZE) build . \ + ) | tee "$$tmpdir/rendered-install.yaml" \ + | $(KUBECTL) --context "$$ctx" apply -f -; \ + mv "$$tmpdir/rendered-install.yaml" "$@" .PHONY: test-e2e-quickstart-helm -test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_HELM) - PROJECT_IMAGE=$(E2E_IMAGE) \ - bash test/e2e/scripts/run-quickstart.sh helm +test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) + CTX=$(CTX) INSTALL_MODE=helm NAMESPACE=$(NAMESPACE) \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=helm \ + E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework .PHONY: test-e2e-quickstart-manifest -test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-installer; \ - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_MANIFEST) - PROJECT_IMAGE=$(E2E_IMAGE) \ - bash test/e2e/scripts/run-quickstart.sh manifest +test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) + CTX=$(CTX) INSTALL_MODE=plain-manifests-file NAMESPACE=$(NAMESPACE) \ + E2E_ENABLE_QUICKSTART_FRAMEWORK=true E2E_QUICKSTART_MODE=plain-manifests-file \ + E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework + -.PHONY: cleanup-port-forwards -cleanup-port-forwards: ## Stop all port-forwards + + +##@ Cleanup +.PHONY: clean +clean: ## Remove build artifacts (binaries, coverage, generated dist files, build stamps) + rm -rf bin/ cover.out dist/ .stamps/ + +.PHONY: clean-installs +clean-installs: + $(DO_CLEANUP_INSTALLS) + +.PHONY: clean-port-forwards +clean-port-forwards: ## Stop all port-forwards @echo "๐Ÿ›‘ Stopping port-forwards..." @-pkill -f "kubectl.*port-forward.*13000" 2>/dev/null || true @-pkill -f "kubectl.*port-forward.*19090" 2>/dev/null || true @echo "โœ… Port-forwards stopped" -.PHONY: cleanup-gitea-e2e -cleanup-gitea-e2e: cleanup-port-forwards ## Clean up Gitea e2e environment - @echo "๐Ÿงน Cleaning up Gitea e2e environment..." - @$(HELM) uninstall gitea --namespace $(GITEA_NAMESPACE) 2>/dev/null || true - @$(KUBECTL) delete namespace $(GITEA_NAMESPACE) 2>/dev/null || true - @echo "โœ… Gitea cleanup completed" +.PHONY: clean-cluster +clean-cluster: ## Tear down the E2E cluster used for tests and remove its stamps + @if $(K3D) cluster list 2>/dev/null | awk '{print $$1}' | grep -q "^$(CLUSTER_NAME)$$"; then \ + echo "๐Ÿงน Deleting k3d cluster '$(CLUSTER_NAME)'"; \ + $(K3D) cluster delete $(CLUSTER_NAME); \ + else \ + echo "โ„น๏ธ k3d cluster '$(CLUSTER_NAME)' does not exist; skipping cleanup"; \ + fi + rm -rf .stamps/cluster/$(CTX) + diff --git a/PROD_DEBUG_REPORT.md b/PROD_DEBUG_REPORT.md new file mode 100644 index 000000000..624cb55ac --- /dev/null +++ b/PROD_DEBUG_REPORT.md @@ -0,0 +1,234 @@ +# Production Debug Report: gitops-reverser + +Date: 2026-02-26 +Workspace: `/workspaces/gitops-reverser2` + +## Scope +- Investigate why configmap/secret changes in `gitops-reverser-example-audit-namespace` are not being tracked. +- Read-only investigation against cluster, except allowed restart of gitops-reverser and changes in `gitops-reverser-example-audit` namespace. + +## Findings Log + +### 1) Context and deployment health + +Command: +```bash +kubectl config current-context +kubectl -n gitops-reverser get all +helm list -A | rg gitops-reverser +``` + +Result: +- Current context: `k3s` +- Pod/Deployment/Service for gitops-reverser are healthy in namespace `gitops-reverser` +- Helm release: `gitops-reverser-gitops-reverser` chart `gitops-reverser-0.17.0+01bf7040249c` + +--- + +### 2) Config objects and watch scope + +Command: +```bash +kubectl -n gitops-reverser-example-audit get gittargets,watchrules -o yaml +kubectl explain watchrule.spec +kubectl get clusterwatchrules -A -o yaml +``` + +Result: +- `WatchRule only-configmaps` exists in `gitops-reverser-example-audit` +- Rule watches resources: `configmaps`, `secrets` +- `WatchRule` is namespace-scoped (per `kubectl explain watchrule.spec`) +- No `ClusterWatchRule` exists + +Interpretation: +- Current setup only tracks objects in namespace `gitops-reverser-example-audit`. + +--- + +### 3) Namespace mismatch check + +Command: +```bash +kubectl get ns | rg gitops-reverser-example +kubectl get ns gitops-reverser-example-audit-namespace -o yaml +``` + +Result: +- Existing namespace: `gitops-reverser-example-audit` +- Namespace `gitops-reverser-example-audit-namespace` does **not** exist + +Interpretation: +- If changes were made in `gitops-reverser-example-audit-namespace`, they cannot be tracked because the namespace does not exist. + +--- + +### 4) Metrics before test + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep "^gitopsreverser_" | sort' +``` + +Key values before controlled test: +- `gitopsreverser_events_received_total`: 6089 +- `gitopsreverser_events_processed_total`: 18 +- `gitopsreverser_commits_total`: 1 +- `gitopsreverser_objects_written_total`: 1 + +--- + +### 5) Controlled end-to-end test in `gitops-reverser-example-audit` + +Command: +```bash +ns="gitops-reverser-example-audit" +cm="codex-debug-cm-081911" +sec="codex-debug-secret-081911" + +kubectl -n "$ns" create configmap "$cm" --from-literal=foo=one +kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=one +kubectl -n "$ns" create configmap "$cm" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - +kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - +kubectl -n "$ns" delete configmap "$cm" --wait=true +kubectl -n "$ns" delete secret "$sec" --wait=true +``` + +Log correlation command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=5m | rg 'codex-debug-cm-081911|codex-debug-secret-081911|Created commit|All events pushed|writeEvents operation|Starting git commit and push' +``` + +Observed logs (highlights): +- `Starting git commit and push` with `eventCount: 6` +- `Created commit` for: + - `CREATE v1/configmaps/codex-debug-cm-081911` + - `CREATE v1/secrets/codex-debug-secret-081911` + - `UPDATE v1/configmaps/codex-debug-cm-081911` + - `UPDATE v1/secrets/codex-debug-secret-081911` + - `DELETE v1/configmaps/codex-debug-cm-081911` + - `DELETE v1/secrets/codex-debug-secret-081911` +- `All events pushed to remote` after each event + +Post-test metrics/status: + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep -E "^gitopsreverser_(events_received_total|events_processed_total|objects_written_total|commits_total|git_operations_total|git_commit_queue_size)" | sort' +kubectl -n gitops-reverser-example-audit get gittarget to-folder-live-cluster -o jsonpath='{.status.lastCommit}{"\n"}{.status.lastReconcileTime}{"\n"}{.status.snapshot.stats.created}{"\n"}{.status.snapshot.stats.deleted}{"\n"}' +``` + +Result: +- `events_processed_total`: 18 -> 24 +- `commits_total`: 1 -> 2 +- `git_operations_total`: 1 -> 7 +- `objects_written_total`: 1 -> 7 +- `GitTarget.status.lastCommit`: changed to `3e4cfd329e3b3e7638253bb3db3beb61ce363a0c` + +Interpretation: +- ConfigMap/Secret tracking is functioning in `gitops-reverser-example-audit`. + +--- + +### 6) Additional finding: startup seed race (likely non-blocking after startup) + +Command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=20m | rg -i 'Failed to route event to GitTargetEventStream|dropping event' +``` + +Observed error: +- `Failed to route event to GitTargetEventStream - dropping event` +- `error: no GitTargetEventStream registered for gitops-reverser-example-audit/to-folder-live-cluster` +- Appears around startup/initial seed window. + +Interpretation: +- There is a race during initial seed listing where events can be dropped before the stream is registered. +- This did **not** prevent later live create/update/delete processing in this test. + +--- + +## Conclusion + +- The system is actively tracking ConfigMap and Secret changes in `gitops-reverser-example-audit` and pushing them to Git. +- Main issue found in this debugging session: namespace mismatch (`gitops-reverser-example-audit-namespace` does not exist; watch rule is namespace-scoped to `gitops-reverser-example-audit`). +- Secondary issue found: startup seed event-drop race exists in logs and may affect initial catch-up behavior at startup. + +--- + +### 7) Root cause for "committed then immediately removed" (`oeps3`) + +#### 7.1 Reproduction evidence from logs + +Command: +```bash +kubectl -n gitops-reverser logs deploy/gitops-reverser --since=90m | rg -n 'oeps3|Starting git commit and push|Deleted file from repository|Created commit' +``` + +Observed sequence: +- `08:21:02` -> `Created commit` `operation:"CREATE"` `resource:"v1/configmaps/oeps3"` +- `08:21:10` -> `Deleted file from repository` `file:"live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml"` +- `08:21:10` -> `Created commit` `operation:"DELETE"` `resource:"live-cluster/v1/configmaps/oeps3"` + +At the same time, object still exists in cluster: + +Command: +```bash +kubectl -n gitops-reverser-example-audit get cm oeps3 -o yaml +``` + +Result: +- `ConfigMap oeps3` exists (`creationTimestamp: 2026-02-26T08:20:55Z`). + +#### 7.2 Why this happens (code-path analysis) + +1) Repo-state reconciliation asks BranchWorker to list resources under GitTarget path: + +- [`internal/watch/event_router.go`](/workspaces/gitops-reverser2/internal/watch/event_router.go:151) + - `worker.ListResourcesInPath(gitTarget.Spec.Path)` + +2) BranchWorker walks `basePath = repoPath + path`, but computes relative path against **repo root**: + +- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:358) +- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:371) + - `relPath := filepath.Rel(repoPath, path)` + +This means files are parsed like: +- `live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml` + +3) Path parser interprets first segment as API group for 5-part paths: + +- [`internal/git/helpers.go`](/workspaces/gitops-reverser2/internal/git/helpers.go:97) + - For 5 parts, sets `group = parts[0]`. + +So parsed identifier becomes effectively: +- `group=live-cluster, version=v1, resource=configmaps, namespace=gitops-reverser-example-audit, name=oeps3` + +instead of core group (`group=""`). + +4) FolderReconciler compares cluster identifiers vs git identifiers and emits deletes for git-only items: + +- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:126) +- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:145) + +Because keys mismatch (`"/v1/..."` vs `"live-cluster/v1/..."`), it emits DELETE. + +5) Event stream behavior makes this asymmetrical: + +- [`internal/reconcile/git_target_event_stream.go`](/workspaces/gitops-reverser2/internal/reconcile/git_target_event_stream.go:161) + - Events with `Object=nil` are skipped **except DELETE**. + +So reconcile-created CREATE events are dropped, but reconcile-created DELETE events are processed, causing file removal commits. + +#### 7.3 Additional corroboration + +Command: +```bash +kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'find /tmp/gitops-reverser-workers/gitops-reverser-example-audit/your-repo/test-gitops-reverser/repos/54d901857c780feacd94c0da35957f7a/live-cluster -type f | sort' +``` + +Result at capture time: +- `live-cluster/v1/configmaps/...` file for `oeps3` was absent in repo cache +- `oeps3` still existed in Kubernetes + +This confirms Git side deletion without cluster deletion. + diff --git a/SIMPLE_FIX_PLAN.md b/SIMPLE_FIX_PLAN.md new file mode 100644 index 000000000..582d9695a --- /dev/null +++ b/SIMPLE_FIX_PLAN.md @@ -0,0 +1,77 @@ +# Simple Fix Plan: "CREATE then immediate DELETE" for namespaced resources + +## Goal +Stop false DELETE commits (for example `oeps3`) while the resource still exists in the cluster. + +## Root Cause (confirmed) +Repo-state listing parses file paths relative to repo root instead of GitTarget base path. + +Example today: +- Actual file: `live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml` +- Parsed as if `group=live-cluster` (wrong) +- Cluster state uses core group (`group=""`) +- Reconciler sees mismatch and emits DELETE for the Git file. + +## Design Principle +Keep the fix minimal and local: +- Fix path normalization at the source (`BranchWorker.listResourceIdentifiersInPath`) +- Do not redesign reconciliation or event stream behavior in this change + +## Implementation Plan +1. Update path handling in `listResourceIdentifiersInPath` +- File: `internal/git/branch_worker.go` +- Change: + - Keep scanning `basePath := repoPath + targetPath`. + - Compute `relPath` relative to `basePath` (not `repoPath`) when `targetPath` is set. + - Normalize separators with `filepath.ToSlash(relPath)` before parsing. +- Expected effect: + - Parser receives `v1/configmaps//.yaml` instead of `live-cluster/v1/...`. + +2. Add regression unit test for path-prefixed repos +- File: `internal/git/branch_worker_test.go` (or dedicated small test file in `internal/git/`) +- Test scenario: + - Create test repo tree containing `live-cluster/v1/configmaps/ns1/oeps3.yaml`. + - Call `ListResourcesInPath("live-cluster")`. + - Assert returned identifier is: + - `Group: ""` + - `Version: "v1"` + - `Resource: "configmaps"` + - `Namespace: "ns1"` + - `Name: "oeps3"` +- Add negative case: + - Ensure marker files (like `.configbutler`) are still ignored. + +3. Add focused reconciler regression test (small) +- File: `internal/reconcile/folder_reconciler_test.go` (or integration test if already present) +- Scenario: + - Cluster state has `/v1/configmaps/ns1/oeps3`. + - Repo state from parsed path should match same key. + - Assert no DELETE is emitted for identical resource identity. + +## Validation Plan +Run in this order: + +```bash +make fmt +make vet +make lint +make test +``` + +Then run required suite before merge: + +```bash +docker info +make test-e2e +make test-e2e-quickstart-manifest +make test-e2e-quickstart-helm +``` + +## Out of Scope (for this fix) +- Changing `GitTargetEventStream` behavior for nil-object CREATE events. +- Reworking snapshot/reconcile architecture. + +## Risk +Low. +- Change is isolated to repo path parsing in one function. +- Regression tests directly cover the broken case. diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index 49e8a379e..ad02d7813 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -6,7 +6,6 @@ metadata: labels: {{- include "gitops-reverser.labels" . | nindent 4 }} app.kubernetes.io/component: manager - app.kubernetes.io/created-by: gitops-reverser app.kubernetes.io/part-of: gitops-reverser control-plane: controller-manager spec: diff --git a/charts/gitops-reverser/templates/rbac.yaml b/charts/gitops-reverser/templates/rbac.yaml index 4b6bd0d5d..786f1d73c 100644 --- a/charts/gitops-reverser/templates/rbac.yaml +++ b/charts/gitops-reverser/templates/rbac.yaml @@ -1,7 +1,7 @@ {{- if .Values.rbac.create -}} -{{- $roleYaml := .Files.Get "config/cluster-role.yaml" -}} +{{- $roleYaml := .Files.Get "config/role.yaml" -}} {{- if eq $roleYaml "" -}} -{{- fail "charts/gitops-reverser/config/cluster-role.yaml is missing. Run `make helm-sync` to regenerate chart RBAC artifacts before templating/installing." -}} +{{- fail "charts/gitops-reverser/config/role.yaml is missing. Run `make helm-sync` to regenerate chart RBAC artifacts before templating/installing." -}} {{- end -}} {{- $roleFile := $roleYaml | fromYaml }} diff --git a/checkmake.ini b/checkmake.ini new file mode 100644 index 000000000..275c70cd4 --- /dev/null +++ b/checkmake.ini @@ -0,0 +1,2 @@ +[maxbodylength] +maxBodyLength = 20 diff --git a/config/deployment.yaml b/config/deployment.yaml index e52e56c46..308c32a6b 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -4,6 +4,7 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: gitops-reverser + app.kubernetes.io/part-of: gitops-reverser control-plane: gitops-reverser name: gitops-reverser namespace: sut diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 570f21c28..1ff22d995 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -12,3 +12,50 @@ images: - name: gitops-reverser newName: gitops-reverser newTag: e2e-local +replacements: +- source: + fieldPath: metadata.namespace + kind: ServiceAccount + name: gitops-reverser + targets: + - fieldPaths: + - metadata.name + select: + kind: Namespace + name: sut + - fieldPaths: + - subjects.0.namespace + select: + kind: ClusterRoleBinding + name: gitops-reverser + - fieldPaths: + - webhooks.0.clientConfig.service.namespace + select: + kind: ValidatingWebhookConfiguration + name: gitops-reverser-validating-webhook-configuration + - fieldPaths: + - metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: / + select: + kind: ValidatingWebhookConfiguration + name: gitops-reverser-validating-webhook-configuration + - fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: . + index: 1 + select: + kind: Certificate + name: gitops-reverser-admission-server-cert + - fieldPaths: + - spec.dnsNames.0 + - spec.dnsNames.1 + options: + delimiter: . + index: 1 + select: + kind: Certificate + name: gitops-reverser-audit-server-cert +namespace: gitops-reverser diff --git a/config/namespace.yaml b/config/namespace.yaml index a79738084..4dd1788cc 100644 --- a/config/namespace.yaml +++ b/config/namespace.yaml @@ -5,4 +5,5 @@ metadata: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: gitops-reverser control-plane: gitops-reverser + pod-security.kubernetes.io/enforce: restricted name: sut diff --git a/config/rbac/cluster-role.yaml b/config/rbac/cluster-role.yaml deleted file mode 100644 index 799c25126..000000000 --- a/config/rbac/cluster-role.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: gitops-reverser -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - patch - - update -- apiGroups: - - configbutler.ai - resources: - - clusterwatchrules - - gitproviders - - gittargets - - watchrules - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - configbutler.ai - resources: - - clusterwatchrules/status - - gitproviders/status - - gittargets/status - - watchrules/status - verbs: - - get - - patch - - update -- apiGroups: - - configbutler.ai - resources: - - gitproviders/finalizers - verbs: - - update diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 7b51eb49b..86f409456 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - sa.yaml - - cluster-role.yaml + - role.yaml - cluster-rolebinding.yaml - cluster-rolebinding-jane.yaml # E2E-only helper: allows impersonated user jane@acme.com writes in tests. diff --git a/config/service.yaml b/config/service.yaml index 7c78d243c..a6669d063 100644 --- a/config/service.yaml +++ b/config/service.yaml @@ -7,7 +7,7 @@ metadata: name: gitops-reverser-service namespace: sut spec: - clusterIP: 10.96.200.200 # This is required because kube-apiserver starts before CoreDNS (so we use a fixed address) + clusterIP: 10.43.200.200 # k3d/k3s service CIDR; kube-apiserver needs a fixed address before CoreDNS is available ports: - name: admission port: 9443 diff --git a/docs/audit-setup/cluster/readme.md b/docs/audit-setup/cluster/readme.md index 34ad9489a..06b18f0cb 100644 --- a/docs/audit-setup/cluster/readme.md +++ b/docs/audit-setup/cluster/readme.md @@ -14,7 +14,7 @@ kube-apiserver-arg: Do the upgrade in order on your contorl plane nodes: -Also copy in the [policy file](/test/e2e/kind/audit/policy.yaml) +Also copy in the [policy file](/test/e2e/cluster/audit/policy.yaml) sudo systemctl restart k3s # Wait 30 seconds diff --git a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md index d4c8857d4..56b1068de 100644 --- a/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md +++ b/docs/ci/E2E_IMAGE_ARTIFACT_REUSE_DESIGN.md @@ -73,7 +73,7 @@ Outcome: - Run `make test-e2e-install-helm` or `make test-e2e-install-manifest` with no `PROJECT_IMAGE`. - Make cleans cluster first (clean-install validation), then sets up e2e infra. - Make auto-builds and Kind-loads `$(E2E_LOCAL_IMAGE)`, then runs smoke install using that image. -- For `test-e2e-install-manifest`, `build-installer` is run first to regenerate `dist/install.yaml`. +- For `test-e2e-install-manifest`, `build-plain-manifests-installer` is run first to regenerate `dist/install.yaml`. Outcome: - Automatic local behavior with explicit clean-install validation for smoke tests. @@ -97,7 +97,7 @@ Outcome: - local fallback path: cleanup cluster, setup infra, build/load local image. - `test-e2e-install-helm`: wrapper to `test-e2e-install`. - `test-e2e-install-manifest`: - - local path: run `build-installer` first. + - local path: run `build-plain-manifests-installer` first. - prebuilt path: use existing manifest artifact. ### Go (`test/e2e/e2e_suite_test.go`) @@ -105,7 +105,7 @@ Outcome: - if `PROJECT_IMAGE` is set: no prep - else: call Make for cluster/image prep (IDE fallback) -### Kind cluster bootstrap (`test/e2e/kind/start-cluster.sh`) +### Kind cluster bootstrap (`test/e2e/cluster/start-cluster.sh`) - Reuses existing Kind cluster if present (no delete/recreate in script). - Creates cluster only when missing. - Still exports/re-writes kubeconfig endpoint for devcontainer networking. @@ -144,7 +144,7 @@ Outcome: - Image pull failures in Kind: - Ensure local build/load ran or `PROJECT_IMAGE` points to reachable registry. - Manifest smoke using stale image: - - Local path: verify `build-installer` ran before smoke target. + - Local path: verify `build-plain-manifests-installer` ran before smoke target. - CI/prebuilt path: verify artifact source and `kubectl set image` override message. - Pod readiness says "no matching resources": - Verify selector in smoke logs (`Pod selector: ...`) and deployment selector labels. diff --git a/docs/ci/FINDINGS.md b/docs/ci/FINDINGS.md index 2df67b0ce..773bc1c38 100644 --- a/docs/ci/FINDINGS.md +++ b/docs/ci/FINDINGS.md @@ -54,7 +54,7 @@ The current working model is: - `--add-host=host.docker.internal:host-gateway` - Kind cluster config sets: - `networking.apiServerAddress: "0.0.0.0"` -- `test/e2e/kind/start-cluster.sh` rewrites kubeconfig server endpoints from +- `test/e2e/cluster/start-cluster.sh` rewrites kubeconfig server endpoints from `127.0.0.1|localhost|0.0.0.0` to `host.docker.internal:` and sets `tls-server-name=localhost` diff --git a/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md index a689f5815..0d990fe4f 100644 --- a/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md +++ b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md @@ -44,7 +44,7 @@ This matches known upstream issue patterns for Kind in DOOD-style environments: ## Implemented Interim Workaround (DOOD) -`test/e2e/kind/start-cluster.sh` now includes a compact self-heal path for known DOOD bootstrap flakes: +`test/e2e/cluster/start-cluster.sh` now includes a compact self-heal path for known DOOD bootstrap flakes: create with `--retain`, wait for API readiness, apply Kind default CNI/storage with `--validate=false`, then wait for `kindnet` and node readiness. @@ -78,8 +78,8 @@ environment, removing host-socket coupling for local cluster lifecycle. ### Phase 2: Kind Config Alignment -1. Keep `test/e2e/kind/cluster-template.yaml` mount source aligned with container-visible path. -2. Review `test/e2e/kind/start-cluster.sh` kubeconfig rewrite logic: +1. Keep `test/e2e/cluster/cluster-template.yaml` mount source aligned with container-visible path. +2. Review `test/e2e/cluster/start-cluster.sh` kubeconfig rewrite logic: - Current rewrite to `host.docker.internal` is DOOD-oriented - In DinD, direct endpoint usage is preferred unless a concrete networking reason remains 3. Validate audit mounts from inside Kind control-plane: diff --git a/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md b/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md index d5e07a99b..bf4b1d95c 100644 --- a/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md +++ b/docs/ci/KUBECTL_TLS_DEBUG_REPORT.md @@ -43,7 +43,7 @@ In `.devcontainer/devcontainer.json`: ### B) Kind API server bind address -In `test/e2e/kind/cluster-template.yaml`: +In `test/e2e/cluster/cluster-template.yaml`: ```yaml networking: @@ -54,7 +54,7 @@ This ensures host publish is reachable from devcontainer via `host.docker.intern ### C) Kubeconfig rewrite in cluster setup script -In `test/e2e/kind/start-cluster.sh`, after `kind export kubeconfig`: +In `test/e2e/cluster/start-cluster.sh`, after `kind export kubeconfig`: - detect kubeconfig server host in `{127.0.0.1, localhost, 0.0.0.0}` - rewrite to `https://host.docker.internal:` diff --git a/docs/e2e-shared-infra-parallel-plan.md b/docs/e2e-shared-infra-parallel-plan.md index b2c0a286b..7f8a1bb05 100644 --- a/docs/e2e-shared-infra-parallel-plan.md +++ b/docs/e2e-shared-infra-parallel-plan.md @@ -104,8 +104,8 @@ Shared prerequisites: Deploy/install actions (to add/refine): - `e2e-deploy-full` (run-scoped install/deploy for full flow) -- `e2e-deploy-quickstart-helm` (run-scoped Helm quickstart deploy) -- `e2e-deploy-quickstart-manifest` (run-scoped manifest quickstart deploy) +- `$(CS)/$(NAMESPACE)/deploy-helm` (run-scoped Helm quickstart deploy) +- `deploy-installer` (run-scoped manifest quickstart deploy) Expected inputs for these targets: diff --git a/docs/e2e/gitea-setup-target-split-plan.md b/docs/e2e/gitea-setup-target-split-plan.md new file mode 100644 index 000000000..fcc0949fb --- /dev/null +++ b/docs/e2e/gitea-setup-target-split-plan.md @@ -0,0 +1,186 @@ +# E2E Gitea Setup Split Plan (Make Targets + Concrete Artifacts) + +## Overview + +This plan splits `test/e2e/scripts/setup-gitea.sh` into clear setup levels and maps each level to explicit Make +targets that produce concrete artifact files. + +Goal: + +1. Keep shared/bootstrap work one-time per cluster context (`CTX`). +2. Keep one active repo per test run/namespace. +3. Store most run artifacts in `$(CS)/$(NAMESPACE)/repo/`. +4. Keep checkout data in `$(CS)/$(NAMESPACE)/repo//` for easy developer inspection. + +## Current Behavior Summary + +Today `setup-gitea.sh` does all of this in one execution: + +1. Check API readiness through localhost port-forward. +2. Ensure org (`testorg`). +3. Create/reuse token. +4. Generate SSH keypair and configure Gitea user key. +5. Create namespace secrets (`git-creds`, `git-creds-ssh`, `git-creds-invalid`). +6. Create a repo. +7. Clone checkout and configure git. + +Call sites: + +- `test/e2e/e2e_test.go` in `BeforeAll` +- `test/e2e/quickstart_framework_e2e_test.go` in `setupGiteaRepository` + +Suite-level prep (`test/e2e/e2e_suite_test.go`) currently runs `make prepare-e2e`, but repo+secret bootstrap still happens +inside per-spec setup scripts. + +## Proposed Split + +### Level 1: Cluster-scoped bootstrap (once per CTX) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/gitea/bootstrap/` + +Concrete artifact targets: + +1. `$(CS)/gitea/bootstrap/api.ready` +2. `$(CS)/gitea/bootstrap/org-testorg.ready` +3. `$(CS)/gitea/bootstrap/ready` (aggregate stamp only after concrete files exist) + +Why this helps: + +- Shared infra readiness is explicit and isolated from run-scoped data. +- Per-run token/SSH/secret generation can happen cleanly after infra is prepared. + +### Level 2: Run-scoped active repo + credentials (per NAMESPACE) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/` + +Concrete artifact targets: + +1. `$(CS)/$(NAMESPACE)/repo/active-repo.txt` (contains repo name) +2. `$(CS)/$(NAMESPACE)/repo/token.txt` +3. `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa` +4. `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa.pub` +5. `$(CS)/$(NAMESPACE)/repo/ssh/known_hosts` +6. `$(CS)/$(NAMESPACE)/repo/secrets.yaml` (single multi-doc manifest for `git-creds`, `git-creds-ssh`, `git-creds-invalid`) +7. `$(CS)/$(NAMESPACE)/repo/secrets.applied` +8. `$(CS)/$(NAMESPACE)/repo/repo.ready` + +Source inputs: + +- `$(CS)/gitea/bootstrap/org-testorg.ready` +- `$(CS)/gitea/bootstrap/api.ready` + +### Level 3: Checkout (per active repo name) + +Artifacts under: + +- `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/` + +Concrete artifact targets: + +1. `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` +2. `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` +3. `$(CS)/$(NAMESPACE)/repo/checkout.ready` (aggregate; uses active repo from `active-repo.txt`) + +Default checkout location for active repo: + +- `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout` +- `CHECKOUT_DIR` may override, but should default to the stamp path above. + +## Script Refactor Shape + +Recommended split while preserving behavior: + +1. `test/e2e/scripts/gitea-bootstrap.sh` +2. `test/e2e/scripts/gitea-run-setup.sh` (active repo + token + ssh + secrets + checkout) +3. Keep `test/e2e/scripts/setup-gitea.sh` as compatibility wrapper (calls bootstrap + run setup). + +This allows incremental migration without breaking existing tests. + +## Makefile Target Contract + +Suggested phony wrappers: + +1. `e2e-gitea-bootstrap` -> `$(CS)/gitea/bootstrap/ready` +2. `e2e-gitea-run-setup` -> `$(CS)/$(NAMESPACE)/repo/repo.ready` and `$(CS)/$(NAMESPACE)/repo/checkout.ready` +3. `e2e-gitea-run-setup` requires `REPO_NAME` and optional `CHECKOUT_DIR` + +Suggested integration: + +1. Keep `prepare-e2e` focused on cluster + controller prep only. +2. Add a second Make call from `ensureE2EPrepared()` in `test/e2e/e2e_suite_test.go`: + - first call: `make ... prepare-e2e` + - second call: `make ... REPO_NAME= e2e-gitea-run-setup` +3. The second call must run after `prepare-e2e`. + +## Producer/Consumer File Map + +| Producer target | Output file | Consumed by | +| --- | --- | --- | +| `$(CS)/gitea/bootstrap/api.ready` | `.stamps/cluster/$(CTX)/gitea/bootstrap/api.ready` | run-setup target | +| `$(CS)/gitea/bootstrap/org-testorg.ready` | `.stamps/cluster/$(CTX)/gitea/bootstrap/org-testorg.ready` | run-setup target | +| `$(CS)/$(NAMESPACE)/repo/active-repo.txt` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/active-repo.txt` | checkout target + suite env export | +| `$(CS)/$(NAMESPACE)/repo/token.txt` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/token.txt` | `secrets.yaml`, checkout clone auth | +| `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/id_rsa` | `secrets.yaml` | +| `$(CS)/$(NAMESPACE)/repo/ssh/id_rsa.pub` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/id_rsa.pub` | Gitea SSH key registration | +| `$(CS)/$(NAMESPACE)/repo/ssh/known_hosts` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/ssh/known_hosts` | `secrets.yaml` | +| `$(CS)/$(NAMESPACE)/repo/secrets.yaml` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/secrets.yaml` | `secrets.applied` | +| `$(CS)/$(NAMESPACE)/repo/secrets.applied` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/secrets.applied` | `repo.ready` | +| `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout.path` | e2e/quickstart tests needing checkout location | +| `$(CS)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/$(REPO_NAME)/checkout/.git/HEAD` | `checkout.ready` | +| `$(CS)/$(NAMESPACE)/repo/repo.ready` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/repo.ready` | suite execution start gate | +| `$(CS)/$(NAMESPACE)/repo/checkout.ready` | `.stamps/cluster/$(CTX)/$(NAMESPACE)/repo/checkout.ready` | suite execution start gate | + +Recommended environment variables per producer: + +1. Bootstrap level: `CTX`, `GITEA_NAMESPACE`, optional `GITEA_ADMIN_USER`, `GITEA_ADMIN_PASS`. +2. Run setup level: `CTX`, `NAMESPACE`, `REPO_NAME`, optional `CHECKOUT_DIR`, `GITEA_NAMESPACE`. + +## Go Test Wiring Plan + +1. `test/e2e/e2e_suite_test.go` + - keep existing `prepare-e2e` invocation + - add second make invocation after `prepare-e2e`: + - `make ... REPO_NAME= e2e-gitea-run-setup` + - export outputs for test usage: + - `E2E_REPO_NAME` + - `E2E_CHECKOUT_DIR` (read from `checkout.path`) + - `E2E_GIT_SECRET_HTTP`, `E2E_GIT_SECRET_SSH`, `E2E_GIT_SECRET_INVALID` (run-scoped names) +2. `test/e2e/e2e_test.go` + - remove direct `setup-gitea.sh` call + - consume suite-exported repo/secret env vars +3. `test/e2e/quickstart_framework_e2e_test.go` + - remove direct `setup-gitea.sh` call + - consume suite-exported repo/secret env vars + - use same active repo model as suite setup + +## Feasibility Notes + +1. High feasibility: bootstrap remains stable in `prepare-e2e`. +2. High feasibility: single active repo model keeps state simpler under `$(CS)/$(NAMESPACE)/repo/`. +3. Main implementation risk: namespace mismatch (`NAMESPACE` vs `SUT_NAMESPACE` / `QUICKSTART_NAMESPACE`). + - Mitigation: always pass explicit `NAMESPACE` to run-setup Make call in suite. + +## Implementation Phases + +1. Phase 1: Keep `prepare-e2e` behavior; add `e2e-gitea-run-setup` writing run artifacts to `$(CS)/$(NAMESPACE)/repo/`. +2. Phase 2: Add second Make invocation in `e2e_suite_test.go` immediately after `prepare-e2e`. +3. Phase 3: Remove direct `setup-gitea.sh` invocations from `e2e_test.go` and quickstart test file. +4. Phase 4: Keep wrapper for compatibility, then trim wrapper to call new split scripts only. + +## Validation Gate (after implementation) + +Required sequence: + +1. `make fmt` +2. `make generate` +3. `make vet` +4. `make lint` +5. `make test` +6. `docker info` (precheck for e2e) +7. `make test-e2e` +8. `make test-e2e-quickstart-manifest` +9. `make test-e2e-quickstart-helm` diff --git a/docs/e2e/kubectl-context.md b/docs/e2e/kubectl-context.md new file mode 100644 index 000000000..b4704bb7b --- /dev/null +++ b/docs/e2e/kubectl-context.md @@ -0,0 +1,76 @@ +We will preserve the user chosen kubectl context: but inside the the tests we will everywhere switch to the right context. + +Make it an environment contract: Make exports either KUBECONFIG and/or a โ€œready-to-appendโ€ CTX (or KUBE_CONTEXT) and every script uses that consistently. + +Option A (my go-to): export KUBECONFIG + CTX from Make + +Makefile: + +KUBECONFIG ?= +KUBE_CONTEXT ?= + +export KUBECONFIG +export CTX := $(if $(KUBE_CONTEXT),--context $(KUBE_CONTEXT),) + +.PHONY: e2e +e2e: + ./hack/run-e2e.sh + +Script(s): + +#!/usr/bin/env bash +set -euo pipefail + +# CTX is either empty or "--context name" +kubectl ${CTX:-} get ns +kubectl ${CTX:-} -n kube-system get pods + +Usage: + +make e2e +KUBE_CONTEXT=prod make e2e +KUBECONFIG=./kubeconfigs/prod.yaml make e2e +KUBECONFIG=./kubeconfigs/prod.yaml KUBE_CONTEXT=prod make e2e + +This works because Make exports env vars to subprocesses, and your scripts just append ${CTX:-}. + +Option B: export KUBE_CONTEXT and have scripts add --context + +Makefile: + +export KUBECONFIG ?= +export KUBE_CONTEXT ?= + +Script: + +kubectl_ctx=() +if [[ -n "${KUBE_CONTEXT:-}" ]]; then + kubectl_ctx+=(--context "$KUBE_CONTEXT") +fi + +kubectl "${kubectl_ctx[@]}" get ns + +This is more robust for quoting (contexts with weird chars/spaces), and avoids word-splitting issues. + +Iโ€™d pick B if you can touch scripts + +Because CTX="--context foo" relies on correct splitting. Itโ€™s usually fine (context names almost never have spaces), but arrays are bulletproof. + +Tiny helper to keep scripts clean + +Drop this in hack/kubectl.sh and source it from scripts: + +kubectl_cmd() { + local args=() + [[ -n "${KUBECONFIG:-}" ]] && export KUBECONFIG + [[ -n "${KUBE_CONTEXT:-}" ]] && args+=(--context "$KUBE_CONTEXT") + command kubectl "${args[@]}" "$@" +} + +Then in scripts: + +source "$(dirname "$0")/kubectl.sh" +kubectl_cmd get ns +kubectl_cmd -n default get pods + +If you show one of your bash scripts (how it currently calls kubectl, and whether it already reads env), Iโ€™ll adapt this to your exact style (and keep it consistent with your existing $(CTX) usage). \ No newline at end of file diff --git a/docs/e2e/namespace-label-cleanup-analysis.md b/docs/e2e/namespace-label-cleanup-analysis.md new file mode 100644 index 000000000..e7425db6f --- /dev/null +++ b/docs/e2e/namespace-label-cleanup-analysis.md @@ -0,0 +1,210 @@ +# E2E Namespace Label Cleanup โ€” Investigation & Analysis + +## The Problem + +Running the following sequence fails intermittently: + +```bash +make cleanup-cluster +make test-e2e # INSTALL_MODE=config-dir (default) +make test-e2e-quickstart-helm # INSTALL_MODE=helm +``` + +The failure: when switching from `config-dir` to `helm`, the cleanup step cannot find the +`gitops-reverser` namespace to delete it. The namespace has no `e2e=true` label, so the label +selector in `cleanup-e2e-installs.done` finds nothing and skips deletion. Helm then tries to +install into an already-provisioned namespace, causing a conflict. + +--- + +## Makefile Architecture (Stamp Files) + +The Makefile uses stamp files (empty files whose existence/mtime tracks build state) to express +a dependency graph. The relevant chain for switching install modes: + +``` +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done + โ””โ”€ deletes all "e2e=true" namespaces in the cluster; removes their stamp dirs +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done + โ””โ”€ installs controller into namespace (creates namespace if needed) +$(CS)/$(NAMESPACE)/label-namespace.ready + โ””โ”€ kubectl label --overwrite ns $(NAMESPACE) e2e=true ... +$(CS)/$(NAMESPACE)/sops-secret.applied +$(CS)/$(NAMESPACE)/controller.deployed +$(CS)/$(NAMESPACE)/prepare-e2e.ready +``` + +Where `$(CS) = .stamps/cluster/$(CTX)` and `$(INSTALL_MODE)` is one of `helm`, `config-dir`, +or `plain-manifests-file`. + +### Why this should work + +`label-namespace.ready` depends on `$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.done`. When +`INSTALL_MODE` changes (e.g. config-dir โ†’ helm): + +- `helm/install.done` does **not** exist +- `label-namespace.ready` is therefore **out-of-date** (its prerequisite is missing) +- GNU Make rebuilds it **after** `helm/install.done` is built +- `helm/install.done` depends on `helm/cleanup-e2e-installs.done` (which runs first) + +So the intended order is: **cleanup โ†’ install โ†’ label**. + +--- + +## Investigation Findings + +### 1. GNU Make correctly handles deleted prerequisites + +When `cleanup-e2e-installs.done` runs, it executes: +```bash +rm -rf ".stamps/cluster/$ctx/$ns" +``` +This deletes **all** stamps under `gitops-reverser/`, including `label-namespace.ready` and +`sops-secret.yaml`. These files were considered up-to-date at **plan time** (before any recipes +ran) but are gone when their recipe would execute. + +**Empirically verified**: GNU Make evaluates the dependency graph once at the start, marks +targets for rebuild, then executes in order. When a prerequisite is deleted mid-build by another +recipe, Make correctly re-creates it โ€” it does not skip it just because its own prerequisites +haven't changed since plan time. + +### 2. Dry-run output shows correct ordering + +Running `make -n prepare-e2e INSTALL_MODE=helm` (with config-dir stamps present) produces: + +``` +# helm/cleanup-e2e-installs.done โ† deletes gitops-reverser namespace + stamps +# helm/install.done โ† helm installs, creates fresh namespace +# label-namespace.ready โ† kubectl label e2e=true applied +# sops-secret.applied โ† secret applied into namespace +# prepare-e2e.ready +``` + +The logical order is correct per the dry-run. + +### 3. `sops-secret.yaml` is NOT in the dry-run plan + +At plan time, `sops-secret.yaml` is considered up-to-date (its deps `age-key.txt` and `Makefile` +are unchanged). But the cleanup deletes it. GNU Make re-creates it correctly because +`sops-secret.applied` depends on it and is in the rebuild plan. + +Additionally, `gen-age-key` calls `os.MkdirAll` before writing, so it can recreate the file +even after the stamp directory was removed. + +### 4. The `config/namespace.yaml` does not include `e2e=true` + +```yaml +# config/namespace.yaml +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + control-plane: gitops-reverser + pod-security.kubernetes.io/enforce: restricted + name: sut # replaced by kustomize to the actual NAMESPACE +``` + +When `config-dir/install.done` runs `kubectl apply` (via kustomize), it sets +`kubectl.kubernetes.io/last-applied-configuration` on the namespace **without** `e2e=true`. + +The `e2e=true` label is only applied by the **subsequent** `label-namespace.ready` step via +`kubectl label --overwrite`. Client-side apply preserves externally-added labels (they are not in +the `last-applied-configuration` so `kubectl apply` does not remove them). However, this relies +on `label-namespace.ready` always running **after** any `kubectl apply` that touches the namespace. + +### 5. `setup-gitea.sh` creates the namespace without `e2e=true` + +In `test/e2e/scripts/setup-gitea.sh`, the `setup_credentials()` function does: +```bash +TARGET_NAMESPACE=${TARGET_NAMESPACE:-${SUT_NAMESPACE:-${QUICKSTART_NAMESPACE:-${NAMESPACE:-sut}}}} +kubectl create namespace "$TARGET_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - +``` + +During `make test-e2e` (INSTALL_MODE=config-dir, NAMESPACE=gitops-reverser): +- `NAMESPACE=gitops-reverser` is in the environment +- `SUT_NAMESPACE` and `QUICKSTART_NAMESPACE` are **not** set +- So `TARGET_NAMESPACE=gitops-reverser` +- `kubectl apply` is called **without `--context`**, using the default kubeconfig context + +This `kubectl apply` on the namespace spec has **no `e2e=true` label**. Whether this strips +the label depends on exact timing (before or after `label-namespace.ready` ran) and on +server-side vs client-side apply semantics. + +During `make test-e2e-quickstart-helm`, the quickstart framework sets `SUT_NAMESPACE=sut` in +the subprocess environment before calling `setup-gitea.sh`, so `TARGET_NAMESPACE=sut` โ€” the +`gitops-reverser` namespace is not touched here. + +### 6. Potential server-side apply conflict + +If the namespace was first created by `kubectl apply` (kustomize) and later `helm install +--create-namespace` tries to manage the same namespace object via server-side apply, there +can be field ownership conflicts. Helm uses SSA which might overwrite or drop labels managed +by other field managers. + +--- + +## Unresolved Root Cause + +Despite the above investigation, the **exact mechanism** causing `e2e=true` to be absent when +`helm/cleanup-e2e-installs.done` runs has not been definitively identified without a live +execution trace. The strongest candidates are: + +| Hypothesis | Likelihood | How to verify | +|---|---|---| +| `setup-gitea.sh` applies namespace without `e2e=true` before `label-namespace.ready` runs | Medium | Add `--context` to every kubectl in setup-gitea.sh; check timing in logs | +| Helm SSA strips labels when reinstalling without deletion first | Medium | Check if cleanup actually deleted the namespace before helm install | +| `label-namespace.ready` stamp survives but Kubernetes label was somehow reverted | Low | Add a check in `prepare-e2e.ready` recipe that verifies the label | +| INSTALL_MODE variable not correctly propagated to child make | Low | Print INSTALL_MODE in cleanup recipe | + +--- + +## The User's Pragmatic Notes + +From `notes-whats-going-wrnog.txt`: + +> Perhaps I should go a bit smarter about this: or more pragmatic. I could also have the same +> namespace, always, for now. And just make sure that I have a task that always first deletes the +> old namespace that would also prevent me from sitting in this rabbit hole. + +This is a sound alternative. Instead of relying on the `e2e=true` label to find-and-delete the +namespace, the cleanup target could unconditionally delete the **known** namespace (`gitops-reverser`) +regardless of labels. The label mechanism adds indirection that is fragile when cluster state and +stamp state diverge. + +Other notes the user captured: +- Move Gitea installation to a higher (cluster-level) stamp, similar to how `age-key.txt` moved + to cluster level โ€” Gitea is cluster-wide infrastructure, not per-namespace +- Move Gitea credential creation (`setup-gitea.sh` secret creation) to a higher level stamp so it + runs once, not per test invocation +- Restructure e2e tests for better parallelism (long-term goal) + +--- + +## Recommended Next Step + +Replace the label-based namespace discovery with a **direct, unconditional namespace delete** in +`cleanup-e2e-installs.done`. The current target already knows `$(NAMESPACE)` โ€” use it directly: + +```makefile +# Instead of: find namespaces by e2e=true label, then delete +# Do: unconditionally delete the known namespace +$(CS)/$(NAMESPACE)/$(INSTALL_MODE)/cleanup-e2e-installs.done: Makefile $(CS)/ready + @set -euo pipefail; \ + ctx="$(CTX)"; ns="$(NAMESPACE)"; \ + if kubectl --context "$$ctx" get ns "$$ns" >/dev/null 2>&1; then \ + echo "๐Ÿงน Deleting namespace '$$ns' in '$$ctx'"; \ + kubectl --context "$$ctx" delete ns "$$ns" --ignore-not-found=true; \ + kubectl --context "$$ctx" wait --for=delete "ns/$$ns" --timeout=300s || true; \ + rm -rf ".stamps/cluster/$$ctx/$$ns" 2>/dev/null || true; \ + else \ + echo "โ„น๏ธ Namespace '$$ns' does not exist in '$$ctx'"; \ + fi; \ + ... +``` + +**Trade-off**: loses the ability to detect "orphaned" namespaces from a previous run with a +*different* `NAMESPACE` value. But since `NAMESPACE` defaults to `gitops-reverser` and that +rarely changes, this is an acceptable trade-off for reliability. + +The `e2e=true` label can be kept for informational / debugging purposes without being load-bearing +for the cleanup logic. diff --git a/docs/e2e/notes-whats-going-wrnog.txt b/docs/e2e/notes-whats-going-wrnog.txt new file mode 100644 index 000000000..ff94ff888 --- /dev/null +++ b/docs/e2e/notes-whats-going-wrnog.txt @@ -0,0 +1,30 @@ +Ok I tried to find the solution but failed. Somehow the label is dissapearing and we don't know anymore what to delete. + +Perhaps I should go a bit smarter about this: or more pragmatic. I could also have the same namespace, always, for now. And just make sure that I have a task that always first deltes the old namespace that would also prevent me from sitting in this rabbit hole. + +Let's celebrate the good things that come from this. + +Let's move the gitea installation a layer higher: which will also be a timesaver + +Let's move the gitea creation of credentials a level higher, let's be more clear on what to expect. + +LEt's finally get that structure in the end2end tests, so that we can run things in parallel (and also write tests that mimic what could happen during a well visited talk!) + + +--- + +What I could wish: + +1. I want a flexible way to verify that my code is behaving like it should. +2. I like speed, so I don't want to wait on useless stuff +3. My controller is complex: it requires an audit webhook in the kubernets cluster +4. I had the dream to run multiple tests in parallel: because of 3 I can't at the moment (could use something to distribute the events but thats to much for now) + * Yesterday I realised that what I'm trying now is to complex, it's not needed to install in multiple namespaces. I can't keep the installations +5. An important thing is: I keep the running application, so if the test fails I can investigate the state afterwards +6. I have automated image updating and I have nice Makefile targets that will 'reinstall' when needed (e.g. config/ folder changes) +7. I can be flexible in how I install, so I can also switch to helm and see if it behaves the same. + + +What I learned: +* Makefile is file based, especially the dependency tree is very cool but also fragile if you are managing Kubernetes clusters with it. +* Idea: what could GitOps do in this? Details do matter in this. diff --git a/docs/make-e2e-deps.md b/docs/make-e2e-deps.md deleted file mode 100644 index e2e905f7d..000000000 --- a/docs/make-e2e-deps.md +++ /dev/null @@ -1,387 +0,0 @@ -# E2E test dependencies with Make stamps - -## What the old setup did (replaced) - -`make test-e2e` used to run these prerequisites unconditionally on every invocation: - -``` -setup-cluster โ†’ cleanup-webhook โ†’ setup-e2e โ†’ check-cert-manager โ†’ e2e-deploy โ†’ setup-port-forwards -``` - -Every single step was `.PHONY`. Make couldn't skip any of them. On a warm devcontainer with a running cluster that hadn't changed at all, you still paid for image rebuild, Helm reinstall, kustomize apply, and full rollout waits. - ---- - -## The fix: stamp files per cluster - -Each cluster-side step is represented as a local file. Make's timestamp logic handles the "did this need to rerun?" question automatically. - -The stamp namespace: - -``` -.stamps/ - image/ - controller.id โ† image ID written after docker build - cluster/ - kind-/ - ready โ† cluster created and kubectl works - cert-manager.installed - gitea.installed - prometheus.installed - image.loaded โ† image loaded into this specific cluster - crds.applied - controller.deployed - portforward.running - e2e.passed โ† test result stamp -``` - -Each stamp is written **only after a real verification step** (rollout status, kubectl wait, etc.). If any prerequisite file is newer than the stamp, Make reruns that step โ€” and only that step. - -When `make cleanup-cluster` runs, it also `rm -rf .stamps/cluster/kind-$(KIND_CLUSTER)` so the next run starts fresh. - ---- - -## CTX-parameterized stamp targets - -All stamp targets are generic โ€” they use a single `CTX` variable instead of hardcoded cluster names. This eliminates duplication: one set of rules serves all clusters. - -### Variables - -```make -KIND_CLUSTER_E2E ?= gitops-reverser-test-e2e -KIND_CLUSTER_QUICKSTART_HELM ?= gitops-reverser-test-e2e-quickstart-helm -KIND_CLUSTER_QUICKSTART_MANIFEST ?= gitops-reverser-test-e2e-quickstart-manifest - -# CTX: kubeconfig context for the cluster being operated on. -# Defaults to the main e2e cluster; override with CTX=kind- to reuse stamp targets for other clusters. -CTX ?= kind-$(KIND_CLUSTER_E2E) -# Derive the Kind cluster name by stripping the "kind-" prefix from CTX. -CLUSTER_FROM_CTX = $(patsubst kind-%,%,$(CTX)) -CS := .stamps/cluster/$(CTX) # cluster stamp directory -IS := .stamps/image # image stamp directory - -GO_SOURCES := $(shell find cmd internal -type f -name '*.go') go.mod go.sum - -KUBECONTEXT_QS_HELM := kind-$(KIND_CLUSTER_QUICKSTART_HELM) -KUBECONTEXT_QS_MANIFEST := kind-$(KIND_CLUSTER_QUICKSTART_MANIFEST) -``` - -> **Note**: `GO_SOURCES` covers `cmd/` and `internal/` โ€” there is no `pkg/` directory in this project. - -To target a different cluster with any stamp target, pass `CTX` on the command line: - -```sh -make CTX=kind-my-other-cluster .stamps/cluster/kind-my-other-cluster/cert-manager.installed -``` - ---- - -## Dependency graph - -``` -make test-e2e (phony) - โ””โ”€ $(CS)/e2e.passed - โ”œโ”€ test/e2e/**/*.go (any test change reruns tests) - โ””โ”€ $(CS)/portforward.running - โ”œโ”€ $(CS)/controller.deployed - โ”‚ โ”œโ”€ $(CS)/crds.applied - โ”‚ โ”‚ โ””โ”€ $(CS)/ready - โ”‚ โ”‚ โ””โ”€ test/e2e/kind/start-cluster.sh - โ”‚ โ”‚ test/e2e/kind/cluster-template.yaml - โ”‚ โ”œโ”€ $(CS)/cert-manager.installed - โ”‚ โ”‚ โ””โ”€ $(CS)/ready - โ”‚ โ”œโ”€ $(CS)/prometheus.installed - โ”‚ โ”‚ โ””โ”€ $(CS)/ready - โ”‚ โ”œโ”€ $(IS)/controller.id - โ”‚ โ”‚ โ””โ”€ cmd/**/*.go internal/**/*.go go.mod go.sum Dockerfile - โ”‚ โ””โ”€ config/** (excluding config/crd/*) - โ”œโ”€ $(CS)/gitea.installed - โ”‚ โ””โ”€ $(CS)/ready - โ””โ”€ $(CS)/prometheus.installed -``` - -What this buys you in practice: - -| Change you make | Steps that actually rerun | -|---|---| -| Edit a Go source file | image rebuild โ†’ load into kind โ†’ controller redeploy โ†’ tests | -| Edit a test file only | tests only | -| Edit a CRD manifest | CRD apply โ†’ controller redeploy โ†’ tests | -| Edit RBAC/manager config | controller redeploy โ†’ tests | -| Edit `gitea-values.yaml` | gitea reinstall โ†’ port-forwards restart โ†’ tests | -| Edit `ensure-prometheus-operator.sh` | prometheus reinstall โ†’ port-forwards restart โ†’ tests | -| Change `CERT_MANAGER_VERSION` | cert-manager reinstall โ†’ controller redeploy โ†’ tests | -| Change nothing | everything skipped โ€” only the test binary is checked | - ---- - -## Stamp targets, step by step - -### 1 โ€” Cluster ready - -```make -$(CS)/ready: test/e2e/kind/start-cluster.sh test/e2e/kind/cluster-template.yaml - mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) bash test/e2e/kind/start-cluster.sh - kubectl --context $(CTX) get ns >/dev/null - touch $@ -``` - -### 2 โ€” cert-manager - -```make -$(CS)/cert-manager.installed: $(CS)/ready - mkdir -p $(CS) - kubectl --context $(CTX) apply -f $(CERT_MANAGER_MANIFEST_URL) | grep -v unchanged - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-webhook --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - touch $@ -``` - -### 3 โ€” Gitea - -```make -$(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml - mkdir -p $(CS) - $(HELM) repo add gitea-charts https://dl.gitea.com/charts/ 2>/dev/null || true - $(HELM) repo update gitea-charts - kubectl --context $(CTX) create namespace $(GITEA_NAMESPACE) --dry-run=client -o yaml \ - | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install gitea gitea-charts/gitea \ - --namespace $(GITEA_NAMESPACE) \ - --version $(GITEA_CHART_VERSION) \ - --values test/e2e/gitea-values.yaml - kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status deploy/gitea --timeout=300s - touch $@ -``` - -### 4 โ€” Prometheus Operator - -```make -$(CS)/prometheus.installed: $(CS)/ready test/e2e/scripts/ensure-prometheus-operator.sh - mkdir -p $(CS) - KUBECONTEXT=$(CTX) bash test/e2e/scripts/ensure-prometheus-operator.sh - touch $@ -``` - -### 5 โ€” Image build - -For local images (no registry push), `RepoDigests` is empty. Use the image ID instead โ€” it changes whenever the build output changes. - -```make -$(IS)/controller.id: $(GO_SOURCES) Dockerfile - mkdir -p $(IS) - $(CONTAINER_TOOL) build -t $(E2E_LOCAL_IMAGE) . - $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(E2E_LOCAL_IMAGE) > $@ -``` - -### 6 โ€” Image loaded into cluster - -A separate stamp for loading the image into a specific cluster. This is used by both the main e2e flow (via `controller.deployed`) and the quickstart targets. - -```make -$(CS)/image.loaded: $(IS)/controller.id $(CS)/ready - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) - touch $@ -``` - -### 7 โ€” CRDs applied - -```make -$(CS)/crds.applied: $(CS)/ready $(shell find config/crd -type f) - mkdir -p $(CS) - kubectl --context $(CTX) apply -k config/crd - kubectl --context $(CTX) wait --for=condition=Established crd --all --timeout=120s - touch $@ -``` - -### 8 โ€” Controller deployed - -`DEPLOY_INPUTS` includes `prometheus.installed` because the controller's ServiceMonitor depends on the Prometheus Operator CRDs being present. - -```make -DEPLOY_INPUTS := $(CS)/crds.applied $(CS)/cert-manager.installed $(CS)/prometheus.installed \ - $(IS)/controller.id \ - $(shell find config -type f -not -path 'config/crd/*') - -$(CS)/controller.deployed: $(DEPLOY_INPUTS) - mkdir -p $(CS) - $(KIND) load docker-image $(E2E_LOCAL_IMAGE) --name $(CLUSTER_FROM_CTX) - kubectl --context $(CTX) delete validatingwebhookconfiguration \ - gitops-reverser-validating-webhook-configuration --ignore-not-found=true - cd config && $(KUSTOMIZE) edit set image gitops-reverser=$(E2E_LOCAL_IMAGE) - $(KUSTOMIZE) build config | kubectl --context $(CTX) apply -f - - kubectl --context $(CTX) -n sut rollout status deploy/gitops-reverser --timeout=180s - touch $@ -``` - -### 9 โ€” Port-forwards - -`setup-port-forwards.sh` uses bare `kubectl` (no `--context` flag), so the current kubeconfig context must be set first with `kubectl config use-context $(CTX)`. - -This target stays a **file target, not `.PHONY`**. The file target skips the recipe when the stamp is newer than all prerequisites. The health-check-first pattern inside the recipe handles the case where forwards are already alive from a previous run but the stamp needed refreshing. - -```make -$(CS)/portforward.running: $(CS)/controller.deployed $(CS)/gitea.installed $(CS)/prometheus.installed - mkdir -p $(CS) - if curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ - curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then \ - echo "port-forwards already healthy, skipping restart"; \ - touch $@; exit 0; \ - fi - kubectl config use-context $(CTX) - bash test/e2e/scripts/setup-port-forwards.sh - curl -fsS http://localhost:13000/api/healthz >/dev/null || { echo "Gitea health check failed"; exit 1; } - curl -fsS http://localhost:19090/-/healthy >/dev/null || { echo "Prometheus health check failed"; exit 1; } - touch $@ -``` - -The three execution paths this covers: - -| Situation | What happens | -|---|---| -| Stamp newer than all prerequisites | Make skips the recipe entirely โ€” zero curl calls | -| Stamp stale, but forwards still alive (e.g. controller redeployed, forwards survived) | Fast path: two curls pass โ†’ touch stamp โ†’ done | -| Stamp stale and forwards are dead | Slow path: restart script โ†’ two curls verify โ†’ touch stamp | - -### 10 โ€” E2E tests with result stamp - -```make -$(CS)/e2e.passed: $(CS)/portforward.running $(shell find test/e2e -name '*.go') - mkdir -p $(CS) - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v - touch $@ - -.PHONY: test-e2e -test-e2e: $(CS)/e2e.passed -``` - -### 11 โ€” Cluster teardown invalidates all stamps - -```make -.PHONY: cleanup-cluster -cleanup-cluster: - @if $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER)$$"; then \ - $(KIND) delete cluster --name $(KIND_CLUSTER); \ - fi - rm -rf .stamps/cluster/kind-$(KIND_CLUSTER) -``` - -`KIND_CLUSTER` defaults to `$(KIND_CLUSTER_E2E)` but can be overridden: `make cleanup-cluster KIND_CLUSTER=my-other-cluster`. - ---- - -## Quickstart tests - -Quickstart tests (`test-e2e-quickstart-helm`, `test-e2e-quickstart-manifest`) validate the user-facing install paths (Helm chart and generated `dist/install.yaml`). They use dedicated clusters to keep them isolated from the main e2e suite. - -### Design: always start with a clean cluster - -Quickstart tests **always delete and recreate their cluster** at the start of every run. This avoids a class of failures where Helm tries to install on top of resources that were previously applied with `kubectl apply` (server-side apply), which causes `metadata.managedFields must be nil` errors. Since `run-quickstart.sh` resets install state but doesn't delete CRDs or ClusterRoles, a clean cluster is the only reliable guarantee. - -### CTX override pattern - -Quickstart targets reuse the generic stamp targets via recursive Make with `CTX` overridden: - -```sh -$(MAKE) CTX=kind-gitops-reverser-test-e2e-quickstart-helm \ - .stamps/cluster/kind-gitops-reverser-test-e2e-quickstart-helm/cert-manager.installed -``` - -The inner Make re-evaluates `CS` and `CLUSTER_FROM_CTX` with the overridden `CTX`, so the stamp file rules match correctly. - -### Helm quickstart target - -```make -.PHONY: test-e2e-quickstart-helm -test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_HELM) - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) CTX=$(KUBECONTEXT_QS_HELM) \ - .stamps/cluster/$(KUBECONTEXT_QS_HELM)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_HELM) - PROJECT_IMAGE=$(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) \ - bash test/e2e/scripts/run-quickstart.sh helm -``` - -### Manifest quickstart target - -```make -.PHONY: test-e2e-quickstart-manifest -test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) - always starts with a clean cluster - $(MAKE) cleanup-cluster KIND_CLUSTER=$(KIND_CLUSTER_QUICKSTART_MANIFEST) - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/cert-manager.installed \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/gitea.installed - @if [ -z "$(PROJECT_IMAGE)" ]; then \ - $(MAKE) build-installer; \ - $(MAKE) CTX=$(KUBECONTEXT_QS_MANIFEST) \ - .stamps/cluster/$(KUBECONTEXT_QS_MANIFEST)/image.loaded; \ - fi - kubectl config use-context $(KUBECONTEXT_QS_MANIFEST) - PROJECT_IMAGE=$(if $(PROJECT_IMAGE),$(PROJECT_IMAGE),$(E2E_LOCAL_IMAGE)) \ - bash test/e2e/scripts/run-quickstart.sh manifest -``` - -The manifest target also calls `make build-installer` (regenerates `dist/install.yaml`) before loading the image if `PROJECT_IMAGE` is not set. - -### `run-quickstart.sh` is self-contained - -`run-quickstart.sh` manages its own Gitea port-forward via an internal `ensure_gitea_api_port_forward()` function. It does **not** call `setup-port-forwards.sh` and does not need Prometheus. The Makefile only needs to ensure cert-manager, Gitea, and the image are loaded before invoking the script. - ---- - -## Targeted test runs with Ginkgo focus flags - -The entire e2e suite lives in one `Describe("Manager", ...)` block. You can run subsets without touching the test file: - -```make -.PHONY: test-e2e-gitprovider -test-e2e-gitprovider: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="GitProvider" - -.PHONY: test-e2e-watchrule -test-e2e-watchrule: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="WatchRule" - -.PHONY: test-e2e-encryption -test-e2e-encryption: $(CS)/portforward.running - KIND_CLUSTER=$(KIND_CLUSTER_E2E) PROJECT_IMAGE=$(E2E_LOCAL_IMAGE) \ - go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="encrypt|SOPS|age" -``` - -These are `.PHONY` (always rerun the test binary) but still benefit from stamp-based infra: they won't rebuild the image or reinstall Gitea unless something actually changed. - ---- - -## What stamps cannot protect you from - -Stamps assume the world doesn't change behind Make's back: - -- **Manual `kubectl delete` of a CRD or deployment**: the stamp still exists. Fix: `make cleanup-cluster` to reset stamps. -- **Kind cluster restart / node failure**: `$(CS)/ready` still exists. The recipe guards against this with a live `kubectl get ns` call. -- **Port-forward dies between test runs**: the `.portforward.running` stamp is stale. The curl health-check at the start of the recipe catches this. -- **cert-manager version bump without touching `CERT_MANAGER_VERSION`**: the stamp won't invalidate. Encoding the version in the stamp path (`.../cert-manager-$(CERT_MANAGER_VERSION).installed`) would prevent this, but is not currently implemented. - ---- - -## Summary - -| What | Current (after stamps) | -|---|---| -| `make test-e2e` with nothing changed | Runs tests only | -| Go code change | Rebuilds image โ†’ redeploys โ†’ tests | -| Test file change only | Tests only | -| CRD change | Reapplies CRDs โ†’ redeploys โ†’ tests | -| Gitea config change | Reinstalls Gitea โ†’ tests | -| `make test-e2e-gitprovider` | Runs GitProvider tests only (infra skipped if unchanged) | -| `make test-e2e-quickstart-helm` | Deletes cluster โ†’ fresh cluster โ†’ cert-manager + gitea + image โ†’ quickstart helm test | -| `make test-e2e-quickstart-manifest` | Deletes cluster โ†’ fresh cluster โ†’ cert-manager + gitea + image โ†’ quickstart manifest test | diff --git a/docs/make-e2e-further.md b/docs/make-e2e-further.md deleted file mode 100644 index c5b3317e3..000000000 --- a/docs/make-e2e-further.md +++ /dev/null @@ -1,163 +0,0 @@ -# E2E: move suite setup into Make stamps (clean plan) - -## Goal - -Make `go test ./test/e2e` run **only test logic**, not cluster provisioning or stable suite setup. - -Principle: -- If a step is deterministic and its completion can be captured by a file mtime, it belongs in a Make stamp. -- If it must be unique per run (e.g., unique repo name), it stays in `BeforeAll`. - -## Target outcome - -After this change, `BeforeAll` does only: -- Gitea per-run repo setup (unique repo + checkout dir) -- Prometheus client init + availability check (pure Go + port-forward) - -Everything else becomes stamp-based and runs *outside* the test binary. - -## Changes (implementation checklist) - -## Proposed dependency graph (parallel-friendly) - -This graph is the target Make stamp topology to enable `make -j` parallelism while keeping dependencies correct. - -```mermaid -flowchart TD - GO[Go sources + Dockerfile] --> IMG[IS/controller.id] - - READY[CS/ready] --> CRDS[CS/crds.applied] - READY --> CM[CS/cert-manager.installed] - READY --> PROM[CS/prometheus.installed] - READY --> GITEA[CS/gitea.installed] - READY --> SUTNS[CS/sut.namespace.ready] - - SUTNS --> AGE[CS/age-key.applied] - - IMG --> CTRL[CS/controller.deployed] - CRDS --> CTRL - CM --> CTRL - PROM --> CTRL - SUTNS --> CTRL - - CTRL --> PF[CS/portforward.running] - GITEA --> PF - PROM --> PF - - PF --> E2E[CS/e2e.passed] - AGE --> E2E - TESTSRC[test/e2e/**/*.go] --> E2E -``` - -Parallelism notes: -- `IS/controller.id` can run in parallel with `CS/ready`. -- After `CS/ready`, `CS/crds.applied`, `CS/cert-manager.installed`, `CS/prometheus.installed`, `CS/gitea.installed`, - and `CS/sut.namespace.ready` can run in parallel. -- `CS/age-key.applied` only needs the `sut` namespace; it should not wait for controller rollout. - -### 1) Replace suite Age key generation with a stamp + Go tool - -Why: -- The suite key does not need to change per run. -- The Secret YAML should be materialized as a real file (auditable, diffable). -- The test should not generate keys or apply Secrets. - -Actions: -1. Add a small Go tool: - - Path: `test/e2e/tools/gen-age-key/main.go` - - Output files: - - `--key-file` writes an `age` identity in `age-keygen`-compatible format. - - `--secret-file` writes a Kubernetes Secret manifest containing `stringData.identity.agekey`. -2. Add a cluster-scoped stamp: - -```make -# Where tests read the private key from (overridable). -E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt - -$(CS)/sut.namespace.ready: $(CS)/ready config/namespace.yaml - mkdir -p $(CS) - kubectl --context $(CTX) apply -f config/namespace.yaml - kubectl --context $(CTX) label --overwrite ns sut pod-security.kubernetes.io/enforce=restricted - touch $@ - -$(CS)/age-key.applied: $(CS)/sut.namespace.ready test/e2e/tools/gen-age-key/main.go - mkdir -p $(CS) - go run ./test/e2e/tools/gen-age-key \ - --key-file $(CS)/age-key.txt \ - --secret-file $(CS)/age-key-secret.yaml \ - --namespace sut \ - --secret-name sops-age-key - kubectl --context $(CTX) apply -f $(CS)/age-key-secret.yaml - touch $@ -``` - -3. Stop copying keys around by making the test configurable: - - In `test/e2e/e2e_test.go`, replace the hardcoded key path with: - - `E2E_AGE_KEY_FILE` env var (default `/tmp/e2e-age-key.txt`). - - In `Makefile` set `E2E_AGE_KEY_FILE=$(CS)/age-key.txt` for the test run. - -### 2) Move namespace creation/labeling out of `BeforeAll` - -Namespace creation already happens via Kustomize (`config/namespace.yaml`). - -Make it explicit and parallel-friendly by adding a namespace stamp that runs immediately after cluster readiness: -- `$(CS)/sut.namespace.ready` (create + label) - -Then: -- remove namespace creation + label commands from `BeforeAll` -- ensure `controller.deployed` depends on `$(CS)/sut.namespace.ready` (so the label is applied before pods are created) - -### 3) Remove certificate Secret polling from `BeforeAll` - -Delete `waitForCertificateSecrets()` usage and the helper entirely. - -Rationale: -- The controller `Deployment` mounts `admission-server-cert` and `audit-server-cert` as Secret volumes. -- Pods cannot become Ready (and `kubectl rollout status` cannot succeed) until those Secrets exist. - -### 4) Run preventive CRD cleanup only when the suite actually runs - -Do not put CRD cleanup in `controller.deployed` (it wonโ€™t re-run when the stamp is fresh). - -Instead, run it in the `$(CS)/e2e.passed` recipe immediately before `go test`: -- `kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true` - -This guarantees: -- Cleanup happens exactly when the test suite runs. -- It does not rely on stamp invalidation. - -### 5) Wire the new stamps into the existing chain - -Keep `portforward.running` focused on port-forward readiness. - -Preferred wiring: -- Add `$(CS)/age-key.applied` (and `$(CS)/namespace.configured` if used) as dependencies of `$(CS)/e2e.passed`. - -Example: - -```make -$(CS)/e2e.passed: $(CS)/portforward.running $(CS)/age-key.applied $(shell find test/e2e -name '*.go') - mkdir -p $(CS) - kubectl --context $(CTX) delete crd icecreamorders.shop.example.com --ignore-not-found=true - KIND_CLUSTER=$(CLUSTER_FROM_CTX) PROJECT_IMAGE=$(E2E_IMAGE) \ - E2E_AGE_KEY_FILE=$(CS)/age-key.txt \ - go test ./test/e2e/ -v -ginkgo.v - touch $@ -``` - -## Code edits summary - -- `Makefile`: add `age-key.applied` (and optionally `namespace.configured`) stamps; wire into `e2e.passed`; export `E2E_AGE_KEY_FILE`. -- `test/e2e/tools/gen-age-key/main.go`: new tool to write key file + Secret YAML. -- `test/e2e/e2e_test.go`: remove suite-side namespace/label, age secret setup, and cert-secret wait; read key path from env. -- `test/e2e/helpers.go`: delete `setupSOPSAgeSecret` and `waitForCertificateSecrets` (and related imports). - -## Validation - -Run: -- `make lint` -- `make test` -- `docker info` (must work before e2e) -- `make test-e2e` -- `make test-e2e-quickstart-manifest` -- `make test-e2e-quickstart-helm` diff --git a/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md b/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md index 1d2db3700..26c985c08 100644 --- a/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md +++ b/docs/past/ci/KIND_AUDIT_WEBHOOK_SETUP.md @@ -25,7 +25,7 @@ Even with files correctly mounted: ### Evidence ```bash # Inside devcontainer: -$ ls test/e2e/kind/audit/policy.yaml +$ ls test/e2e/cluster/audit/policy.yaml -rw-r--r-- 1 vscode vscode 1858 policy.yaml # โœ… File exists # From Docker on host: @@ -54,7 +54,7 @@ This injects the physical host path (e.g., `/home/user/projects/gitops-reverser2 #### 2. Created Template Configuration -**File:** [`test/e2e/kind/cluster-template.yaml`](../test/e2e/kind/cluster-template.yaml) +**File:** [`test/e2e/cluster/cluster-template.yaml`](../test/e2e/cluster/cluster-template.yaml) ```yaml kind: Cluster @@ -62,7 +62,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraMounts: - - hostPath: ${HOST_PROJECT_PATH}/test/e2e/kind/audit # โ† Mount to node + - hostPath: ${HOST_PROJECT_PATH}/test/e2e/cluster/audit # โ† Mount to node containerPath: /etc/kubernetes/audit readOnly: false @@ -85,7 +85,7 @@ nodes: #### 3. Created Cluster Startup Script -**File:** [`test/e2e/kind/start-cluster.sh`](../test/e2e/kind/start-cluster.sh) +**File:** [`test/e2e/cluster/start-cluster.sh`](../test/e2e/cluster/start-cluster.sh) The script: 1. Checks if `HOST_PROJECT_PATH` environment variable is set @@ -103,7 +103,7 @@ setup-cluster: ## Set up a Kind cluster for e2e tests if it does not exist @if ! command -v $(KIND) >/dev/null 2>&1; then \ echo "Kind is not installed - skipping..."; \ else \ - KIND_CLUSTER=$(KIND_CLUSTER) bash test/e2e/kind/start-cluster.sh; \ + KIND_CLUSTER=$(KIND_CLUSTER) bash test/e2e/cluster/start-cluster.sh; \ fi ``` @@ -144,7 +144,7 @@ patches: #### 3. Updated Audit Webhook Config -**File:** [`test/e2e/kind/audit/webhook-config.yaml`](../test/e2e/kind/audit/webhook-config.yaml) +**File:** [`test/e2e/cluster/audit/webhook-config.yaml`](../test/e2e/cluster/audit/webhook-config.yaml) ```yaml clusters: @@ -176,15 +176,15 @@ steps: # Generate cluster.yaml from template with HOST_PROJECT_PATH substitution - name: Generate Kind cluster config from template run: | - envsubst < test/e2e/kind/cluster-template.yaml > test/e2e/kind/cluster.yaml - cat test/e2e/kind/cluster.yaml + envsubst < test/e2e/cluster/cluster-template.yaml > test/e2e/cluster/cluster.yaml + cat test/e2e/cluster/cluster.yaml # Use helm/kind-action with our custom config - name: Set up Kind cluster with audit webhook support uses: helm/kind-action@v1.13.0 with: cluster_name: gitops-reverser-test-e2e - config: test/e2e/kind/cluster.yaml # Our generated config + config: test/e2e/cluster/cluster.yaml # Our generated config wait: 5m ``` @@ -350,8 +350,8 @@ The test **"should receive audit webhook events from kube-apiserver"** should pa # Old directories created by previous failed attempts still exist on host # Delete them manually: docker run --rm -v /:/host busybox rm -rf \ - /host/workspaces/gitops-reverser2/test/e2e/kind/audit/policy.yaml \ - /host/workspaces/gitops-reverser2/test/e2e/kind/audit/webhook-config.yaml + /host/workspaces/gitops-reverser2/test/e2e/cluster/audit/policy.yaml \ + /host/workspaces/gitops-reverser2/test/e2e/cluster/audit/webhook-config.yaml # Then try creating cluster again kind delete cluster --name gitops-reverser-test-e2e @@ -377,11 +377,11 @@ docker exec gitops-reverser-test-e2e-control-plane crictl logs $(docker exec git ## Files Modified - [`.devcontainer/devcontainer.json`](../.devcontainer/devcontainer.json) - Added `HOST_PROJECT_PATH` env var -- [`test/e2e/kind/cluster-template.yaml`](../test/e2e/kind/cluster-template.yaml) - Template with placeholder -- [`test/e2e/kind/start-cluster.sh`](../test/e2e/kind/start-cluster.sh) - Script to substitute and create cluster +- [`test/e2e/cluster/cluster-template.yaml`](../test/e2e/cluster/cluster-template.yaml) - Template with placeholder +- [`test/e2e/cluster/start-cluster.sh`](../test/e2e/cluster/start-cluster.sh) - Script to substitute and create cluster - [`Makefile`](../Makefile) - Updated `setup-cluster` target - [`test/e2e/e2e_test.go`](../test/e2e/e2e_test.go) - Added audit webhook test -- [`test/e2e/kind/README.md`](../test/e2e/kind/README.md) - Documentation +- [`test/e2e/cluster/README.md`](../test/e2e/cluster/README.md) - Documentation ## Verification: It Should Work Now! diff --git a/hack/cleanup-installs.sh b/hack/cleanup-installs.sh new file mode 100755 index 000000000..bf15cbe49 --- /dev/null +++ b/hack/cleanup-installs.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +ctx="${CTX:-${1:-}}" +kubectl_bin="${KUBECTL:-${2:-kubectl}}" +base=".stamps/cluster/${ctx}" + +if [[ -z "${ctx}" ]]; then + echo "ERROR: CTX is required (set CTX or pass as arg 1)" >&2 + exit 2 +fi + +echo "๐Ÿงน Cleanup installs in context '${ctx}' (base: ${base})" + +cleanup_ns() { + local ns="$1" + + echo "๐Ÿงน Cleaning namespace '${ns}'" + + if [[ -d "${base}/${ns}" ]]; then + while IFS= read -r -d '' f; do + echo " kubectl delete -f ${f}" + "${kubectl_bin}" --context "${ctx}" delete -f "${f}" --ignore-not-found=true || true + done < <(find "${base}/${ns}" -type f -name "install.yaml" -print0 2>/dev/null || true) + fi + + if "${kubectl_bin}" --context "${ctx}" get namespace "${ns}" >/dev/null 2>&1; then + echo " kubectl delete namespace ${ns}" + "${kubectl_bin}" --context "${ctx}" delete namespace "${ns}" --ignore-not-found=true || true + fi + + if [[ -d "${base}/${ns}" ]]; then + echo " rm -rf ${base}/${ns}" + rm -rf "${base:?}/${ns}" || true + fi +} + +# Match: .stamps/cluster////install.yaml +declare -A seen_ns=() +base_regex="${base//./\\.}" +while IFS= read -r -d '' m; do + if [[ "${m}" =~ ^${base_regex}/([^/]+)/([^/]+)/install\.yaml$ ]]; then + ns="${BASH_REMATCH[1]}" + seen_ns["${ns}"]=1 + fi +done < <(find "${base}" -type f -name "install.yaml" -print0 2>/dev/null || true) + +if [[ "${#seen_ns[@]}" -eq 0 ]]; then + echo "๐Ÿงน No installs found under ${base}" + exit 0 +fi + +while IFS= read -r ns; do + [[ -n "${ns}" ]] && cleanup_ns "${ns}" +done < <(printf '%s\n' "${!seen_ns[@]}" | sort) + +"${kubectl_bin}" --context "${ctx}" delete crd \ + gitproviders.configbutler.ai \ + gittargets.configbutler.ai \ + watchrules.configbutler.ai \ + clusterwatchrules.configbutler.ai \ + --ignore-not-found=true || true diff --git a/test/e2e/E2E_DEBUGGING.md b/test/e2e/E2E_DEBUGGING.md index baff12228..4bf78ef97 100644 --- a/test/e2e/E2E_DEBUGGING.md +++ b/test/e2e/E2E_DEBUGGING.md @@ -77,7 +77,7 @@ Kind Cluster โ”‚ โ””โ”€ Prometheus Operator + Prometheus (scrapes metrics via ServiceMonitor) โ”œโ”€ gitea-e2e namespace โ”‚ โ””โ”€ Gitea (Git server) - โ””โ”€ sut namespace (System Under Test) + โ””โ”€ controller namespace (System Under Test; set via `NAMESPACE`) โ””โ”€ Controller pods (2 replicas, HTTPS metrics) ``` @@ -101,7 +101,7 @@ Kind Cluster 4. **View controller logs:** ```bash - kubectl logs -n sut -l control-plane=gitops-reverser --tail=100 + kubectl logs -n "${NAMESPACE}" -l control-plane=gitops-reverser --tail=100 ``` 5. **Check Prometheus scrape status:** diff --git a/test/e2e/cluster/README.md b/test/e2e/cluster/README.md new file mode 100644 index 000000000..16d3c061f --- /dev/null +++ b/test/e2e/cluster/README.md @@ -0,0 +1,77 @@ +# E2E Cluster (k3d) with Audit Webhook + +This directory contains the remaining e2e cluster assets after the Kind-specific files were removed. +The e2e environment now uses k3d with kube-apiserver audit webhook flags configured at cluster create time. + +## Current Files + +- [`start-cluster.sh`](start-cluster.sh): creates/reuses a k3d cluster and configures kubeconfig for devcontainer usage +- [`audit/policy.yaml`](audit/policy.yaml): Kubernetes audit policy used by kube-apiserver +- [`audit/webhook-config.yaml`](audit/webhook-config.yaml): webhook target used by kube-apiserver audit backend + +## How It Works + +`start-cluster.sh` mounts the local audit directory into the k3d server node and sets: + +- `audit-policy-file=/etc/kubernetes/audit/policy.yaml` +- `audit-webhook-config-file=/etc/kubernetes/audit/webhook-config.yaml` +- `audit-webhook-batch-max-wait=1s` +- `audit-webhook-batch-max-size=10` + +The webhook URL in [`audit/webhook-config.yaml`](audit/webhook-config.yaml) targets: + +`https://10.43.200.200:9444/audit-webhook/kind-e2e` + +Notes: +- `insecure-skip-tls-verify: true` is intentional for local e2e +- `kind-e2e` in the path is the cluster ID label used by tests/metrics + +## Usage + +Run the full e2e suite: + +```bash +make test-e2e +``` + +Run quickstart e2e variants: + +```bash +make test-e2e-quickstart-manifest +make test-e2e-quickstart-helm +``` + +## Verification + +Confirm cluster is up: + +```bash +kubectl get nodes +``` + +Confirm audit files are mounted in the k3d server node: + +```bash +docker exec k3d-gitops-reverser-test-e2e-server-0 ls -l /etc/kubernetes/audit +``` + +Confirm audit metrics are incrementing: + +```promql +gitopsreverser_audit_events_received_total +``` + +## Troubleshooting + +`failed to stat file/directory ... volume mount ...`: this can appear in Docker-outside-of-Docker setups when the host path is valid for the Docker daemon but not visible inside the devcontainer. `start-cluster.sh` includes compatibility symlink logic for this. + +No audit events: + +1. Check webhook config in the mounted directory. +2. Check the operator service and deployment in `gitops-reverser`. +3. Check controller logs for `audit-handler` entries. + +## Reference + +- [Kubernetes Audit Documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/) +- [Audit Handler Implementation](../../../internal/webhook/audit_handler.go) diff --git a/test/e2e/kind/audit/policy.yaml b/test/e2e/cluster/audit/policy.yaml similarity index 100% rename from test/e2e/kind/audit/policy.yaml rename to test/e2e/cluster/audit/policy.yaml diff --git a/test/e2e/kind/audit/webhook-config.yaml b/test/e2e/cluster/audit/webhook-config.yaml similarity index 74% rename from test/e2e/kind/audit/webhook-config.yaml rename to test/e2e/cluster/audit/webhook-config.yaml index 4d3b4f2df..ac9f361d0 100644 --- a/test/e2e/kind/audit/webhook-config.yaml +++ b/test/e2e/cluster/audit/webhook-config.yaml @@ -1,4 +1,4 @@ -# Kubernetes audit webhook configuration for Kind cluster +# Kubernetes audit webhook configuration for k3d cluster # This configures the kube-apiserver to send audit events to the gitops-reverser webhook apiVersion: v1 kind: Config @@ -8,8 +8,8 @@ clusters: cluster: # IMPORTANT: Use fixed ClusterIP instead of DNS name # kube-apiserver starts before CoreDNS, so DNS resolution fails at startup - # The ClusterIP is set in config/default/audit_webhook_service_fixed_ip_patch.yaml - server: https://10.96.200.200:9444/audit-webhook/kind-e2e + # The ClusterIP is set in config/service.yaml + server: https://10.43.200.200:9444/audit-webhook/kind-e2e # Skip TLS verification for testing (webhook uses self-signed cert from cert-manager) insecure-skip-tls-verify: true contexts: diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh new file mode 100644 index 000000000..0bb34c4a3 --- /dev/null +++ b/test/e2e/cluster/start-cluster.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# Script to create k3d cluster with audit webhook support for Docker-outside-Docker devcontainers +# Keeps the "HOST_PROJECT_PATH must be mountable by the Docker daemon" logic from the kind version. +# Lets k3d pick the API port. Rewrites kubeconfig to host.docker.internal: when available. + +set -euo pipefail + +CLUSTER_NAME="${CLUSTER_NAME:-gitops-reverser-test-e2e}" +AUDIT_DIR_REL="test/e2e/cluster/audit" +K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" +REPO_PWD="$(pwd -P)" + +# These filenames match your previous kind cluster-template.yaml +AUDIT_POLICY_FILE="policy.yaml" +AUDIT_WEBHOOK_CONFIG_FILE="webhook-config.yaml" + +docker_can_mount_repo() { + local candidate="$1" + docker run --rm -v "${candidate}:/hostproj:ro" busybox:1.36.1 sh -c \ + "test -f /hostproj/${AUDIT_DIR_REL}/${AUDIT_POLICY_FILE} && test -f /hostproj/${AUDIT_DIR_REL}/${AUDIT_WEBHOOK_CONFIG_FILE}" \ + >/dev/null 2>&1 +} + +resolve_host_project_path() { + local candidates=() + candidates+=("${REPO_PWD}") + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + candidates+=("${HOST_PROJECT_PATH}") + fi + + local candidate + for candidate in "${candidates[@]}"; do + if docker_can_mount_repo "$candidate"; then + echo "$candidate" + return 0 + fi + done + + echo "โŒ ERROR: Unable to determine a usable HOST_PROJECT_PATH for k3d volume mounts." >&2 + echo "Tried:" >&2 + if [ -n "${HOST_PROJECT_PATH:-}" ]; then + echo " - HOST_PROJECT_PATH=${HOST_PROJECT_PATH}" >&2 + else + echo " - HOST_PROJECT_PATH=" >&2 + fi + echo " - pwd=${REPO_PWD}" >&2 + echo "" >&2 + echo "Fix: set HOST_PROJECT_PATH to the path visible to the Docker daemon that contains this repo." >&2 + echo "Examples:" >&2 + echo " - Docker-in-Docker (daemon in this container): export HOST_PROJECT_PATH='${REPO_PWD}'" >&2 + echo " - Docker-outside-of-Docker (daemon on your host): export HOST_PROJECT_PATH=''" >&2 + return 1 +} + +cluster_exists() { + # Use JSON output to avoid formatting differences across versions + k3d cluster list -o json 2>/dev/null | grep -q "\"name\":\"${CLUSTER_NAME}\"" +} + +ensure_k3d_stat_compat_path() { + local host_project_path="$1" + local can_sudo=false + + if [ "${host_project_path}" = "${REPO_PWD}" ]; then + return 0 + fi + + if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then + can_sudo=true + fi + + # k3d validates -v SOURCE with a local stat() before creating containers. + # In Docker-outside-of-Docker setups, HOST_PROJECT_PATH can exist on the host + # daemon but not inside this devcontainer, which causes a noisy warning. + # Creating a local symlink keeps k3d's preflight happy without changing the + # real bind source path passed to Docker. + # Example of the warning that we prevent: WARN[0000] failed to stat file/directory '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/cluster/audit' volume mount '/home/runner/work/gitops-reverser/gitops-reverser/test/e2e/cluster/audit:/etc/kubernetes/audit': please make sure it exists + local parent_dir + parent_dir="$(dirname "${host_project_path}")" + mkdir -p "${parent_dir}" 2>/dev/null || { + if [ "${can_sudo}" = true ]; then + sudo -n mkdir -p "${parent_dir}" >/dev/null 2>&1 || { + echo "โš ๏ธ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 + return 0 + } + else + echo "โš ๏ธ Could not create parent '${parent_dir}' for HOST_PROJECT_PATH compatibility symlink" >&2 + return 0 + fi + } + + if [ -L "${host_project_path}" ]; then + ln -sfn "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { + if [ "${can_sudo}" = true ]; then + sudo -n ln -sfn "${REPO_PWD}" "${host_project_path}" >/dev/null 2>&1 || { + echo "โš ๏ธ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } + else + echo "โš ๏ธ Could not refresh HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + fi + } + return 0 + fi + + if [ -e "${host_project_path}" ]; then + return 0 + fi + + ln -s "${REPO_PWD}" "${host_project_path}" 2>/dev/null || { + if [ "${can_sudo}" = true ]; then + sudo -n ln -s "${REPO_PWD}" "${host_project_path}" >/dev/null 2>&1 || { + echo "โš ๏ธ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + } + else + echo "โš ๏ธ Could not create HOST_PROJECT_PATH compatibility symlink '${host_project_path}'" >&2 + return 0 + fi + } +} + +create_cluster() { + local host_project_path="$1" + local audit_host_dir="${host_project_path}/${AUDIT_DIR_REL}" + + echo "๐Ÿš€ Creating k3d cluster '${CLUSTER_NAME}' with audit webhook support..." + echo "๐Ÿ”ง Using HOST_PROJECT_PATH: ${host_project_path}" + echo "๐Ÿ”ง Mounting audit dir: ${audit_host_dir} -> /etc/kubernetes/audit" + + # NOTE: no --api-port: let k3d pick an available port. + # + # Mount audit dir into the server container and pass kube-apiserver audit flags + # via k3s "--kube-apiserver-arg=..." options. + k3d cluster create "${CLUSTER_NAME}" \ + --kubeconfig-update-default \ + --kubeconfig-switch-context \ + -v "${audit_host_dir}:/etc/kubernetes/audit@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-policy-file=/etc/kubernetes/audit/${AUDIT_POLICY_FILE}@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-config-file=/etc/kubernetes/audit/${AUDIT_WEBHOOK_CONFIG_FILE}@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-batch-max-wait=1s@server:0" \ + --k3s-arg "--kube-apiserver-arg=audit-webhook-batch-max-size=10@server:0" \ + 2>&1 | tee "${K3D_CREATE_LOG_FILE}" +} + +merge_kubeconfig() { + # Ensure kubeconfig is present and current-context points at this cluster + k3d kubeconfig merge "${CLUSTER_NAME}" \ + --kubeconfig-switch-context \ + --kubeconfig-merge-default >/dev/null +} + +rewrite_kubeconfig_for_devcontainer() { + local cluster_entry server host port + + cluster_entry="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" + server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" + + # Parse https://HOST:PORT + if [[ "$server" =~ ^https://([^:/]+):([0-9]+)$ ]]; then + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + else + echo "โ„น๏ธ kubeconfig server is '${server}' (couldn't parse host/port; no rewrite)" + return 0 + fi + + # In devcontainers we want to talk to the docker host. + if getent hosts host.docker.internal >/dev/null 2>&1; then + # Only rewrite if it's something that won't work inside the container + if [[ "$host" == "0.0.0.0" || "$host" == "127.0.0.1" || "$host" == "localhost" ]]; then + echo "๐Ÿ” Rewriting kubeconfig server endpoint to host.docker.internal:${port}..." + kubectl config set-cluster "$cluster_entry" \ + --server="https://host.docker.internal:${port}" \ + --tls-server-name=localhost >/dev/null + echo "โœ… kubeconfig endpoint updated for devcontainer networking" + else + echo "โ„น๏ธ kubeconfig server host is '${host}' (no rewrite needed)" + fi + else + echo "โ„น๏ธ host.docker.internal not resolvable; keeping server as ${server}" + fi +} + +main() { + if cluster_exists; then + echo "โ™ป๏ธ Reusing existing k3d cluster '${CLUSTER_NAME}' (no delete/recreate)" + else + export HOST_PROJECT_PATH + HOST_PROJECT_PATH="$(resolve_host_project_path)" + ensure_k3d_stat_compat_path "${HOST_PROJECT_PATH}" + create_cluster "${HOST_PROJECT_PATH}" + fi + + echo "๐Ÿ“‹ Configuring kubeconfig for cluster '${CLUSTER_NAME}'..." + merge_kubeconfig + rewrite_kubeconfig_for_devcontainer + + echo "โœ… Cluster setup complete!" + echo "Try:" + echo " kubectl get nodes" +} + +main "$@" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ad119805e..75d8b9417 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -19,12 +19,18 @@ limitations under the License. package e2e import ( + "errors" "fmt" "os" + "os/exec" + "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/ConfigButler/gitops-reverser/test/utils" ) // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, @@ -39,13 +45,95 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { if img := os.Getenv("PROJECT_IMAGE"); img == "" { - // In local runs, the Makefile guarantees the cluster and image are ready - // before go test is invoked. Nothing to do here. - By("local run: cluster and image prepared by Makefile") + By("local run: preparing cluster via Makefile target") } else { By(fmt.Sprintf("using pre-built image: %s", img)) } + + By("preparing e2e prerequisites via Makefile target") + ensureE2EPrepared() }) var _ = AfterSuite(func() { }) + +func ensureE2EPrepared() { + ctx := resolveE2EContext() + setE2EKubectlContext(ctx) + ns := resolveE2ENamespace() + installMode, err := resolveE2EInstallMode() + Expect(err).NotTo(HaveOccurred(), "INSTALL_MODE environment variable must be set for e2e runs") + installName := resolveE2EInstallName(ns) + cmd := makeCommand( + fmt.Sprintf("CTX=%s", ctx), + fmt.Sprintf("INSTALL_NAME=%s", installName), + fmt.Sprintf("INSTALL_MODE=%s", installMode), + fmt.Sprintf("NAMESPACE=%s", ns), + "prepare-e2e", + ) + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "failed to run make target for e2e prepare") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + + // Some e2e shell scripts call `kubectl` without an explicit `--context` flag. Ensure `kubectl` is + // pointed at the intended cluster context for the remainder of the test run. + output, err = kubectlRun("config", "use-context", ctx) + Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + + By("ensuring IceCreamOrder CRD is removed before tests") + output, err = kubectlRun( + "delete", "crd", "icecreamorders.shop.example.com", + "--ignore-not-found=true", + ) + Expect(err).NotTo(HaveOccurred(), "failed to delete IceCreamOrder CRD before tests") + _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) + + // The Makefile prepares the age key under the stamp directory. When running `go test` directly (without + // `make test-e2e`), ensure the suite uses that prepared key file. + if strings.TrimSpace(os.Getenv("E2E_AGE_KEY_FILE")) == "" { + wd, err := os.Getwd() + Expect(err).NotTo(HaveOccurred(), "failed to get working directory for e2e run") + ageKeyPath := filepath.Join(wd, ".stamps", "cluster", ctx, "age-key.txt") + Expect(os.Setenv("E2E_AGE_KEY_FILE", ageKeyPath)).To(Succeed()) + } +} + +func makeCommand(args ...string) *exec.Cmd { + cmd := exec.Command("make", args...) + _, _ = fmt.Fprintf( + GinkgoWriter, + "make invocation: args=%v\n", + args, + ) + return cmd +} + +func resolveE2EInstallName(namespace string) string { + if value := strings.TrimSpace(os.Getenv("INSTALL_NAME")); value != "" { + return value + } + return namespace +} + +func resolveE2EInstallMode() (string, error) { + if value := strings.TrimSpace(os.Getenv("INSTALL_MODE")); value != "" { + return value, nil + } + return "", errors.New("INSTALL_MODE environment variable must be set") +} + +func resolveE2EContext() string { + if value := strings.TrimSpace(os.Getenv("CTX")); value != "" { + return value + } + + output, err := kubectlRun("config", "current-context") + if err == nil { + if value := strings.TrimSpace(output); value != "" { + return value + } + } + + return "kind-gitops-reverser-test-e2e" +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4dadc526f..dfd64336d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,13 +20,14 @@ package e2e import ( "bytes" + "context" "encoding/base64" "encoding/json" - "errors" "fmt" "os" "os/exec" "path/filepath" + "sort" "strings" "time" @@ -98,8 +99,12 @@ var _ = Describe("Manager", Ordered, func() { specReport := CurrentSpecReport() if specReport.Failed() { By("Fetching controller manager pod logs") - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace, "--tail=200") - controllerLogs, err := utils.Run(cmd) + controllerLogs, err := kubectlRunInNamespace( + namespace, + "logs", + controllerPodName, + "--tail=200", + ) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Last 200 lines of controller logs:\n %s", controllerLogs) } else { @@ -107,8 +112,12 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching Kubernetes events") - cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.metadata.creationTimestamp") - eventsOutput, err := utils.Run(cmd) + eventsOutput, err := kubectlRunInNamespace( + namespace, + "get", + "events", + "--sort-by=.metadata.creationTimestamp", + ) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events (newest last):\n%s", eventsOutput) } else { @@ -116,8 +125,7 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching controller manager pod description") - cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) - podDescription, err := utils.Run(cmd) + podDescription, err := kubectlRunInNamespace(namespace, "describe", "pod", controllerPodName) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Pod description:\n%s", podDescription) } else { @@ -127,7 +135,7 @@ var _ = Describe("Manager", Ordered, func() { }) // Optimize timeouts for faster test execution - SetDefaultEventuallyTimeout(20 * time.Second) + SetDefaultEventuallyTimeout(30 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) Context("Manager", func() { @@ -136,16 +144,17 @@ var _ = Describe("Manager", Ordered, func() { By("validating that the gitops-reverser pods are running as expected") verifyControllerUp := func(g Gomega) { // Get the names of the gitops-reverser pods - cmd := exec.Command("kubectl", "get", - "pods", "-l", "control-plane=gitops-reverser", + podOutput, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + "-l", + "control-plane=gitops-reverser", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ "{{ .metadata.name }}"+ "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, ) - - podOutput, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve gitops-reverser pod information") podNames := utils.GetNonEmptyLines(podOutput) g.Expect(podNames).To(HaveLen(1), "expected exactly 1 controller pod running") @@ -154,11 +163,14 @@ var _ = Describe("Manager", Ordered, func() { // Validate all pods' status for _, podName := range podNames { - cmd = exec.Command("kubectl", "get", - "pods", podName, "-o", "jsonpath={.status.phase}", - "-n", namespace, + output, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + podName, + "-o", + "jsonpath={.status.phase}", ) - output, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("Running"), fmt.Sprintf("Incorrect status for pod %s", podName)) } @@ -170,10 +182,14 @@ var _ = Describe("Manager", Ordered, func() { By("verifying controller service selects the running controller pod") verifyWebhookService := func(g Gomega) { // Get controller service endpoints - cmd := exec.Command("kubectl", "get", "endpoints", - controllerServiceName, "-n", namespace, - "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "get", + "endpoints", + controllerServiceName, + "-o", + "jsonpath={.subsets[*].addresses[*].targetRef.name}", + ) g.Expect(err).NotTo(HaveOccurred(), "Failed to get controller service endpoints") // Filter out kubectl deprecation warnings from output @@ -203,16 +219,19 @@ var _ = Describe("Manager", Ordered, func() { It("should expose admission and audit ports on one controller service", func() { By("verifying controller service exists") - cmd := exec.Command("kubectl", "get", "svc", controllerServiceName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) Expect(err).NotTo(HaveOccurred(), "Controller service should exist") By("verifying controller service routes to the controller pod") Eventually(func(g Gomega) { - endpointsCmd := exec.Command("kubectl", "get", "endpoints", - controllerServiceName, "-n", namespace, - "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}") - output, endpointsErr := utils.Run(endpointsCmd) + output, endpointsErr := kubectlRunInNamespace( + namespace, + "get", + "endpoints", + controllerServiceName, + "-o", + "jsonpath={.subsets[*].addresses[*].targetRef.name}", + ) g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get controller service endpoints") lines := utils.GetNonEmptyLines(output) @@ -239,8 +258,7 @@ var _ = Describe("Manager", Ordered, func() { verifyWebhook := func(g Gomega) { jsonPath := "jsonpath={.items[?(@.metadata.name=='gitops-reverser-validating-webhook-configuration')]" + ".webhooks[*].name}" - cmd := exec.Command("kubectl", "get", "validatingwebhookconfigurations", "-o", jsonPath) - output, err := utils.Run(cmd) + output, err := kubectlRun("get", "validatingwebhookconfigurations", "-o", jsonPath) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("gitops-reverser.configbutler.ai"), "Event webhook should be registered") @@ -250,14 +268,12 @@ var _ = Describe("Manager", Ordered, func() { It("should ensure the metrics endpoint is serving metrics", func() { By("validating that the controller service is available for metrics") - cmd := exec.Command("kubectl", "get", "service", controllerServiceName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "get", "service", controllerServiceName) Expect(err).NotTo(HaveOccurred(), "Controller service should exist") By("waiting for the metrics endpoint to be ready") verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", controllerServiceName, "-n", namespace) - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "get", "endpoints", controllerServiceName) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") } @@ -265,8 +281,7 @@ var _ = Describe("Manager", Ordered, func() { By("verifying that the controller manager is serving the metrics server") verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "logs", controllerPodName) g.Expect(err).NotTo(HaveOccurred()) jsonMetricsLogLine := "\"logger\":\"controller-runtime.metrics\"," + "\"msg\":\"Serving metrics server\"" @@ -306,10 +321,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("๐Ÿ“Š Baseline events: %.0f\n", baselineEvents) By("creating a ConfigMap to trigger webhook call") - cmd := exec.Command("kubectl", "create", "configmap", "webhook-test-cm", - "--namespace", namespace, - "--from-literal=test=webhook") - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + "webhook-test-cm", + "--from-literal=test=webhook", + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed with working webhook") By("waiting for webhook event metric to increment") @@ -327,8 +345,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("๐Ÿ“Š Inspect metrics: %s\n", getPrometheusURL()) By("cleaning up webhook test resources") - cmd = exec.Command("kubectl", "delete", "configmap", "webhook-test-cm", "--namespace", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + "webhook-test-cm", + "--ignore-not-found=true", + ) }) It("should receive audit webhook events from kube-apiserver", func() { @@ -343,16 +366,23 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("๐Ÿ“Š Baseline kind-e2e audit events: %.0f\n", baselineClusterAuditEvents) By("creating a ConfigMap to trigger audit events") - cmd := exec.Command("kubectl", "create", "configmap", "audit-test-cm", - "--namespace", namespace, - "--from-literal=test=audit") - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + "audit-test-cm", + "--from-literal=test=audit", + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed") - cmd = exec.Command("kubectl", "patch", "configmap", "audit-test-cm", - "--namespace", namespace, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "configmap", + "audit-test-cm", "--type=merge", - "--patch", `{"data":{"test":"audit-updated"}}`) - _, err = utils.Run(cmd) + "--patch", + `{"data":{"test":"audit-updated"}}`, + ) Expect(err).NotTo(HaveOccurred(), "ConfigMap update should succeed") By("waiting for audit event metric to increment") @@ -376,8 +406,13 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("๐Ÿ“Š Total audit events: %.0f\n", currentAuditEvents) By("cleaning up audit test resources") - cmd = exec.Command("kubectl", "delete", "configmap", "audit-test-cm", "--namespace", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + "audit-test-cm", + "--ignore-not-found=true", + ) }) It("should validate GitProvider with real Gitea repository", func() { @@ -431,8 +466,7 @@ var _ = Describe("Manager", Ordered, func() { showControllerLogs("before SSH test") By("๐Ÿ“‹ Checking SSH secret exists") - cmd := exec.Command("kubectl", "get", "secret", "git-creds-ssh", "-n", namespace, "-o", "yaml") - secretOutput, err := utils.Run(cmd) + secretOutput, err := kubectlRunInNamespace(namespace, "get", "secret", "git-creds-ssh", "-o", "yaml") if err != nil { fmt.Printf("โŒ SSH secret not found: %v\n", err) } else { @@ -518,20 +552,28 @@ var _ = Describe("Manager", Ordered, func() { verifyResourceStatus("gittarget", destName, namespace, "True", "Ready", "") By("creating Secret in watched namespace") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") - cmd := exec.Command("kubectl", "create", "secret", "generic", secretName, - "--from-literal=password=do-not-commit", "-n", namespace) - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace( + namespace, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") By("patching Secret once to avoid informer start race and force an update event") - patchCmd := exec.Command( - "kubectl", "patch", "secret", secretName, "-n", namespace, - "--type=merge", "--patch", `{"stringData":{"password":"never-commit-this"}}`, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"never-commit-this"}}`, ) - _, err = utils.Run(patchCmd) Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") By("verifying Secret file is committed and does not contain plaintext data") @@ -571,8 +613,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) By("cleaning up test resources") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) }) @@ -584,8 +625,13 @@ var _ = Describe("Manager", Ordered, func() { generatedSecretName := "sops-age-key-autogen" By("ensuring generated encryption secret does not exist before test") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", generatedSecretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) By("creating GitTarget with age recipient auto-generation enabled") destName := watchRuleName + "-dest" @@ -617,8 +663,14 @@ var _ = Describe("Manager", Ordered, func() { var generatedAgeKey string var generatedRecipient string Eventually(func(g Gomega) { - cmd := exec.Command("kubectl", "get", "secret", generatedSecretName, "-n", namespace, "-o", "json") - output, getErr := utils.Run(cmd) + output, getErr := kubectlRunInNamespace( + namespace, + "get", + "secret", + generatedSecretName, + "-o", + "json", + ) g.Expect(getErr).NotTo(HaveOccurred()) var secretObj map[string]interface{} @@ -668,19 +720,27 @@ var _ = Describe("Manager", Ordered, func() { }, "30s", "2s").Should(Succeed()) By("creating Secret in watched namespace") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) - cmd := exec.Command("kubectl", "create", "secret", "generic", secretName, - "--from-literal=password=do-not-commit", "-n", namespace) - _, err = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") + _, err = kubectlRunInNamespace( + namespace, + "create", + "secret", + "generic", + secretName, + "--from-literal=password=do-not-commit", + ) Expect(err).NotTo(HaveOccurred(), "Secret creation should succeed") By("patching Secret once to avoid informer start race and force an update event") - patchCmd := exec.Command( - "kubectl", "patch", "secret", secretName, "-n", namespace, - "--type=merge", "--patch", `{"stringData":{"password":"autogen-never-commit-this"}}`, + _, err = kubectlRunInNamespace( + namespace, + "patch", + "secret", + secretName, + "--type=merge", + "--patch", + `{"stringData":{"password":"autogen-never-commit-this"}}`, ) - _, err = utils.Run(patchCmd) Expect(err).NotTo(HaveOccurred(), "Secret patch should succeed") By("verifying committed secret is encrypted and decryptable with generated key") @@ -716,10 +776,14 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyEncryptedSecretCommitted, "30s", "2s").Should(Succeed()) By("cleaning up test resources") - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", secretName, - "-n", namespace, "--ignore-not-found=true")) - _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", generatedSecretName, - "-n", namespace, "--ignore-not-found=true")) + _, _ = kubectlRunInNamespace(namespace, "delete", "secret", secretName, "--ignore-not-found=true") + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "secret", + generatedSecretName, + "--ignore-not-found=true", + ) cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) }) @@ -771,9 +835,14 @@ var _ = Describe("Manager", Ordered, func() { By("waiting for controller reconciliation of ConfigMap event") verifyReconciliationLogs := func(g Gomega) { // Get controller logs from all pods (single-pod mode still uses label selector). - cmd := exec.Command("kubectl", "logs", "-l", "control-plane=gitops-reverser", - "-n", namespace, "--tail=500", "--prefix=true") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "logs", + "-l", + "control-plane=gitops-reverser", + "--tail=500", + "--prefix=true", + ) g.Expect(err).NotTo(HaveOccurred()) // Check for git commit operation in logs @@ -842,8 +911,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyGitCommit).Should(Succeed()) By("cleaning up test resources") - cmd := exec.Command("kubectl", "delete", "configmap", configMapName, "-n", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "configmap", configMapName, "--ignore-not-found=true") cleanupWatchRule(watchRuleName, namespace) cleanupGitTarget(destName, namespace) @@ -912,8 +980,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileCreated).Should(Succeed()) By("deleting the ConfigMap to trigger DELETE operation") - cmd := exec.Command("kubectl", "delete", "configmap", configMapName, "-n", namespace) - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace(namespace, "delete", "configmap", configMapName) Expect(err).NotTo(HaveOccurred(), "ConfigMap deletion should succeed") By("verifying ConfigMap file is deleted from Git repository") @@ -983,15 +1050,18 @@ var _ = Describe("Manager", Ordered, func() { verifyResourceStatus("gittarget", destName, namespace, "True", "Ready", "") By("installing the IceCreamOrder CRD to trigger Git commit") - cmd := exec.Command("kubectl", "apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - _, err = utils.Run(cmd) + _, err = kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") Expect(err).NotTo(HaveOccurred(), "Failed to install CRD") By("waiting for CRD to be established") verifyCRDEstablished := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "crd", crdName, - "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}") - output, err := utils.Run(cmd) + output, err := kubectlRun( + "get", + "crd", + crdName, + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("True")) } @@ -1041,15 +1111,18 @@ var _ = Describe("Manager", Ordered, func() { watchRuleName := "watchrule-icecream-orders" By("installing the IceCreamOrder CRD first (needed for custom resource tests)") - cmd := exec.Command("kubectl", "apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") - _, err := utils.Run(cmd) + _, err := kubectlRun("apply", "-f", "test/e2e/templates/icecreamorder-crd.yaml") Expect(err).NotTo(HaveOccurred(), "Failed to install sample CRD") By("waiting for CRD to be established") verifyCRDEstablished := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "crd", "icecreamorders.shop.example.com", - "-o", "jsonpath={.status.conditions[?(@.type=='Established')].status}") - output, err := utils.Run(cmd) + output, err := kubectlRun( + "get", + "crd", + "icecreamorders.shop.example.com", + "-o", + "jsonpath={.status.conditions[?(@.type=='Established')].status}", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(Equal("True")) } @@ -1122,9 +1195,14 @@ var _ = Describe("Manager", Ordered, func() { By("waiting for controller reconciliation of CRD instance event") verifyReconciliationLogs := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", "-l", "control-plane=gitops-reverser", - "-n", namespace, "--tail=500", "--prefix=true") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace( + namespace, + "logs", + "-l", + "control-plane=gitops-reverser", + "--tail=500", + "--prefix=true", + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(output).To(ContainSubstring("git commit"), "Should see git commit operation in logs") @@ -1190,9 +1268,16 @@ var _ = Describe("Manager", Ordered, func() { By("applying status update to the IceCreamOrder CR") statusPatch := `{"status":{"orderStatus":"pending","preparationTime":"5m","totalPrice":"$12.50"}}` - statusCmd := exec.Command("kubectl", "patch", "icecreamorder", crdInstanceName, - "-n", namespace, "--type=merge", "--subresource=status", "-p", statusPatch) - statusOutput, statusErr := utils.Run(statusCmd) + statusOutput, statusErr := kubectlRunInNamespace( + namespace, + "patch", + "icecreamorder", + crdInstanceName, + "--type=merge", + "--subresource=status", + "-p", + statusPatch, + ) if statusErr != nil { // Status subresource might not be configured for this CRD, which is fine for this test By(fmt.Sprintf("โš ๏ธ Status patch not supported (expected): %v", statusErr)) @@ -1239,8 +1324,7 @@ var _ = Describe("Manager", Ordered, func() { By("โœ… Status update verified - no Git commit created and status not in file") By("cleaning up IceCreamOrder instance") - cmd2 := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, _ = utils.Run(cmd2) + _, _ = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") @@ -1355,8 +1439,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyUpdatedFile).Should(Succeed()) By("cleaning up IceCreamOrder instance") - cmd := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) By("Note: GitTarget, WatchRule, GitProvider, and CRD kept for subsequent tests") @@ -1418,8 +1501,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileCreated).Should(Succeed()) By("deleting the CR to trigger DELETE operation") - cmd := exec.Command("kubectl", "delete", "icecreamorder", crdInstanceName, "-n", namespace) - _, err = utils.Run(cmd) + _, err = kubectlRunInNamespace(namespace, "delete", "icecreamorder", crdInstanceName) Expect(err).NotTo(HaveOccurred(), "CRD instance deletion should succeed") By("verifying CRD instance file is deleted from Git repository") @@ -1493,8 +1575,7 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyFileExists).Should(Succeed()) By("deleting the CRD to trigger DELETE operation") - deleteCmd := exec.Command("kubectl", "delete", "crd", crdName) - _, deleteErr := utils.Run(deleteCmd) + _, deleteErr := kubectlRun("delete", "crd", crdName) Expect(deleteErr).NotTo(HaveOccurred(), "CRD deletion should succeed") By("verifying CRD file is deleted from Git repository") verifyFileDeleted := func(g Gomega) { @@ -1517,7 +1598,7 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(string(logOutput)).To(ContainSubstring("DELETE"), "Git log should contain DELETE operation") } - Eventually(verifyFileDeleted).Should(Succeed()) + Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) By("cleaning up test resources") cleanupClusterWatchRule(clusterWatchRuleName) @@ -1539,9 +1620,12 @@ var _ = Describe("Manager", Ordered, func() { cleanupGitProvider("gitprovider-normal") // Clean up IceCreamOrder CRD - cmd := exec.Command("kubectl", "delete", "crd", - "icecreamorders.shop.example.com", "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRun( + "delete", + "crd", + "icecreamorders.shop.example.com", + "--ignore-not-found=true", + ) }) // +kubebuilder:scaffold:e2e-webhooks-checks @@ -1580,24 +1664,18 @@ func deriveAgeRecipient(identityString string) (string, error) { } func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) { - podCmd := exec.Command( - "kubectl", "get", "pods", - "-l", "control-plane=gitops-reverser", - "-n", namespace, - "-o", "jsonpath={.items[0].metadata.name}", - ) - podOutput, err := utils.Run(podCmd) + podName, err := discoverControllerPodName(namespace) if err != nil { - return "", fmt.Errorf("get controller pod: %w", err) - } - podName := strings.TrimSpace(podOutput) - if podName == "" { - return "", errors.New("controller pod name is empty") + return "", err } - cmd := exec.Command( - "kubectl", "exec", "-i", - podName, "-n", namespace, "--", + cmd := kubectlCmdInNamespace( + context.Background(), + namespace, + "exec", + "-i", + podName, + "--", "env", fmt.Sprintf("SOPS_AGE_KEY=%s", ageKey), "/usr/local/bin/sops", "--decrypt", "--input-type", "yaml", "--output-type", "yaml", "/dev/stdin", ) @@ -1610,6 +1688,78 @@ func decryptWithControllerSOPS(ciphertext []byte, ageKey string) (string, error) return string(output), nil } +func discoverControllerPodName(ns string) (string, error) { + deploymentsOutput, err := kubectlRunInNamespace( + ns, + "get", + "deployments", + "-o", + "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}", + ) + if err != nil { + return "", fmt.Errorf("get deployments in namespace %s: %w", ns, err) + } + + deployments := utils.GetNonEmptyLines(deploymentsOutput) + if len(deployments) != 1 { + return "", fmt.Errorf("expected exactly 1 Deployment in namespace %s, found %d", ns, len(deployments)) + } + deploymentName := deployments[0] + + deploymentOutput, err := kubectlRunInNamespace(ns, "get", "deployment", deploymentName, "-o", "json") + if err != nil { + return "", fmt.Errorf("get deployment %s/%s: %w", ns, deploymentName, err) + } + + var deploymentObj unstructured.Unstructured + if unmarshalErr := json.Unmarshal([]byte(deploymentOutput), &deploymentObj); unmarshalErr != nil { + return "", fmt.Errorf("unmarshal deployment %s/%s: %w", ns, deploymentName, unmarshalErr) + } + + matchLabels, found, matchLabelErr := unstructured.NestedStringMap( + deploymentObj.Object, + "spec", + "selector", + "matchLabels", + ) + if matchLabelErr != nil { + return "", fmt.Errorf("read deployment selector labels %s/%s: %w", ns, deploymentName, matchLabelErr) + } + if !found || len(matchLabels) == 0 { + return "", fmt.Errorf("deployment selector labels are empty for %s/%s", ns, deploymentName) + } + + keys := make([]string, 0, len(matchLabels)) + for key := range matchLabels { + keys = append(keys, key) + } + sort.Strings(keys) + selectorParts := make([]string, 0, len(keys)) + for _, key := range keys { + selectorParts = append(selectorParts, fmt.Sprintf("%s=%s", key, matchLabels[key])) + } + selector := strings.Join(selectorParts, ",") + + podOutput, err := kubectlRunInNamespace( + ns, + "get", + "pods", + "-l", + selector, + "-o", + "jsonpath={.items[0].metadata.name}", + ) + if err != nil { + return "", fmt.Errorf("get controller pod for selector %q in namespace %s: %w", selector, ns, err) + } + podName := strings.TrimSpace(podOutput) + if podName == "" { + return "", fmt.Errorf("controller pod name is empty for selector %q in namespace %s", selector, ns) + } + + return podName, nil +} + // createGitProviderWithURL creates a GitProvider resource with the specified URL. func createGitProviderWithURL(name, branch, secretName, repoURL string) { By(fmt.Sprintf("creating GitProvider '%s' with branch '%s', secret '%s' and URL '%s'", @@ -1667,8 +1817,7 @@ func verifyResourceStatus(resourceType, name, ns, expectedStatus, expectedReason args = append(args, "-n", ns) } - cmd := exec.Command("kubectl", args...) - output, err := utils.Run(cmd) + output, err := kubectlRun(args...) g.Expect(err).NotTo(HaveOccurred()) var obj unstructured.Unstructured @@ -1707,8 +1856,13 @@ func verifyResourceStatus(resourceType, name, ns, expectedStatus, expectedReason // cleanupGitProvider deletes a GitProvider resource. func cleanupGitProvider(name string) { By(fmt.Sprintf("cleaning up GitProvider '%s'", name)) - cmd := exec.Command("kubectl", "delete", "gitprovider", name, "-n", namespace, "--ignore-not-found=true") - _, err := utils.Run(cmd) + _, err := kubectlRunInNamespace( + namespace, + "delete", + "gitprovider", + name, + "--ignore-not-found=true", + ) Expect(err).NotTo(HaveOccurred()) } @@ -1717,9 +1871,15 @@ func showControllerLogs(context string) { By(fmt.Sprintf("๐Ÿ“‹ Controller logs %s:", context)) // Get the controller pod name dynamically - cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=gitops-reverser", - "-o", "jsonpath={.items[0].metadata.name}", "-n", namespace) - podName, err := utils.Run(cmd) + podName, err := kubectlRunInNamespace( + namespace, + "get", + "pods", + "-l", + "control-plane=gitops-reverser", + "-o", + "jsonpath={.items[0].metadata.name}", + ) if err != nil { fmt.Printf("โš ๏ธ Failed to get controller pod name: %v\n", err) return @@ -1731,8 +1891,7 @@ func showControllerLogs(context string) { } // Get the logs - cmd = exec.Command("kubectl", "logs", strings.TrimSpace(podName), "-n", namespace, "--tail=20") - output, err := utils.Run(cmd) + output, err := kubectlRunInNamespace(namespace, "logs", strings.TrimSpace(podName), "--tail=20") if err != nil { fmt.Printf("โŒ Failed to get controller logs: %v\n", err) return diff --git a/test/e2e/gitea-values.yaml b/test/e2e/gitea-values.yaml index 103368f16..551c7c747 100644 --- a/test/e2e/gitea-values.yaml +++ b/test/e2e/gitea-values.yaml @@ -75,11 +75,9 @@ service: # Resource limits for CI environments resources: limits: - memory: "512Mi" - cpu: "500m" + memory: "1024Mi" requests: - memory: "256Mi" - cpu: "250m" + memory: "512Mi" podSecurityContext: runAsNonRoot: true diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index ac1ecb1e2..80ba1c2d4 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -26,8 +26,6 @@ import ( "bytes" "context" "fmt" - "os/exec" - "strings" "text/template" "time" @@ -36,12 +34,10 @@ import ( "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" - - "github.com/ConfigButler/gitops-reverser/test/utils" ) // namespace where the project is deployed in. -const namespace = "sut" +var namespace = resolveE2ENamespace() //nolint:gochecknoglobals // used across e2e tests const metricWaitDefaultTimeout = 30 * time.Second const e2eEncryptionRefName = "sops-age-key" @@ -147,24 +143,8 @@ func applyFromTemplate(templatePath string, data interface{}, namespace string, return err } - // 1. Define the binary explicitly - const binary = "kubectl" - - args := []string{ - "apply", "-f", "-", - } - - // Add extra arguments - args = append(args, extraArgs...) - - if namespace != "" { - args = append(args, "-n", namespace) - } - - ctx := context.Background() - cmd := exec.CommandContext(ctx, binary, args...) - cmd.Stdin = strings.NewReader(yamlContent) - _, err = utils.Run(cmd) + args := append([]string{"apply", "-f", "-"}, extraArgs...) + _, err = kubectlRunWithStdin(namespace, yamlContent, args...) return err } @@ -175,8 +155,6 @@ func getBaseFolder() string { } // createGitTarget creates a GitTarget that binds a GitProvider, branch and path. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity in template calls func createGitTarget(name, namespace, providerName, path, branch string) { createGitTargetWithEncryptionOptions( name, @@ -225,32 +203,19 @@ func createGitTargetWithEncryptionOptions( } // cleanupGitTarget deletes a GitTarget resource. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity func cleanupGitTarget(name, namespace string) { By(fmt.Sprintf("cleaning up GitTarget '%s' in ns '%s'", name, namespace)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "gittarget", name, - "-n", namespace, "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "gittarget", name, "--ignore-not-found=true") } // cleanupWatchRule deletes a WatchRule resource. -// -//nolint:unparam // in e2e helpers we accept constant namespace ("sut"); keep signature for clarity func cleanupWatchRule(name, namespace string) { By(fmt.Sprintf("cleaning up WatchRule '%s' in ns '%s'", name, namespace)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "watchrule", name, - "-n", namespace, "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRunInNamespace(namespace, "delete", "watchrule", name, "--ignore-not-found=true") } // cleanupClusterWatchRule deletes a ClusterWatchRule resource. func cleanupClusterWatchRule(name string) { By(fmt.Sprintf("cleaning up ClusterWatchRule '%s'", name)) - ctx := context.Background() - cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterwatchrule", name, - "--ignore-not-found=true") - _, _ = utils.Run(cmd) + _, _ = kubectlRun("delete", "clusterwatchrule", name, "--ignore-not-found=true") } diff --git a/test/e2e/kind/README.md b/test/e2e/kind/README.md deleted file mode 100644 index d63c922d3..000000000 --- a/test/e2e/kind/README.md +++ /dev/null @@ -1,155 +0,0 @@ -# Kind Cluster with Audit Webhook Support - -This directory contains configuration files to set up a Kind cluster with Kubernetes audit webhook support for e2e testing. - -## Overview - -The gitops-reverser operator exposes an experimental audit webhook endpoint at `/audit-webhook/{clusterID}` that receives audit events from the Kubernetes API server. This setup configures Kind to send audit events to this endpoint for testing and metrics collection. - -## Files - -- **[`cluster.yaml`](cluster.yaml)**: Kind cluster configuration with audit webhook support -- **[`audit/policy.yaml`](audit/policy.yaml)**: Kubernetes audit policy (filters noise, captures intent) -- **[`audit/webhook-config.yaml`](audit/webhook-config.yaml)**: Audit webhook configuration (points to operator service) - -## Configuration Details - -### Cluster Configuration - -The [`cluster.yaml`](cluster.yaml:1) mounts the audit policy and webhook configuration files into the Kind control plane node and configures the kube-apiserver with: - -- `audit-policy-file`: Filters which events are audited -- `audit-webhook-config-file`: Configures where to send audit events -- `audit-webhook-batch-max-wait`: Maximum wait time before sending a batch (5s) -- `audit-webhook-batch-max-size`: Maximum events in a batch (100) - -### Webhook Configuration - -The [`webhook-config.yaml`](audit/webhook-config.yaml:1) configures the kube-apiserver to send audit events to: - -``` -https://10.96.200.200:443/audit-webhook/kind-e2e -``` - -**Important**: Uses `insecure-skip-tls-verify: true` because the webhook service uses a self-signed certificate from cert-manager. - -### Audit Policy - -The [`policy.yaml`](audit/policy.yaml:1) is copied from [`docs/audit-setup/cluster/audit/policy.yaml`](../../../docs/audit-setup/cluster/audit/policy.yaml:1) and filters: - -- **Noise**: Drops status updates, heartbeats, ephemeral checks -- **Security**: Filters secrets and sensitive operations -- **Intent**: Captures create/update/patch/delete operations on resources - -## Usage - -### Creating a Cluster - -The Makefile automatically uses this configuration when creating Kind clusters: - -```bash -make setup-cluster -``` - -This will create a cluster named `gitops-reverser-test-e2e` with audit webhook support. - -### Running E2E Tests - -The e2e test suite includes a test that verifies audit events are received: - -```bash -make test-e2e -``` - -Look for the test: **"should receive audit webhook events from kube-apiserver"** - -### Quick Verification - -Use the provided verification script to check if audit webhook is properly configured: - -```bash -./test/e2e/kind/verify-audit.sh -``` - -This script checks: -- Audit configuration files are mounted in the control plane -- kube-apiserver is configured with audit parameters -- Operator and webhook service are deployed -- Basic audit logging is working - -### Verifying Audit Events - -After deploying the operator, you can verify audit events are being received: - -1. **Check metrics** (via Prometheus at `http://localhost:19090`): - ```promql - gitopsreverser_audit_events_received_total - ``` - -2. **Check controller logs**: - ```bash - kubectl logs -n gitops-reverser-system -l control-plane=controller-manager --tail=100 | grep "audit" - ``` - -3. **Create a test resource** to trigger audit events: - ```bash - kubectl create configmap audit-test -n gitops-reverser-system --from-literal=test=value - ``` - -## Metrics - -The audit webhook tracks metrics with labels: - -- `gvr`: Group/Version/Resource (e.g., `apps/v1/deployments`) -- `action`: Verb (e.g., `create`, `update`, `delete`) -- `user`: Username who performed the action -- `processed`: Whether the event was processed (filters status updates) - -## Troubleshooting - -### Audit Events Not Received - -1. **Check kube-apiserver logs**: - ```bash - docker exec gitops-reverser-test-e2e-control-plane cat /var/log/kubernetes/kube-apiserver-audit.log - ``` - -2. **Verify audit webhook service exists**: - ```bash - kubectl get svc -n gitops-reverser-system gitops-reverser-service - ``` - -3. **Check if kube-apiserver can reach the webhook**: - ```bash - kubectl logs -n kube-system -l component=kube-apiserver - ``` - -### File Mounting Issues - -If audit configuration files aren't found, ensure paths in [`cluster.yaml`](cluster.yaml:7-11) are correct relative to the project root: - -```yaml -hostPath: ./test/e2e/kind/audit/policy.yaml -hostPath: ./test/e2e/kind/audit/webhook-config.yaml -``` - -## Differences from K3s Setup - -The K3s setup in [`docs/audit-setup/cluster/`](../../../docs/audit-setup/cluster/) uses: - -- Fixed ClusterIP: `10.43.200.200` (K3s default) -- Service name: Generic reference -- Config location: `/etc/rancher/k3s/audit/` - -The Kind setup uses: - -- Fixed ClusterIP: `10.96.200.200` -- Path-based cluster identity: `/audit-webhook/kind-e2e` -- Config location: `/etc/kubernetes/` (Kind standard) - -## References - -- [Kubernetes Audit Documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/) -- [Kind Extra Mounts](https://kind.sigs.k8s.io/docs/user/configuration/#extra-mounts) -- [Kubeadm Config Patches](https://kind.sigs.k8s.io/docs/user/configuration/#kubeadm-config-patches) -- [Audit Handler Implementation](../../../internal/webhook/audit_handler.go:1) diff --git a/test/e2e/kind/cluster-template.yaml b/test/e2e/kind/cluster-template.yaml deleted file mode 100644 index 2b9619bce..000000000 --- a/test/e2e/kind/cluster-template.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# kind cluster configuration template with audit webhook support -# This template uses ${HOST_PROJECT_PATH} which gets replaced at runtime -# with the actual host path by the start-cluster.sh script -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: - # Bind the API server on all host interfaces so devcontainer clients can - # reach it via host.docker.internal. - apiServerAddress: "0.0.0.0" -nodes: -- role: control-plane - # Mount the entire audit directory using $HOST_PROJECT_PATH. - extraMounts: - - hostPath: ${HOST_PROJECT_PATH}/test/e2e/kind/audit - containerPath: /etc/kubernetes/audit - readOnly: false - - # Configure kube-apiserver with audit webhook - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - audit-policy-file: /etc/kubernetes/audit/policy.yaml - audit-webhook-config-file: /etc/kubernetes/audit/webhook-config.yaml - audit-webhook-batch-max-wait: "1s" - audit-webhook-batch-max-size: "10" - extraVolumes: - - name: audit-config - hostPath: /etc/kubernetes/audit - mountPath: /etc/kubernetes/audit - readOnly: true - pathType: DirectoryOrCreate diff --git a/test/e2e/kind/start-cluster.sh b/test/e2e/kind/start-cluster.sh deleted file mode 100755 index c60a57d3d..000000000 --- a/test/e2e/kind/start-cluster.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Script to create Kind cluster with proper host path substitution for Docker-in-Docker - -set -euo pipefail - -CLUSTER_NAME="${KIND_CLUSTER:-gitops-reverser-test-e2e}" -TEMPLATE_FILE="test/e2e/kind/cluster-template.yaml" -CONFIG_FILE="test/e2e/kind/cluster.ignore.yaml" -KIND_CREATE_LOG_FILE="${TMPDIR:-/tmp}/kind-create-${CLUSTER_NAME}.log" -POD_SUBNET="${KIND_POD_SUBNET:-10.244.0.0/16}" - -create_primary_cluster() { - kind create cluster --name "$CLUSTER_NAME" --config "$CONFIG_FILE" --wait 5m --retain 2>&1 | tee "$KIND_CREATE_LOG_FILE" -} - -# Known Kind bootstrap race in DOOD setups: https://github.com/kubernetes-sigs/kind/issues/2867 -is_known_kind_bootstrap_flake() { - grep -Eq \ - "failed to apply overlay network|failed to remove control plane taint|failed to remove control plane load balancer label|failed to download openapi|couldn't get current server API group list|The connection to the server .*:6443 was refused" \ - "$KIND_CREATE_LOG_FILE" -} - -run_dood_self_heal() { - local control_plane_container="${CLUSTER_NAME}-control-plane" - docker inspect "$control_plane_container" >/dev/null 2>&1 || return 1 - - local ready=false - for _ in $(seq 1 120); do - if docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw=/readyz >/dev/null 2>&1; then - ready=true - break - fi - sleep 2 - done - if [ "$ready" != "true" ]; then - docker exec "$control_plane_container" crictl ps -a || true - return 1 - fi - - local applied=false - for _ in $(seq 1 40); do - if docker exec "$control_plane_container" sh -lc \ - "sed 's|{{ \\.PodSubnet }}|${POD_SUBNET}|g' /kind/manifests/default-cni.yaml > /tmp/default-cni-rendered.yaml && \ - kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /tmp/default-cni-rendered.yaml && \ - kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /kind/manifests/default-storage.yaml"; then - applied=true - break - fi - sleep 2 - done - [ "$applied" = "true" ] || return 1 - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - taint nodes --all node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - label nodes --all node.kubernetes.io/exclude-from-external-load-balancers- >/dev/null 2>&1 || true - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - -n kube-system rollout status daemonset/kindnet --timeout=300s - - docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ - wait --for=condition=Ready nodes --all --timeout=300s -} - -if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then - echo "โ™ป๏ธ Reusing existing Kind cluster '$CLUSTER_NAME' (no delete/recreate)" -else - # HOST_PROJECT_PATH is only needed when creating a new cluster (for host-path volume mounts). - # Clusters pre-created by CI (helm/kind-action) don't need it. - if [ -z "${HOST_PROJECT_PATH:-}" ]; then - echo "โŒ ERROR: HOST_PROJECT_PATH environment variable is not set" - echo "This should be set in .devcontainer/devcontainer.json or passed by CI" - exit 1 - fi - - echo "๐Ÿ”ง Using HOST_PROJECT_PATH: $HOST_PROJECT_PATH" - echo "๐Ÿ“ Generating Kind cluster configuration from template..." - envsubst < "$TEMPLATE_FILE" > "$CONFIG_FILE" - echo "โœ… Generated configuration:" - cat "$CONFIG_FILE" - echo "" - - echo "๐Ÿš€ Creating Kind cluster '$CLUSTER_NAME' with audit webhook support..." - if create_primary_cluster; then - echo "โœ… Kind cluster created successfully" - elif is_known_kind_bootstrap_flake && run_dood_self_heal; then - echo "โœ… Kind cluster self-healed successfully" - else - echo "โŒ Kind cluster creation failed." - echo "๐Ÿ“„ See log: $KIND_CREATE_LOG_FILE" - exit 1 - fi -fi - -echo "๐Ÿ“‹ Configuring kubeconfig for cluster '$CLUSTER_NAME'..." -kind export kubeconfig --name "$CLUSTER_NAME" - -current_cluster_name="$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')" -current_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" - -if [[ "$current_server" =~ ^https://(127\.0\.0\.1|localhost|0\.0\.0\.0):([0-9]+)$ ]]; then - apiserver_port="${BASH_REMATCH[2]}" - if getent hosts host.docker.internal >/dev/null 2>&1; then - echo "๐Ÿ” Rewriting kubeconfig server endpoint to host.docker.internal:${apiserver_port}..." - kubectl config set-cluster "$current_cluster_name" \ - --server="https://host.docker.internal:${apiserver_port}" \ - --tls-server-name=localhost >/dev/null - echo "โœ… kubeconfig endpoint updated for devcontainer networking" - else - echo "โ„น๏ธ host.docker.internal not resolvable; keeping server as ${current_server} (--network host or native environment)" - fi -else - echo "โ„น๏ธ kubeconfig server is '$current_server' (no rewrite needed)" -fi - -echo "โœ… Cluster setup complete!" diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go new file mode 100644 index 000000000..c76ac05ff --- /dev/null +++ b/test/e2e/kubectl.go @@ -0,0 +1,126 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + "os/exec" + "strings" + + "github.com/ConfigButler/gitops-reverser/test/utils" +) + +var e2eKubectlContext string //nolint:gochecknoglobals // set once in suite, used across e2e helpers + +const kubectlNamespaceArgCount = 2 + +func setE2EKubectlContext(ctx string) { + e2eKubectlContext = strings.TrimSpace(ctx) +} + +func kubectlContext() string { + if value := strings.TrimSpace(e2eKubectlContext); value != "" { + return value + } + + // Keep compatibility with existing Makefile/shell usage. + if value := strings.TrimSpace(os.Getenv("CTX")); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("KUBE_CONTEXT")); value != "" { + return value + } + + return "" +} + +func kubectlArgs(args ...string) []string { + ctx := kubectlContext() + if ctx == "" { + return args + } + return append([]string{"--context", ctx}, args...) +} + +func kubectlArgsInNamespace(namespace string, args ...string) []string { + if strings.TrimSpace(namespace) == "" { + return kubectlArgs(args...) + } + + withCtx := kubectlArgs(args...) + if hasNamespaceArg(withCtx) { + return withCtx + } + + // Ensure we don't append namespace after a "--" separator (e.g., `kubectl exec ... -- ...`), + // since anything after "--" is passed to the executed command. + doubleDashIdx := -1 + for i, arg := range withCtx { + if arg == "--" { + doubleDashIdx = i + break + } + } + if doubleDashIdx == -1 { + return append(withCtx, "-n", namespace) + } + + updated := make([]string, 0, len(withCtx)+kubectlNamespaceArgCount) + updated = append(updated, withCtx[:doubleDashIdx]...) + updated = append(updated, "-n", namespace) + updated = append(updated, withCtx[doubleDashIdx:]...) + return updated +} + +func hasNamespaceArg(args []string) bool { + for _, arg := range args { + if arg == "-n" || arg == "--namespace" { + return true + } + if strings.HasPrefix(arg, "--namespace=") { + return true + } + } + return false +} + +func kubectlCmd(ctx context.Context, args ...string) *exec.Cmd { + //nolint:gosec // e2e helper intentionally shells out to kubectl + return exec.CommandContext(ctx, "kubectl", kubectlArgs(args...)...) +} + +func kubectlCmdInNamespace(ctx context.Context, namespace string, args ...string) *exec.Cmd { + //nolint:gosec // e2e helper intentionally shells out to kubectl + return exec.CommandContext(ctx, "kubectl", kubectlArgsInNamespace(namespace, args...)...) +} + +func kubectlRun(args ...string) (string, error) { + return utils.Run(kubectlCmd(context.Background(), args...)) +} + +func kubectlRunInNamespace(namespace string, args ...string) (string, error) { + return utils.Run(kubectlCmdInNamespace(context.Background(), namespace, args...)) +} + +func kubectlRunWithStdin(namespace, stdin string, args ...string) (string, error) { + cmd := kubectlCmdInNamespace(context.Background(), namespace, args...) + cmd.Stdin = strings.NewReader(stdin) + return utils.Run(cmd) +} diff --git a/test/e2e/namespace.go b/test/e2e/namespace.go new file mode 100644 index 000000000..145e0b347 --- /dev/null +++ b/test/e2e/namespace.go @@ -0,0 +1,33 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os" + "strings" +) + +const defaultE2ENamespace = "sut" + +func resolveE2ENamespace() string { + if value := strings.TrimSpace(os.Getenv("NAMESPACE")); value != "" { + return value + } + return defaultE2ENamespace +} diff --git a/test/e2e/quickstart_framework_e2e_test.go b/test/e2e/quickstart_framework_e2e_test.go new file mode 100644 index 000000000..710f60011 --- /dev/null +++ b/test/e2e/quickstart_framework_e2e_test.go @@ -0,0 +1,506 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ConfigButler/gitops-reverser/test/utils" +) + +const ( + quickstartFrameworkEnabledEnv = "E2E_ENABLE_QUICKSTART_FRAMEWORK" + quickstartFrameworkModeEnv = "E2E_QUICKSTART_MODE" + quickstartSetupNamespaceEnv = "QUICKSTART_NAMESPACE" + quickstartTimeoutSecondsEnv = "QUICKSTART_TIMEOUT_SECONDS" +) + +type quickstartFrameworkRun struct { + mode string + testID string + repoName string + checkoutDir string + repoURL string + providerName string + targetName string + watchRuleName string + invalidProvName string + encryptionName string +} + +var _ = Describe("Quickstart Framework", Label("quickstart-framework"), Ordered, func() { + var run quickstartFrameworkRun + + BeforeAll(func() { + if !quickstartFrameworkEnabled() { + Skip(fmt.Sprintf( + "quickstart framework is disabled; set %s=true to run", + quickstartFrameworkEnabledEnv, + )) + } + + run = newQuickstartFrameworkRun() + }) + + It("sets up quickstart flow via Go framework", func() { + By("creating dedicated Gitea repository and bootstrap credentials") + run.setupGiteaRepository() + + By("applying quickstart resources from Go") + run.applyQuickstartResources() + + By("verifying quickstart resources become Ready") + run.verifyQuickstartResourcesReady() + + By("verifying generated encryption secret and commit flow") + generatedAgeKey := run.verifyGeneratedEncryptionSecret() + + By("verifying quickstart commits for create/update/delete") + run.verifyQuickstartConfigMapCommits() + + By("verifying quickstart encrypted Secret commit is decryptable") + run.verifyQuickstartSecretEncryption(generatedAgeKey) + + By("verifying invalid credentials provider shows actionable message") + run.verifyInvalidProviderActionableMessage() + }) + +}) + +func quickstartFrameworkEnabled() bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(quickstartFrameworkEnabledEnv))) + return value == "1" || value == "true" || value == "yes" +} + +func quickstartFrameworkMode() string { + mode := strings.ToLower(strings.TrimSpace(os.Getenv(quickstartFrameworkModeEnv))) + if mode == "" { + return "helm" + } + + Expect(mode == "config-dir" || mode == "helm" || mode == "plain-manifests-file").To( + BeTrue(), + fmt.Sprintf( + "unsupported %s value %q (expected config-dir|helm|plain-manifests-file)", + quickstartFrameworkModeEnv, + mode, + ), + ) + return mode +} + +func newQuickstartFrameworkRun() quickstartFrameworkRun { + mode := quickstartFrameworkMode() + testID := strconv.FormatInt(time.Now().UnixNano(), 10) + repoName := fmt.Sprintf("quickstart-framework-%s-%s", mode, testID) + checkoutDir := filepath.Join("/tmp/gitops-reverser", repoName) + + return quickstartFrameworkRun{ + mode: mode, + testID: testID, + repoName: repoName, + checkoutDir: checkoutDir, + repoURL: fmt.Sprintf("http://gitea-http.gitea-e2e.svc.cluster.local:13000/testorg/%s.git", repoName), + providerName: fmt.Sprintf("quickstart-provider-%s", testID), + targetName: fmt.Sprintf("quickstart-target-%s", testID), + watchRuleName: fmt.Sprintf("quickstart-watchrule-%s", testID), + invalidProvName: fmt.Sprintf("quickstart-invalid-provider-%s", testID), + encryptionName: fmt.Sprintf("quickstart-sops-age-key-%s", testID), + } +} + +func (r *quickstartFrameworkRun) setupGiteaRepository() { + cmd := exec.Command("bash", "test/e2e/scripts/setup-gitea.sh", r.repoName, r.checkoutDir) + projectDir, err := utils.GetProjectDir() + Expect(err).NotTo(HaveOccurred(), "failed to resolve project directory for quickstart") + + cmd.Dir = projectDir + cmd.Env = append( + os.Environ(), + "GO111MODULE=on", + fmt.Sprintf("SUT_NAMESPACE=%s", quickstartSetupNamespace()), + ) + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to bootstrap gitea repository for quickstart: %s", output)) +} + +func (r *quickstartFrameworkRun) applyQuickstartResources() { + qsNamespace := quickstartSetupNamespace() + createGitProviderWithURLInNamespace(r.providerName, qsNamespace, "main", "git-creds", r.repoURL) + + createGitTargetWithEncryptionOptions( + r.targetName, + qsNamespace, + r.providerName, + "live-cluster", + "main", + r.encryptionName, + true, + ) + + watchRuleData := struct { + Name string + Namespace string + DestinationName string + }{ + Name: r.watchRuleName, + Namespace: qsNamespace, + DestinationName: r.targetName, + } + + err := applyFromTemplate("test/e2e/templates/watchrule.tmpl", watchRuleData, qsNamespace) + Expect(err).NotTo(HaveOccurred(), "failed to apply quickstart watchrule") + + createGitProviderWithURLInNamespace(r.invalidProvName, qsNamespace, "main", "git-creds-invalid", r.repoURL) +} + +func (r *quickstartFrameworkRun) verifyQuickstartResourcesReady() { + ns := quickstartSetupNamespace() + verifyResourceStatus("gitprovider", r.providerName, ns, "True", "Ready", "") + verifyResourceStatus("gittarget", r.targetName, ns, "True", "Ready", "") + verifyResourceStatus("watchrule", r.watchRuleName, ns, "True", "Ready", "") +} + +func (r *quickstartFrameworkRun) verifyGeneratedEncryptionSecret() string { + ns := quickstartSetupNamespace() + var generatedAgeKey string + + Eventually(func(g Gomega) { + output, err := kubectlRunInNamespace(ns, "get", "secret", r.encryptionName, "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + + var secretObj map[string]interface{} + g.Expect(json.Unmarshal([]byte(output), &secretObj)).To(Succeed()) + + annotations, _, annoErr := unstructured.NestedStringMap(secretObj, "metadata", "annotations") + g.Expect(annoErr).NotTo(HaveOccurred()) + g.Expect(annotations).To(HaveKeyWithValue("configbutler.ai/backup-warning", "REMOVE_AFTER_BACKUP")) + g.Expect(annotations).To(HaveKey("configbutler.ai/age-recipient")) + g.Expect(annotations["configbutler.ai/age-recipient"]).To(HavePrefix("age1")) + + secretData, found, keyErr := unstructured.NestedStringMap(secretObj, "data") + g.Expect(keyErr).NotTo(HaveOccurred()) + g.Expect(found).To(BeTrue()) + + var sopsAgeKeyB64 string + for key, value := range secretData { + if strings.HasSuffix(key, ".agekey") { + sopsAgeKeyB64 = value + break + } + } + g.Expect(sopsAgeKeyB64).NotTo(BeEmpty()) + + keyBytes, decodeErr := base64.StdEncoding.DecodeString(sopsAgeKeyB64) + g.Expect(decodeErr).NotTo(HaveOccurred()) + generatedAgeKey = strings.TrimSpace(string(keyBytes)) + g.Expect(generatedAgeKey).To(HavePrefix("AGE-SECRET-KEY-")) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + return generatedAgeKey +} + +func (r *quickstartFrameworkRun) verifyQuickstartConfigMapCommits() { + ns := quickstartSetupNamespace() + configMapName := fmt.Sprintf("quickstart-config-%s", r.testID) + expectedFile := filepath.Join( + r.checkoutDir, + "live-cluster", + "v1", + "configmaps", + ns, + fmt.Sprintf("%s.yaml", configMapName), + ) + + _, _ = kubectlRunInNamespace(ns, "delete", "configmap", configMapName, "--ignore-not-found=true") + + commitsBefore, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = kubectlRunInNamespace( + ns, + "create", "configmap", + configMapName, + "--from-literal=value=one", + ) + Expect(err).NotTo(HaveOccurred(), "failed to create quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("value: one")) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsBefore)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + commitsAfterCreate, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = kubectlRunInNamespace( + ns, + "patch", "configmap", + configMapName, + "--type", + "merge", + "--patch", + `{"data":{"value":"two"}}`, + ) + Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("ConfigMap file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("value: two")) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsAfterCreate)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + commitsAfterUpdate, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, err = kubectlRunInNamespace( + ns, + "delete", "configmap", + configMapName, + ) + Expect(err).NotTo(HaveOccurred(), "failed to delete quickstart ConfigMap") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred()) + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), fmt.Sprintf("ConfigMap file should be deleted: %s", expectedFile)) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsAfterUpdate)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) +} + +func (r *quickstartFrameworkRun) verifyQuickstartSecretEncryption(generatedAgeKey string) { + ns := quickstartSetupNamespace() + secretName := fmt.Sprintf("quickstart-secret-%s", r.testID) + secretValueOne := fmt.Sprintf("quickstart-plaintext-one-%s", r.testID) + secretValueTwo := fmt.Sprintf("quickstart-plaintext-two-%s", r.testID) + secretValueOneB64 := base64.StdEncoding.EncodeToString([]byte(secretValueOne)) + secretValueTwoB64 := base64.StdEncoding.EncodeToString([]byte(secretValueTwo)) + + expectedFile := filepath.Join( + r.checkoutDir, + "live-cluster", + "v1", + "secrets", + ns, + fmt.Sprintf("%s.sops.yaml", secretName), + ) + + commitsBefore, err := r.gitCommitCount() + Expect(err).NotTo(HaveOccurred()) + + _, _ = kubectlRunInNamespace( + ns, + "delete", "secret", + secretName, + "--ignore-not-found=true", + ) + + _, err = kubectlRunInNamespace( + ns, + "create", "secret", "generic", + secretName, + "--from-literal", + fmt.Sprintf("password=%s", secretValueOne), + ) + Expect(err).NotTo(HaveOccurred(), "failed to create quickstart Secret") + + _, err = kubectlRunInNamespace( + ns, + "patch", "secret", + secretName, + "--type", + "merge", + "--patch", + fmt.Sprintf(`{"stringData":{"password":"%s"}}`, secretValueTwo), + ) + Expect(err).NotTo(HaveOccurred(), "failed to patch quickstart Secret") + + Eventually(func(g Gomega) { + g.Expect(r.gitPull()).To(Succeed()) + + content, readErr := os.ReadFile(expectedFile) + g.Expect(readErr).NotTo(HaveOccurred(), fmt.Sprintf("Secret file must exist at %s", expectedFile)) + g.Expect(string(content)).To(ContainSubstring("sops:")) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueOne)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueTwo)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueOneB64)) + g.Expect(string(content)).NotTo(ContainSubstring(secretValueTwoB64)) + + decryptedOutput, decryptErr := decryptWithControllerSOPS(content, generatedAgeKey) + g.Expect(decryptErr).NotTo(HaveOccurred(), "failed to decrypt committed secret via controller sops binary") + g.Expect(decryptedOutput).To(ContainSubstring(secretValueTwoB64)) + + commitsAfter, countErr := r.gitCommitCount() + g.Expect(countErr).NotTo(HaveOccurred()) + g.Expect(commitsAfter).To(BeNumerically(">", commitsBefore)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) + + _, _ = kubectlRunInNamespace( + ns, + "delete", "secret", + secretName, + "--ignore-not-found=true", + ) +} + +func (r *quickstartFrameworkRun) verifyInvalidProviderActionableMessage() { + ns := quickstartSetupNamespace() + Eventually(func(g Gomega) { + output, err := kubectlRunInNamespace(ns, "get", "gitprovider", r.invalidProvName, "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + + var obj unstructured.Unstructured + g.Expect(json.Unmarshal([]byte(output), &obj)).To(Succeed()) + + conditions, found, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + g.Expect(found).To(BeTrue(), "status.conditions not found") + + var readyStatus, readyReason, readyMessage string + for _, cond := range conditions { + condMap, ok := cond.(map[string]interface{}) + if !ok { + continue + } + if condMap["type"] == "Ready" { + readyStatus, _ = condMap["status"].(string) + readyReason, _ = condMap["reason"].(string) + readyMessage, _ = condMap["message"].(string) + break + } + } + + g.Expect(readyStatus).To(Equal("False")) + g.Expect(readyReason).To(Equal("ConnectionFailed")) + g.Expect(strings.TrimSpace(readyMessage)).NotTo(BeEmpty()) + + message := strings.ToLower(readyMessage) + actionable := strings.Contains(message, "auth") || + strings.Contains(message, "credential") || + strings.Contains(message, "connect") || + strings.Contains(message, "repository") || + strings.Contains(message, "secret") + g.Expect(actionable).To(BeTrue(), fmt.Sprintf("expected actionable message, got: %q", readyMessage)) + }, quickstartTimeout(), 2*time.Second).Should(Succeed()) +} + +func (r *quickstartFrameworkRun) gitPull() error { + pullCmd := exec.Command("git", "pull", "--ff-only") + pullCmd.Dir = r.checkoutDir + output, err := pullCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("git pull: %w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func (r *quickstartFrameworkRun) gitCommitCount() (int, error) { + cmd := exec.Command("git", "rev-list", "--count", "--all") + cmd.Dir = r.checkoutDir + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("git rev-list --count: %w: %s", err, strings.TrimSpace(string(output))) + } + + value := strings.TrimSpace(string(output)) + if value == "" { + return 0, nil + } + + count, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("parse git commit count %q: %w", value, err) + } + return count, nil +} + +func quickstartTimeout() time.Duration { + value := strings.TrimSpace(os.Getenv(quickstartTimeoutSecondsEnv)) + if value == "" { + return 180 * time.Second + } + + seconds, err := strconv.Atoi(value) + if err != nil || seconds <= 0 { + return 180 * time.Second + } + + return time.Duration(seconds) * time.Second +} + +func createGitProviderWithURLInNamespace(name, ns, branch, secretName, repoURL string) { + By(fmt.Sprintf("creating GitProvider '%s' in ns '%s' with branch '%s', secret '%s' and URL '%s'", + name, ns, branch, secretName, repoURL)) + + data := struct { + Name string + Namespace string + RepoURL string + Branch string + SecretName string + }{ + Name: name, + Namespace: ns, + RepoURL: repoURL, + Branch: branch, + SecretName: secretName, + } + + err := applyFromTemplate("test/e2e/templates/gitprovider.tmpl", data, ns) + Expect(err).NotTo(HaveOccurred(), "failed to apply GitProvider") +} + +func quickstartSetupNamespace() string { + if value := strings.TrimSpace(os.Getenv(quickstartSetupNamespaceEnv)); value != "" { + return value + } + if value := strings.TrimSpace(os.Getenv("SUT_NAMESPACE")); value != "" { + return value + } + return "sut" +} diff --git a/test/e2e/scripts/ensure-prometheus-operator.sh b/test/e2e/scripts/ensure-prometheus-operator.sh index a94b5391e..819cd1d52 100755 --- a/test/e2e/scripts/ensure-prometheus-operator.sh +++ b/test/e2e/scripts/ensure-prometheus-operator.sh @@ -3,11 +3,21 @@ set -euo pipefail PROM_OPERATOR_VERSION="${PROM_OPERATOR_VERSION:-0.89.0}" PROM_OPERATOR_NAMESPACE="${PROM_OPERATOR_NAMESPACE:-prometheus-operator}" +KUBE_CONTEXT="${KUBECONTEXT:-${CTX:-}}" + +if [[ -z "${KUBE_CONTEXT}" ]]; then + KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" +fi + +if [[ -z "${KUBE_CONTEXT}" ]]; then + echo "โŒ Kubernetes context is required (set KUBECONTEXT or CTX)" + exit 1 +fi is_fully_installed() { - kubectl get deployment prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" >/dev/null 2>&1 && - kubectl get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1 && - kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 + kubectl --context "${KUBE_CONTEXT}" get deployment prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" >/dev/null 2>&1 && + kubectl --context "${KUBE_CONTEXT}" get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1 && + kubectl --context "${KUBE_CONTEXT}" get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 } if is_fully_installed; then @@ -16,7 +26,8 @@ if is_fully_installed; then fi echo "Installing Prometheus Operator v${PROM_OPERATOR_VERSION} in namespace ${PROM_OPERATOR_NAMESPACE}..." -kubectl create namespace "${PROM_OPERATOR_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl --context "${KUBE_CONTEXT}" create namespace "${PROM_OPERATOR_NAMESPACE}" --dry-run=client -o yaml | \ + kubectl --context "${KUBE_CONTEXT}" apply -f - tmp_dir="$(mktemp -d)" cleanup() { @@ -32,12 +43,13 @@ EOF ( cd "${tmp_dir}" NAMESPACE="${PROM_OPERATOR_NAMESPACE}" kustomize edit set namespace "${PROM_OPERATOR_NAMESPACE}" - if ! kubectl create -k .; then + if ! kubectl --context "${KUBE_CONTEXT}" create -k .; then echo "kubectl create -k failed (likely already exists), reconciling with server-side apply ..." - kubectl apply --server-side -k . + kubectl --context "${KUBE_CONTEXT}" apply --server-side -k . fi ) -kubectl rollout status deployment/prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" --timeout=180s +kubectl --context "${KUBE_CONTEXT}" rollout status deployment/prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" \ + --timeout=180s echo "โœ… Prometheus Operator is ready" diff --git a/test/e2e/scripts/run-quickstart.sh b/test/e2e/scripts/run-quickstart.sh deleted file mode 100755 index 1887ec720..000000000 --- a/test/e2e/scripts/run-quickstart.sh +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MODE="${1:-}" -NAMESPACE="gitops-reverser" -QUICKSTART_NAMESPACE="sut" -HELM_CHART_SOURCE="${HELM_CHART_SOURCE:-charts/gitops-reverser}" -WAIT_TIMEOUT="${WAIT_TIMEOUT:-60s}" -QUICKSTART_TIMEOUT_SECONDS="${QUICKSTART_TIMEOUT_SECONDS:-180}" -PROJECT_IMAGE="${PROJECT_IMAGE:-}" - -TEST_ID="$(date +%s)-$RANDOM" -REPO_NAME="quickstart-smoke-${MODE}-${TEST_ID}" -CHECKOUT_DIR="/tmp/gitops-reverser/${REPO_NAME}" - -GIT_PROVIDER_NAME="quickstart-provider-${TEST_ID}" -GIT_TARGET_NAME="quickstart-target-${TEST_ID}" -WATCHRULE_NAME="quickstart-watchrule-${TEST_ID}" -INVALID_PROVIDER_NAME="quickstart-invalid-provider-${TEST_ID}" -ENCRYPTION_SECRET_NAME="quickstart-sops-age-key-${TEST_ID}" -CONFIGMAP_NAME="quickstart-config-${TEST_ID}" -EXPECTED_CONFIGMAP_FILE="${CHECKOUT_DIR}/live-cluster/v1/configmaps/${QUICKSTART_NAMESPACE}/${CONFIGMAP_NAME}.yaml" -SECRET_NAME="quickstart-secret-${TEST_ID}" -SECRET_VALUE_ONE="quickstart-plaintext-one-${TEST_ID}" -SECRET_VALUE_TWO="quickstart-plaintext-two-${TEST_ID}" -EXPECTED_SECRET_FILE="${CHECKOUT_DIR}/live-cluster/v1/secrets/${QUICKSTART_NAMESPACE}/${SECRET_NAME}.sops.yaml" -GITEA_PORT_FORWARD_PID="" - -if [[ -z "${MODE}" ]]; then - echo "usage: $0 " - exit 1 -fi - -get_controller_pod_selector() { - local selector - selector="$(kubectl -n "${NAMESPACE}" get deployment gitops-reverser \ - -o jsonpath='{range $k,$v := .spec.selector.matchLabels}{$k}={$v},{end}' 2>/dev/null || true)" - selector="${selector%,}" - - if [[ -z "${selector}" ]]; then - selector="app.kubernetes.io/name=gitops-reverser" - fi - - printf '%s' "${selector}" -} - -install_helm() { - local helm_image_args=() - local helm_crd_args=() - - if [[ -n "${PROJECT_IMAGE}" ]]; then - local image_no_digest image_repo image_tag - image_no_digest="${PROJECT_IMAGE%%@*}" - if [[ "${image_no_digest##*/}" == *:* ]]; then - image_repo="${image_no_digest%:*}" - image_tag="${image_no_digest##*:}" - else - image_repo="${image_no_digest}" - image_tag="latest" - fi - helm_image_args+=(--set "image.repository=${image_repo}" --set "image.tag=${image_tag}") - echo "Overriding chart image from PROJECT_IMAGE=${PROJECT_IMAGE}" - fi - - if kubectl get crd gitproviders.configbutler.ai gittargets.configbutler.ai \ - watchrules.configbutler.ai clusterwatchrules.configbutler.ai >/dev/null 2>&1; then - helm_crd_args+=(--skip-crds) - echo "Detected existing CRDs in cluster; installing Helm release with --skip-crds" - fi - - echo "Installing from Helm chart (mode=helm, source=${HELM_CHART_SOURCE})" - helm upgrade --install "name-is-cool-but-not-relevant" "${HELM_CHART_SOURCE}" \ - --namespace "${NAMESPACE}" \ - --create-namespace \ - --set fullnameOverride=gitops-reverser \ - "${helm_crd_args[@]}" \ - "${helm_image_args[@]}" -} - -install_manifest() { - echo "Installing from generated dist/install.yaml (mode=manifest)" - kubectl apply -f dist/install.yaml - - if [[ -n "${PROJECT_IMAGE}" ]]; then - echo "Overriding manifest deployment image from PROJECT_IMAGE=${PROJECT_IMAGE}" - kubectl -n "${NAMESPACE}" set image deployment/gitops-reverser manager="${PROJECT_IMAGE}" - fi -} - -print_debug_info() { - local pod_selector - pod_selector="$(get_controller_pod_selector)" - - echo - echo "Install quickstart smoke diagnostics (${MODE})" - echo "Namespace: ${NAMESPACE}" - echo "Quickstart namespace: ${QUICKSTART_NAMESPACE}" - echo "Pod selector: ${pod_selector}" - echo - echo "Deployment status:" - kubectl -n "${NAMESPACE}" get deployment gitops-reverser -o wide || true - echo - echo "Deployment describe:" - kubectl -n "${NAMESPACE}" describe deployment gitops-reverser || true - echo - echo "Pods:" - kubectl -n "${NAMESPACE}" get pods -o wide || true - echo - echo "Controller-manager pod describe:" - kubectl -n "${NAMESPACE}" describe pod -l "${pod_selector}" || true - echo - echo "Controller-manager logs (last 200 lines):" - kubectl -n "${NAMESPACE}" logs -l "${pod_selector}" --tail=200 --all-containers=true || true - echo - echo "Quickstart resources:" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider,gittarget,watchrule \ - "${GIT_PROVIDER_NAME}" "${GIT_TARGET_NAME}" "${WATCHRULE_NAME}" "${INVALID_PROVIDER_NAME}" 2>/dev/null || true - echo - echo "Quickstart resource status dumps:" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${GIT_PROVIDER_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get gittarget "${GIT_TARGET_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get watchrule "${WATCHRULE_NAME}" -o yaml 2>/dev/null || true - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${INVALID_PROVIDER_NAME}" -o yaml 2>/dev/null || true - echo - echo "Recent namespace events (${NAMESPACE}):" - kubectl -n "${NAMESPACE}" get events --sort-by=.metadata.creationTimestamp | tail -n 50 || true - echo - echo "Recent namespace events (${QUICKSTART_NAMESPACE}):" - kubectl -n "${QUICKSTART_NAMESPACE}" get events --sort-by=.metadata.creationTimestamp | tail -n 50 || true - echo - echo "Git checkout state (${CHECKOUT_DIR}):" - git -C "${CHECKOUT_DIR}" status --short 2>/dev/null || true - git -C "${CHECKOUT_DIR}" --no-pager log --oneline -n 10 2>/dev/null || true -} - -run_or_debug() { - local description="$1" - shift - echo "${description}" - if ! "$@"; then - echo "FAILED: ${description}" >&2 - print_debug_info - return 1 - fi -} - -cleanup_gitea_port_forward() { - if [[ -n "${GITEA_PORT_FORWARD_PID}" ]]; then - kill "${GITEA_PORT_FORWARD_PID}" >/dev/null 2>&1 || true - wait "${GITEA_PORT_FORWARD_PID}" >/dev/null 2>&1 || true - GITEA_PORT_FORWARD_PID="" - fi -} - -ensure_gitea_api_port_forward() { - cleanup_gitea_port_forward - pkill -f "kubectl.*port-forward.*13000:13000" >/dev/null 2>&1 || true - - kubectl -n gitea-e2e port-forward svc/gitea-http 13000:13000 >/tmp/gitea-port-forward.log 2>&1 & - GITEA_PORT_FORWARD_PID="$!" - trap cleanup_gitea_port_forward EXIT - - for _ in {1..30}; do - if curl -s -f "http://localhost:13000/api/v1/version" >/dev/null 2>&1; then - echo "Gitea API port-forward is ready on localhost:13000" - return 0 - fi - sleep 2 - done - - echo "Timed out waiting for Gitea API port-forward readiness" - return 1 -} - -reset_install_state() { - echo "Resetting install state for clean first-time install validation" - kubectl delete clusterrole \ - gitops-reverser \ - gitops-reverser-metrics-reader \ - gitops-reverser-proxy-role \ - --ignore-not-found=true >/dev/null || true - kubectl delete clusterrolebinding \ - gitops-reverser \ - gitops-reverser-proxy-rolebinding \ - --ignore-not-found=true >/dev/null || true - kubectl delete validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration \ - --ignore-not-found=true >/dev/null || true - kubectl delete namespace "${NAMESPACE}" --wait=true --ignore-not-found=true >/dev/null || true -} - -verify_installation() { - local pod_selector - pod_selector="$(get_controller_pod_selector)" - - run_or_debug \ - "Waiting for deployment rollout (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" rollout status deployment/gitops-reverser --timeout="${WAIT_TIMEOUT}" - - run_or_debug \ - "Checking deployment availability (timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" wait --for=condition=available deployment/gitops-reverser --timeout="${WAIT_TIMEOUT}" - - run_or_debug \ - "Checking pod readiness (selector=${pod_selector}, timeout=${WAIT_TIMEOUT})" \ - kubectl -n "${NAMESPACE}" wait --for=condition=ready pod -l "${pod_selector}" --timeout="${WAIT_TIMEOUT}" - - echo "Checking CRDs" - kubectl get crd \ - gitproviders.configbutler.ai \ - gittargets.configbutler.ai \ - watchrules.configbutler.ai \ - clusterwatchrules.configbutler.ai >/dev/null - - echo "Checking validating webhook configuration" - kubectl get validatingwebhookconfiguration gitops-reverser-validating-webhook-configuration >/dev/null -} - -wait_for_ready() { - local kind="$1" - local name="$2" - local namespace="$3" - local timeout_seconds="${4}" - local start_epoch now status reason message - start_epoch="$(date +%s)" - - while true; do - status="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - reason="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - message="$(kubectl -n "${namespace}" get "${kind}" "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" - - if [[ "${status}" == "True" ]]; then - echo "${kind}/${name} Ready=True (${reason})" - return 0 - fi - - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for ${kind}/${name} Ready=True (status=${status} reason=${reason})" - echo "Last message: ${message}" - kubectl -n "${namespace}" get "${kind}" "${name}" -o yaml || true - return 1 - fi - sleep 2 - done -} - -wait_for_connection_failed_actionable_message() { - local name="$1" - local timeout_seconds="${2}" - local start_epoch now status reason message - start_epoch="$(date +%s)" - - while true; do - status="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - reason="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - message="$(kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" - - if [[ "${status}" == "False" && "${reason}" == "ConnectionFailed" && -n "${message}" ]]; then - shopt -s nocasematch - if [[ "${message}" =~ auth|credential|connect|repository|secret ]]; then - shopt -u nocasematch - echo "gitprovider/${name} exposes actionable failure message" - return 0 - fi - shopt -u nocasematch - fi - - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for actionable ConnectionFailed message on gitprovider/${name}" - echo "Last seen status=${status} reason=${reason} message=${message}" - kubectl -n "${QUICKSTART_NAMESPACE}" get gitprovider "${name}" -o yaml || true - return 1 - fi - sleep 2 - done -} - -git_commit_count() { - git -C "${CHECKOUT_DIR}" rev-list --count --all 2>/dev/null || echo 0 -} - -wait_for_file_exists() { - local file_path="$1" - local timeout_seconds="$2" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]]; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file to exist: ${file_path}" - return 1 - fi - sleep 2 - done -} - -wait_for_file_contains() { - local file_path="$1" - local expected_text="$2" - local timeout_seconds="$3" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]] && grep -Fq "${expected_text}" "${file_path}"; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file content: ${file_path} to contain '${expected_text}'" - return 1 - fi - sleep 2 - done -} - -wait_for_file_absent() { - local file_path="$1" - local timeout_seconds="$2" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ ! -f "${file_path}" ]]; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file to be deleted: ${file_path}" - return 1 - fi - sleep 2 - done -} - -wait_for_file_not_contains() { - local file_path="$1" - local unexpected_text="$2" - local timeout_seconds="$3" - local start_epoch now - start_epoch="$(date +%s)" - - while true; do - git -C "${CHECKOUT_DIR}" pull --ff-only >/dev/null 2>&1 || true - if [[ -f "${file_path}" ]] && ! grep -Fq "${unexpected_text}" "${file_path}"; then - return 0 - fi - now="$(date +%s)" - if (( now - start_epoch >= timeout_seconds )); then - echo "Timed out waiting for file content: ${file_path} to NOT contain '${unexpected_text}'" - return 1 - fi - sleep 2 - done -} - -extract_generated_age_key() { - local secret_name="$1" - local key_data age_key - - key_data="$( - kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${secret_name}" \ - -o go-template='{{ range $k, $v := .data }}{{ printf "%s=%s\n" $k $v }}{{ end }}' 2>/dev/null | - grep '\.agekey=' | head -n1 | cut -d'=' -f2- || true - )" - if [[ -z "${key_data}" ]]; then - echo "No .agekey entry found in secret ${QUICKSTART_NAMESPACE}/${secret_name}" - return 1 - fi - - age_key="$(printf '%s' "${key_data}" | base64 -d 2>/dev/null | awk '/AGE-SECRET-KEY-/{print; exit}')" - if [[ -z "${age_key}" ]]; then - echo "Failed to decode AGE-SECRET-KEY from secret ${QUICKSTART_NAMESPACE}/${secret_name}" - return 1 - fi - - printf '%s' "${age_key}" -} - -decrypt_file_with_controller_sops() { - local file_path="$1" - local age_key="$2" - local pod_selector pod_name - - pod_selector="$(get_controller_pod_selector)" - pod_name="$( - kubectl -n "${NAMESPACE}" get pod -l "${pod_selector}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true - )" - if [[ -z "${pod_name}" ]]; then - echo "Failed to find controller pod in namespace ${NAMESPACE}" - return 1 - fi - - kubectl -n "${NAMESPACE}" exec -i "${pod_name}" -- \ - env "SOPS_AGE_KEY=${age_key}" \ - /usr/local/bin/sops --decrypt --input-type yaml --output-type yaml /dev/stdin < "${file_path}" -} - -run_quickstart_flow() { - ensure_gitea_api_port_forward - - echo "Setting up quickstart repo and credentials" - run_or_debug \ - "Creating dedicated Gitea repo for quickstart smoke (${REPO_NAME})" \ - bash test/e2e/scripts/setup-gitea.sh "${REPO_NAME}" "${CHECKOUT_DIR}" - - echo "Applying minimal GitProvider/GitTarget/WatchRule quickstart resources" - cat </dev/null - local warning_anno recipient_anno - warning_anno="$(kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${ENCRYPTION_SECRET_NAME}" \ - -o jsonpath='{.metadata.annotations.configbutler\.ai/backup-warning}')" - recipient_anno="$(kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${ENCRYPTION_SECRET_NAME}" \ - -o jsonpath='{.metadata.annotations.configbutler\.ai/age-recipient}')" - if [[ "${warning_anno}" != "REMOVE_AFTER_BACKUP" ]]; then - echo "Expected backup warning annotation REMOVE_AFTER_BACKUP, got '${warning_anno}'" - return 1 - fi - if [[ "${recipient_anno}" != age1* ]]; then - echo "Expected age recipient annotation to start with age1, got '${recipient_anno}'" - return 1 - fi - - echo "Running create/update/delete quickstart functional checks" - local commits_before_create commits_after_create commits_after_update commits_after_delete - commits_before_create="$(git_commit_count)" - - kubectl -n "${QUICKSTART_NAMESPACE}" delete configmap "${CONFIGMAP_NAME}" --ignore-not-found=true >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" create configmap "${CONFIGMAP_NAME}" --from-literal=value=one >/dev/null - wait_for_file_exists "${EXPECTED_CONFIGMAP_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_contains "${EXPECTED_CONFIGMAP_FILE}" "value: one" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_create="$(git_commit_count)" - if (( commits_after_create <= commits_before_create )); then - echo "Expected commit count to increase after create (${commits_before_create} -> ${commits_after_create})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" patch configmap "${CONFIGMAP_NAME}" --type merge --patch '{"data":{"value":"two"}}' >/dev/null - wait_for_file_contains "${EXPECTED_CONFIGMAP_FILE}" "value: two" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_update="$(git_commit_count)" - if (( commits_after_update <= commits_after_create )); then - echo "Expected commit count to increase after update (${commits_after_create} -> ${commits_after_update})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" delete configmap "${CONFIGMAP_NAME}" --ignore-not-found=true >/dev/null - wait_for_file_absent "${EXPECTED_CONFIGMAP_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - commits_after_delete="$(git_commit_count)" - if (( commits_after_delete <= commits_after_update )); then - echo "Expected commit count to increase after delete (${commits_after_update} -> ${commits_after_delete})" - return 1 - fi - - echo "Running encrypted Secret commit check" - local commits_before_secret commits_after_secret secret_value_one_b64 secret_value_two_b64 - commits_before_secret="$(git_commit_count)" - secret_value_one_b64="$(printf '%s' "${SECRET_VALUE_ONE}" | base64 | tr -d '\n')" - secret_value_two_b64="$(printf '%s' "${SECRET_VALUE_TWO}" | base64 | tr -d '\n')" - - kubectl -n "${QUICKSTART_NAMESPACE}" delete secret "${SECRET_NAME}" --ignore-not-found=true >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" create secret generic "${SECRET_NAME}" \ - --from-literal=password="${SECRET_VALUE_ONE}" >/dev/null - kubectl -n "${QUICKSTART_NAMESPACE}" patch secret "${SECRET_NAME}" --type merge \ - --patch "{\"stringData\":{\"password\":\"${SECRET_VALUE_TWO}\"}}" >/dev/null - - wait_for_file_exists "${EXPECTED_SECRET_FILE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_contains "${EXPECTED_SECRET_FILE}" "sops:" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${SECRET_VALUE_ONE}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${SECRET_VALUE_TWO}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${secret_value_one_b64}" "${QUICKSTART_TIMEOUT_SECONDS}" - wait_for_file_not_contains "${EXPECTED_SECRET_FILE}" "${secret_value_two_b64}" "${QUICKSTART_TIMEOUT_SECONDS}" - - echo "Validating encrypted Secret is decryptable with generated key" - local generated_age_key decrypted_secret - generated_age_key="$(extract_generated_age_key "${ENCRYPTION_SECRET_NAME}")" - decrypted_secret="$(decrypt_file_with_controller_sops "${EXPECTED_SECRET_FILE}" "${generated_age_key}")" - if ! grep -Fq "${secret_value_two_b64}" <<<"${decrypted_secret}"; then - echo "Expected decrypted secret payload to contain updated value (base64) after patch" - return 1 - fi - - commits_after_secret="$(git_commit_count)" - if (( commits_after_secret <= commits_before_secret )); then - echo "Expected commit count to increase after secret write (${commits_before_secret} -> ${commits_after_secret})" - return 1 - fi - - kubectl -n "${QUICKSTART_NAMESPACE}" delete secret "${SECRET_NAME}" --ignore-not-found=true >/dev/null - - echo "Running invalid-credentials quickstart UX check" - cat </dev/null 2>&1; then @@ -49,21 +48,46 @@ test_api_connectivity() { # Function to create organization and get token setup_organization_and_token() { - echo "๐Ÿข Creating test organization '$ORG_NAME'..." - ORG_RESPONSE=$(curl -s -w "%{http_code}" -o /tmp/org_response.json \ - -X POST "$API_URL/orgs" \ - -H "Content-Type: application/json" \ - -u "$ADMIN_USER:$ADMIN_PASS" \ - -d "{\"username\":\"$ORG_NAME\",\"full_name\":\"Test Organization\",\"description\":\"E2E Test Organization\"}") + organization_exists() { + local status + status=$(curl -s -w "%{http_code}" -o /tmp/org_get_response.json \ + -X GET "$API_URL/orgs/$ORG_NAME" \ + -H "Content-Type: application/json" \ + -u "$ADMIN_USER:$ADMIN_PASS" || true) + [[ "$status" == "200" ]] + } - if [[ "$ORG_RESPONSE" == "201" ]]; then - echo "โœ… Organization created successfully" - elif [[ "$ORG_RESPONSE" == "409" ]] || [[ "$ORG_RESPONSE" == "422" ]]; then + echo "๐Ÿข Ensuring test organization '$ORG_NAME' exists..." + if organization_exists; then echo "โ„น๏ธ Organization already exists" else - echo "โš ๏ธ Unexpected response creating organization: $ORG_RESPONSE" - cat /tmp/org_response.json 2>/dev/null || true - echo "โ„น๏ธ Continuing setup despite organization creation issue..." + local attempt + for attempt in {1..5}; do + ORG_RESPONSE=$(curl -s -w "%{http_code}" -o /tmp/org_response.json \ + -X POST "$API_URL/orgs" \ + -H "Content-Type: application/json" \ + -u "$ADMIN_USER:$ADMIN_PASS" \ + -d "{\"username\":\"$ORG_NAME\",\"full_name\":\"Test Organization\",\"description\":\"E2E Test Organization\"}" || true) + + if [[ "$ORG_RESPONSE" == "201" ]]; then + echo "โœ… Organization created successfully" + break + fi + if [[ "$ORG_RESPONSE" == "409" ]] || [[ "$ORG_RESPONSE" == "422" ]]; then + echo "โ„น๏ธ Organization already exists" + break + fi + if organization_exists; then + echo "โ„น๏ธ Organization already exists" + break + fi + + echo "โš ๏ธ Attempt $attempt/5: failed to create organization (status=${ORG_RESPONSE:-none})" + cat /tmp/org_response.json 2>/dev/null || true + if [[ "$attempt" -lt 5 ]]; then + sleep 1 + fi + done fi # Generate or reuse access token @@ -349,4 +373,4 @@ echo " # Cleanup temporary files rm -f /tmp/org_response.json /tmp/repo_response.json /tmp/token_response.json /tmp/token_list.json \ - /tmp/ssh_key_response.json + /tmp/ssh_key_response.json /tmp/org_get_response.json diff --git a/test/e2e/scripts/setup-port-forwards.sh b/test/e2e/scripts/setup-port-forwards.sh index df0347f29..a4bde5373 100755 --- a/test/e2e/scripts/setup-port-forwards.sh +++ b/test/e2e/scripts/setup-port-forwards.sh @@ -5,29 +5,58 @@ set -euo pipefail GITEA_NAMESPACE=${GITEA_NAMESPACE:-gitea-e2e} PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-prometheus-operator} PROMETHEUS_INSTANCE_NAME=${PROMETHEUS_INSTANCE_NAME:-prometheus-shared-e2e} +KUBE_CONTEXT=${E2E_KUBECONTEXT:-${CTX:-${KUBECONTEXT:-}}} + +if [[ -z "${KUBE_CONTEXT}" ]]; then + KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" +fi + +if [[ -z "${KUBE_CONTEXT}" ]]; then + echo "โŒ Kubernetes context is required (set E2E_KUBECONTEXT or CTX)" + exit 1 +fi + +has_expected_forward_processes() { + local gitea_pf + local prom_pf + + gitea_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/${GITEA_SERVICE:-gitea-http}( |.* )13000:13000" | grep -v grep || true)" + prom_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/prometheus-operated( |.* )19090:9090" | grep -v grep || true)" + + [[ -n "${gitea_pf}" && -n "${prom_pf}" ]] +} + +# Fast path: keep existing healthy forwards only if they belong to this context. +if has_expected_forward_processes && \ + curl -fsS http://localhost:13000/api/healthz >/dev/null 2>&1 && \ + curl -fsS http://localhost:19090/-/healthy >/dev/null 2>&1; then + echo "โœ… Existing port-forwards are healthy for context ${KUBE_CONTEXT}; skipping restart" + exit 0 +fi # Cleanup old port-forwards echo "๐Ÿงน Cleaning up old port-forwards..." -pkill -f "kubectl port-forward.*19090" || true -pkill -f "kubectl port-forward.*13000" || true +pkill -f "kubectl.*port-forward.*19090" || true +pkill -f "kubectl.*port-forward.*13000" || true sleep 1 echo "๐Ÿ”Œ Setting up port-forwards for e2e testing..." +echo "๐ŸŽฏ Using kube context: ${KUBE_CONTEXT}" echo "โณ Waiting for Prometheus pod to be ready..." -kubectl wait --for=condition=ready pod \ +kubectl --context "$KUBE_CONTEXT" wait --for=condition=ready pod \ -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" \ -n "$PROMETHEUS_NAMESPACE" \ --timeout=180s || { echo "โŒ Prometheus pod failed to become ready" - kubectl get pods -n "$PROMETHEUS_NAMESPACE" -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" || true + kubectl --context "$KUBE_CONTEXT" get pods -n "$PROMETHEUS_NAMESPACE" -l "prometheus=${PROMETHEUS_INSTANCE_NAME}" || true exit 1 } echo "โœ… Prometheus pod is ready" echo "โณ Waiting for Gitea pod to be ready..." -kubectl wait --for=condition=ready pod \ +kubectl --context "$KUBE_CONTEXT" wait --for=condition=ready pod \ -l app.kubernetes.io/name=gitea \ -n "$GITEA_NAMESPACE" \ --timeout=180s || { @@ -47,14 +76,14 @@ setup_port_forward() { local remote_port="$5" # Check if already running - if ps aux | grep -E "kubectl.*port-forward.*${service}.*${local_port}:${remote_port}" | grep -v grep >/dev/null 2>&1; then + if ps aux | grep -E "kubectl.*--context[= ]${KUBE_CONTEXT}.*port-forward.*${service}.*${local_port}:${remote_port}" | grep -v grep >/dev/null 2>&1; then echo "โœ… ${service_name} port-forward already running" return 0 fi echo "๐Ÿ”Œ Starting ${service_name} port-forward..." local pf_log="/tmp/${service_name}-pf.log" - kubectl port-forward --address 127.0.0.1 \ + kubectl --context "$KUBE_CONTEXT" port-forward --address 127.0.0.1 \ -n "$namespace" "svc/${service}" "${local_port}:${remote_port}" > "$pf_log" 2>&1 & local pf_pid=$! @@ -94,6 +123,16 @@ setup_port_forward() { setup_port_forward "Prometheus" "$PROMETHEUS_NAMESPACE" "prometheus-operated" "19090" "9090" setup_port_forward "Gitea" "$GITEA_NAMESPACE" "gitea-http" "13000" "13000" +# Validate HTTP endpoints before returning success. +curl -fsS http://localhost:13000/api/healthz >/dev/null || { + echo "โŒ Gitea health check failed after port-forward setup" + exit 1 +} +curl -fsS http://localhost:19090/-/healthy >/dev/null || { + echo "โŒ Prometheus health check failed after port-forward setup" + exit 1 +} + echo "" echo "๐Ÿ“Š Prometheus: http://localhost:19090" echo "๐Ÿ“ Gitea: http://localhost:13000" diff --git a/test/e2e/tools/gen-age-key/main.go b/test/e2e/tools/gen-age-key/main.go index fa83b9495..a0279498b 100644 --- a/test/e2e/tools/gen-age-key/main.go +++ b/test/e2e/tools/gen-age-key/main.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "filippo.io/age" @@ -40,29 +41,27 @@ func main() { var namespace string var secretName string - flag.StringVar(&keyFile, "key-file", "", "path to write age identity file") - flag.StringVar(&secretFile, "secret-file", "", "path to write Kubernetes Secret manifest") - flag.StringVar(&namespace, "namespace", "", "Secret namespace") - flag.StringVar(&secretName, "secret-name", "", "Secret name") + flag.StringVar(&keyFile, "key-file", "", "path to write (or read, if it already exists) age identity file") + flag.StringVar(&secretFile, "secret-file", "", + "path to write Kubernetes Secret manifest (optional; requires --namespace and --secret-name)") + flag.StringVar(&namespace, "namespace", "", "Secret namespace (required with --secret-file)") + flag.StringVar(&secretName, "secret-name", "", "Secret name (required with --secret-file)") flag.Parse() - if keyFile == "" || secretFile == "" || namespace == "" || secretName == "" { - exitf("all flags are required: --key-file --secret-file --namespace --secret-name") + if keyFile == "" { + exitf("--key-file is required") + } + if secretFile != "" && (namespace == "" || secretName == "") { + exitf("--namespace and --secret-name are required when --secret-file is provided") } - identity, err := age.GenerateX25519Identity() + identity, err := loadOrGenerateIdentity(keyFile) if err != nil { - exitf("generate age identity: %v", err) + exitf("%v", err) } - keyContent := fmt.Sprintf( - "# created: %s\n# public key: %s\n%s\n", - time.Now().UTC().Format(time.RFC3339), - identity.Recipient(), - identity.String(), - ) - if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { - exitf("write key file: %v", err) + if secretFile == "" { + return } secretContent := fmt.Sprintf(`apiVersion: v1 @@ -80,6 +79,50 @@ stringData: } } +// loadOrGenerateIdentity returns the X25519Identity from keyFile if it already exists, +// or generates a fresh one and writes it to keyFile. +func loadOrGenerateIdentity(keyFile string) (*age.X25519Identity, error) { + _, statErr := os.Stat(keyFile) + if statErr != nil && !os.IsNotExist(statErr) { + return nil, fmt.Errorf("stat key file: %w", statErr) + } + if statErr == nil { + return parseIdentityFile(keyFile) + } + + // File does not exist โ€” generate a fresh identity. + identity, err := age.GenerateX25519Identity() + if err != nil { + return nil, fmt.Errorf("generate age identity: %w", err) + } + keyContent := fmt.Sprintf( + "# created: %s\n# public key: %s\n%s\n", + time.Now().UTC().Format(time.RFC3339), + identity.Recipient(), + identity.String(), + ) + if err := writeFile(keyFile, []byte(keyContent), keyFileMode); err != nil { + return nil, fmt.Errorf("write key file: %w", err) + } + return identity, nil +} + +// parseIdentityFile reads an age identity file, locates the AGE-SECRET-KEY line, +// and returns the parsed X25519Identity. +func parseIdentityFile(path string) (*age.X25519Identity, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "AGE-SECRET-KEY-") { + return age.ParseX25519Identity(line) + } + } + return nil, fmt.Errorf("no AGE-SECRET-KEY line found in %s", path) +} + func writeFile(path string, content []byte, mode os.FileMode) error { parent := filepath.Dir(path) if parent != "" && parent != "." { diff --git a/test/improving.md b/test/improving.md index d5ee5e97a..a2ab556c7 100644 --- a/test/improving.md +++ b/test/improving.md @@ -16,7 +16,8 @@ But the basics are the same, and they should be able to handle multiple gitops-r What would be needed to get that right? We need to continue our quest for removing repitiont: getting back the essence, properly defining what is really dependent on what. -The first next step would be to analyse test/e2e/scripts/run-quickstart.sh -> I'm sure a lot can be removed and can be picked up by the 'normal' stuff. +The first next step used to be analysing `test/e2e/scripts/run-quickstart.sh`, but quickstart smoke assertions now live +in Go (`test/e2e/quickstart_framework_e2e_test.go`) and the shell harness is retired. It would be really cool to create a new namespace: "run-e2e-full-{same-number-as-repo-basedupon-time}" @@ -26,4 +27,3 @@ It would be really cool to create a new namespace: I'm also wondering if the tests for the quickstart shouldnt be in a go e2e_test.go like file -> in the end we are just doing the same type of assertions. It would be really awesome if we could redo all tests in the proper way. - diff --git a/test/utils/utils.go b/test/utils/utils.go index 057d1a957..8630ce957 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -22,7 +22,6 @@ package utils import ( "bufio" "bytes" - "context" "fmt" "os" "os/exec" @@ -51,19 +50,6 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } -// LoadImageToKindClusterWithName loads a local docker image to the kind cluster. -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - ctx := context.Background() - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.CommandContext(ctx, "kind", kindOptions...) - _, err := Run(cmd) - return err -} - // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string {