diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 932ba050f..b855c3ab8 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -1,13 +1,14 @@ { "permissions": { "allow": [ - "Bash(go test:*)", - "Bash(make lint:*)", - "Bash(make test-e2e:*)", - "Bash(go build:*)", + "Bash(make:*)", "Bash(kubectl apply:*)", "Bash(kubectl wait:*)", - "Bash(chmod +x /workspaces/gitops-reverser/hack/demo/setup-holland-stuff-deployment.sh)" + "Bash(chmod +x /workspaces/gitops-reverser/hack/demo/setup-holland-stuff-deployment.sh)", + "Bash(make vet:*)", + "Bash(go:*)", + "Bash(helm template:*", + "Bash(find:*" ] } } diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3dde6f611..a65f77bf8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,13 +33,13 @@ RUN apt-get update \ # Tool versions - centralized for easy updates # This devcontainer currently supports amd64 tool downloads only. -ENV KUBECTL_VERSION=v1.35.0 \ +ENV KUBECTL_VERSION=v1.35.3 \ KUSTOMIZE_VERSION=5.8.1 \ - KUBEBUILDER_VERSION=4.11.1 \ - GOLANGCI_LINT_VERSION=v2.10.1 \ - HELM_VERSION=v4.0.0 \ + KUBEBUILDER_VERSION=4.13.1 \ + GOLANGCI_LINT_VERSION=v2.11.4 \ + HELM_VERSION=v4.1.4 \ K3D_VERSION=v5.8.3 \ - FLUX_VERSION=2.8.0 + FLUX_VERSION=2.8.5 # Fail early on unsupported architectures instead of producing a partial image. RUN test "$(dpkg --print-architecture)" = "amd64" \ diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 5b5f07644..958e749d5 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -1,50 +1,181 @@ # Development Container -Quick-start development environment with all tools pre-installed. +Development environment for `gitops-reverser` with Go, Kubernetes, Docker CLI, and local SSH commit +signing support. ## Quick Start ### VS Code -1. Install [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) -2. Open project in VS Code: `code .` -3. Press `F1` → `Dev Containers: Reopen in Container` -4. Wait for initial build (~5-10 min first time) + +1. Install the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) +2. Open the project in VS Code: `code .` +3. Press `F1` and run `Dev Containers: Reopen in Container` +4. Wait for the initial build to finish + +### Before Reopening In Container + +This repo expects SSH agent forwarding for commit signing. + +On your host machine: + +```bash +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/id_ed25519 +ssh-add -L +git config --global user.name "Your Name" +git config --global user.email "you@example.com" +``` + +If `ssh-add -L` shows no keys, commit signing inside the devcontainer will fail. ### Verify + ```bash -go version # 1.26.1 -kind version # v0.30.0 -kubectl version # v1.32.3 -golangci-lint version # v2.4.0 +go version +kubectl version --client +golangci-lint version +docker version +git config --get gpg.format +git config --get user.signingkey +ssh-add -L ``` -### Run Tests +Expected Git signing values: + ```bash -make test # Unit tests -make lint # Linting -make test-e2e # E2E tests (creates Kind cluster) +git config --get gpg.format # ssh +git config --get commit.gpgsign # true +git config --get user.signingkey # /home/vscode/.ssh/devcontainer_signing_key.pub ``` -## Architecture +## How SSH Signing Works Here -Multi-stage Dockerfile: -- **`ci` stage** - CI base (Go tools, no Docker) - Used in GitHub Actions -- **`dev` stage** - Full dev (extends ci stage, adds Docker+Kind) - Local only +The devcontainer uses SSH commit signing, not GPG keyring-based signing. -Local dev builds directly from dev stage, no separate initialization needed. +- [`post-create.sh`](./post-create.sh) bootstraps Git identity and calls the signing sync helper +- [`sync-signing-key.sh`](./sync-signing-key.sh) reads the forwarded SSH agent and refreshes: + - `${HOME}/.ssh/devcontainer_signing_key.pub` + - `${HOME}/.config/git/allowed_signers` +- [`devcontainer.json`](./devcontainer.json) runs the helper on: + - `postCreateCommand` + - `postStartCommand` -## Files +This keeps the configured public key file aligned with the currently forwarded agent key. + +## Best Practices + +### SSH Signing + +1. Treat forwarded SSH agent keys as ephemeral + Key order and loaded keys can change between sessions. -- `Dockerfile` - Multi-stage container (ci and dev stages) -- `devcontainer.json` - VS Code configuration -- `README.md` - This file +2. Refresh signing config on create and on start + `postCreate` is not enough when the host agent changes later. + +3. Keep signing setup in one small script + This repo uses [`sync-signing-key.sh`](./sync-signing-key.sh) for that reason. + +4. Prefer deterministic key selection + The helper first looks for an SSH key whose comment matches `git user.email`. + If none matches, it falls back to the first key from `ssh-add -L`. + +5. Keep signing and verification data in sync + `user.signingkey` and `gpg.ssh.allowedSignersFile` are regenerated together. + +6. Store only public key material in the container + The private key stays on the host and is accessed through the forwarded SSH agent. + +7. Fail early with actionable messages + Missing agent, missing keys, or missing Git identity should stop setup immediately. + +### General Devcontainer Maintenance + +1. Keep lifecycle hooks small and explicit + Bootstrap in `post-create.sh`, refresh in `sync-signing-key.sh`. + +2. Prefer reusable helpers over duplicated shell snippets + That makes drift easier to spot and fixes easier to apply. + +3. Make runtime state easy to inspect + Useful checks: + +```bash +echo "$SSH_AUTH_SOCK" +ssh-add -L +cat ~/.ssh/devcontainer_signing_key.pub +cat ~/.config/git/allowed_signers +git config --show-origin --get user.signingkey +``` ## Troubleshooting -**Container won't build** → Ensure Docker is running -**Slow rebuild** → Normal, only rebuilds when tools/deps change +### Commit Signing Fails + +Check: + +```bash +ssh-add -L +cat ~/.ssh/devcontainer_signing_key.pub +git config --get user.signingkey +``` + +Then refresh the signing setup manually: + +```bash +bash .devcontainer/sync-signing-key.sh +``` + +Common causes: + +- no host `ssh-agent` running +- no key loaded into the host agent +- the forwarded agent key changed after the container was created +- `user.name` or `user.email` is missing + +Typical signing error when the configured public key is stale: + +```text +error: No private key found for public key "/home/vscode/.ssh/devcontainer_signing_key.pub"? +``` + +### Push Fails But Commit Signing Works + +Commit signing and Git push authentication are separate concerns. + +This repo currently uses an HTTPS remote, so push failures are often caused by: + +- missing upstream branch +- non-fast-forward push rejection +- GitHub HTTPS credential issues + +Check: + +```bash +git remote -v +git branch -vv +git push --dry-run -u origin HEAD +``` + +### Container Won't Build + +Ensure Docker is running on the host. + +### Slow Rebuild + +Usually normal. Rebuild time mostly depends on tool installation and cache reuse. + +## Files + +- [`Dockerfile`](./Dockerfile) - Multi-stage container image +- [`devcontainer.json`](./devcontainer.json) - VS Code devcontainer configuration +- [`post-create.sh`](./post-create.sh) - Initial bootstrap +- [`sync-signing-key.sh`](./sync-signing-key.sh) - SSH signing refresh helper +- [`README.md`](./README.md) - This document + +## References -Kind/DOOD failure findings + DinD migration plan: +Kind/DOOD failure findings + DinD migration plan: [`KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md`](./KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md) -See [`docs/COMPLETE_SOLUTION.md`](../docs/COMPLETE_SOLUTION.md) for details. +Additional project details: +[`docs/COMPLETE_SOLUTION.md`](../docs/COMPLETE_SOLUTION.md) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c840c394b..cf54c1fba 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -73,6 +73,7 @@ } }, "postCreateCommand": "bash .devcontainer/post-create.sh '${containerWorkspaceFolder}'", + "postStartCommand": "bash .devcontainer/sync-signing-key.sh", "remoteUser": "vscode", "mounts": [ "source=gomodcache,target=/go/pkg/mod,type=volume", diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 4826b9396..a5ddbb845 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -39,39 +39,8 @@ if ! git config --global --get user.email >/dev/null 2>&1; then git config --global user.email "${git_email}" fi -# Require SSH agent forwarding for signing. -if [ -z "${SSH_AUTH_SOCK:-}" ]; then - fail "SSH agent not available in the devcontainer. Start ssh-agent on your machine, load your key with ssh-add, then reopen the devcontainer." -fi - -agent_keys_file="$(mktemp)" -trap 'rm -f "${agent_keys_file}"' EXIT - -if ! ssh-add -L >"${agent_keys_file}" 2>/dev/null; then - fail "Could not read SSH keys from agent. Make sure your key is loaded on your machine with ssh-add, then reopen the devcontainer." -fi - -if ! grep -qE '^ssh-' "${agent_keys_file}"; then - fail "SSH agent is running but has no keys loaded. Run ssh-add ~/.ssh/id_ed25519 on your machine, then reopen the devcontainer." -fi - -first_pubkey="$(head -n 1 "${agent_keys_file}")" - -# Enforce SSH commit signing. -git config --global gpg.format ssh -git config --global commit.gpgsign true - -mkdir -p /home/vscode/.config/git /home/vscode/.ssh -printf '%s\n' "${first_pubkey}" > /home/vscode/.ssh/devcontainer_signing_key.pub -chmod 600 /home/vscode/.ssh/devcontainer_signing_key.pub -git config --global user.signingkey /home/vscode/.ssh/devcontainer_signing_key.pub - -# Useful for local verification output. -printf '%s <%s> %s\n' "${git_name}" "${git_email}" "${first_pubkey}" > /home/vscode/.config/git/allowed_signers -chmod 600 /home/vscode/.config/git/allowed_signers -git config --global gpg.ssh.allowedSignersFile /home/vscode/.config/git/allowed_signers - -log "Git identity and SSH signing configured" +log "Refreshing Git SSH signing configuration" +bash "${workspace_dir}/.devcontainer/sync-signing-key.sh" log "Ensuring Go cache directories exist" sudo mkdir -p \ diff --git a/.devcontainer/sync-signing-key.sh b/.devcontainer/sync-signing-key.sh new file mode 100644 index 000000000..efae60ee9 --- /dev/null +++ b/.devcontainer/sync-signing-key.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail + +log() { + echo "[sync-signing-key] $*" +} + +fail() { + echo "[sync-signing-key] ERROR: $*" >&2 + exit 1 +} + +git_name="$(git config --get user.name || true)" +git_email="$(git config --get user.email || true)" + +if [ -z "${git_name}" ] || [ -z "${git_email}" ]; then + fail "Missing Git identity. Configure user.name and user.email before syncing SSH signing." +fi + +if [ -z "${SSH_AUTH_SOCK:-}" ]; then + fail "SSH agent not available in the devcontainer. Start ssh-agent on your machine, load your key with ssh-add, then reopen the devcontainer." +fi + +agent_keys_file="$(mktemp)" +trap 'rm -f "${agent_keys_file}"' EXIT + +if ! ssh-add -L >"${agent_keys_file}" 2>/dev/null; then + fail "Could not read SSH keys from agent. Make sure your key is loaded on your machine with ssh-add, then reopen the devcontainer." +fi + +if ! grep -qE '^ssh-' "${agent_keys_file}"; then + fail "SSH agent is running but has no keys loaded. Run ssh-add ~/.ssh/id_ed25519 on your machine, then reopen the devcontainer." +fi + +selected_pubkey="$(awk -v email="${git_email}" '$0 ~ email { print; exit }' "${agent_keys_file}")" +if [ -z "${selected_pubkey}" ]; then + selected_pubkey="$(head -n 1 "${agent_keys_file}")" + log "No SSH key comment matched ${git_email}; using the first agent key" +else + log "Using SSH key whose comment matches ${git_email}" +fi + +signing_key_path="${HOME}/.ssh/devcontainer_signing_key.pub" +allowed_signers_path="${HOME}/.config/git/allowed_signers" + +git config --global gpg.format ssh +git config --global commit.gpgsign true + +mkdir -p "${HOME}/.config/git" "${HOME}/.ssh" +printf '%s\n' "${selected_pubkey}" > "${signing_key_path}" +chmod 600 "${signing_key_path}" +git config --global user.signingkey "${signing_key_path}" + +printf '%s <%s> %s\n' "${git_name}" "${git_email}" "${selected_pubkey}" > "${allowed_signers_path}" +chmod 600 "${allowed_signers_path}" +git config --global gpg.ssh.allowedSignersFile "${allowed_signers_path}" + +log "Git SSH signing configuration refreshed" diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 9c0853be9..f93983c0a 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -33,10 +33,10 @@ runs: using: 'composite' steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} @@ -44,7 +44,7 @@ runs: - name: Build and push Docker image id: build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . platforms: ${{ inputs.platforms }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 435d94170..612772fc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,6 +260,7 @@ jobs: env: PROJECT_IMAGE: ${{ needs.docker-build.outputs.image }} CI_CONTAINER: ${{ needs.build-ci-container.outputs.image }} + IMAGE_DELIVERY_MODE: pull steps: - name: Checkout code @@ -277,6 +278,7 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -w "${{ env.CI_WORKDIR }}" \ -e PROJECT_IMAGE=${{ env.PROJECT_IMAGE }} \ + -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \ -e HOST_PROJECT_PATH=${{ github.workspace }} \ ${{ env.CI_CONTAINER }} \ bash -c " @@ -295,6 +297,7 @@ jobs: env: PROJECT_IMAGE: ${{ needs.docker-build.outputs.image }} CI_CONTAINER: ${{ needs.build-ci-container.outputs.image }} + IMAGE_DELIVERY_MODE: pull steps: - name: Checkout code @@ -319,6 +322,7 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -w "${{ env.CI_WORKDIR }}" \ -e PROJECT_IMAGE=${{ env.PROJECT_IMAGE }} \ + -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \ -e HELM_CHART_SOURCE="./gitops-reverser.tgz" \ -e HOST_PROJECT_PATH=${{ github.workspace }} \ ${{ env.CI_CONTAINER }} \ diff --git a/AGENTS.md b/AGENTS.md index 847ceab2d..9a71eb8c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,6 +99,4 @@ make test-e2e-quickstart-helm ## REFERENCES -- Full guidelines: [`DEVELOPMENT_RULES.md`](../../DEVELOPMENT_RULES.md) -- Implementation checklist: [`IMPLEMENTATION_CHECKLIST.md`](../../IMPLEMENTATION_CHECKLIST.md) -- Contributing guide: [`CONTRIBUTING.md`](../../CONTRIBUTING.md) \ No newline at end of file +- Contributing guide: [`CONTRIBUTING.md`](./CONTRIBUTING.md) \ No newline at end of file diff --git a/Makefile b/Makefile index b70d34fa1..2a1484f07 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,11 @@ PROJECT_IMAGE := $(if $(PROJECT_IMAGE_INPUT),$(PROJECT_IMAGE_INPUT),$(E2E_LOCAL_ PROJECT_IMAGE_PROVIDED := $(if $(filter-out $(E2E_LOCAL_IMAGE),$(PROJECT_IMAGE_INPUT)),true,) export PROJECT_IMAGE export PROJECT_IMAGE_PROVIDED +# IMAGE_DELIVERY_MODE controls how the cluster gets the controller image: +# - load (default): import into k3d from the local Docker daemon +# - pull: let Kubernetes pull from the registry at rollout time +IMAGE_DELIVERY_MODE ?= load +export IMAGE_DELIVERY_MODE E2E_AGE_KEY_FILE ?= /tmp/e2e-age-key.txt # CTX: kubeconfig context for the cluster being operated on. @@ -119,6 +124,12 @@ CTX ?= k3d-gitops-reverser-test-e2e CLUSTER_NAME ?= $(patsubst kind-%,%,$(patsubst k3d-%,%,$(CTX))) CS := .stamps/cluster/$(CTX) IS := .stamps/image +AUDIT_ASSET_SOURCE_DIR := test/e2e/cluster/audit +AUDIT_ASSET_DIR := $(CS)/audit +AUDIT_POLICY_SOURCE := $(AUDIT_ASSET_SOURCE_DIR)/policy.yaml +AUDIT_WEBHOOK_BOOTSTRAP_SOURCE := $(AUDIT_ASSET_SOURCE_DIR)/webhook-config.yaml +AUDIT_POLICY_PATH := $(AUDIT_ASSET_DIR)/policy.yaml +AUDIT_WEBHOOK_CONFIG_PATH := $(AUDIT_ASSET_DIR)/webhook-config.yaml GO_SOURCES = $(shell find cmd internal api -type f -name '*.go' \ ! -name '*_test.go' ! -name 'zz_generated.deepcopy.go') go.mod go.sum @@ -190,9 +201,16 @@ $(CS)/$(NAMESPACE)/git-$(REPO_NAME)/checkout.ready: $(CS)/gitea/bootstrap/org-$( # Called by the full e2e suite. # For now: clean the sut namespace, recreate it, run the installer, and deploy the controller image. # Prefer depending on stamps; this target must not invoke Go e2e tests (Go calls this target). +$(CS)/$(NAMESPACE)/webhook-tls.ready: $(CS)/$(NAMESPACE)/controller.deployed | $(CS)/$(NAMESPACE) + CTX="$(CTX)" CLUSTER_NAME="$(CLUSTER_NAME)" NAMESPACE="$(NAMESPACE)" \ + WEBHOOK_CONFIG="$(AUDIT_WEBHOOK_CONFIG_PATH)" \ + bash hack/e2e/inject-webhook-tls.sh + touch $@ + $(CS)/$(NAMESPACE)/prepare-e2e.ready: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml \ $(CS)/image.loaded \ $(CS)/$(NAMESPACE)/controller.deployed \ + $(CS)/$(NAMESPACE)/webhook-tls.ready \ $(CS)/$(NAMESPACE)/sops-secret.applied \ | $(CS)/$(NAMESPACE) touch $@ @@ -265,6 +283,7 @@ dist/install.yaml: $(HELM_SYNC_OUTPUTS) $$(CHART_INPUTS) | dist ## Generate cons --namespace $(NAMESPACE) \ --set labels.managedBy=kubectl \ --set createNamespace=true \ + --set queue.redis.addr=$(DEFAULT_AUDIT_REDIS_ADDR) \ --include-crds \ > $@ @@ -293,6 +312,10 @@ PROMETHEUS_PORT ?= 19090 # Valkey E2E Configuration VALKEY_PORT ?= 16379 +DEFAULT_AUDIT_REDIS_ADDR ?= valkey:6379 +E2E_AUDIT_REDIS_ADDR ?= valkey.valkey-e2e.svc.cluster.local:6379 +E2E_VALKEY_PASSWORD ?= e2e-valkey-password +E2E_CONTROLLER_SERVICE_CLUSTER_IP ?= 10.43.200.200 FLUX_NAMESPACE ?= flux-system FLUX_WAIT_TIMEOUT ?= 300s @@ -313,12 +336,19 @@ $(ENVTEST_STAMP): Makefile ##@ E2E Stamp Targets (cluster-parameterized; pass CTX=k3d- to target a different cluster) -$(CS) $(IS) $(CS)/$(NAMESPACE) $(CS)/gitea/bootstrap \ +$(CS) $(IS) $(CS)/audit $(CS)/$(NAMESPACE) $(CS)/gitea/bootstrap \ $(CS)/$(NAMESPACE)/helm $(CS)/$(NAMESPACE)/config-dir $(CS)/$(NAMESPACE)/plain-manifests-file dist: mkdir -p $@ -$(CS)/ready: test/e2e/cluster/start-cluster.sh | $(CS) +$(AUDIT_POLICY_PATH): $(AUDIT_POLICY_SOURCE) | $(CS)/audit + cp "$(AUDIT_POLICY_SOURCE)" "$@" + +$(AUDIT_WEBHOOK_CONFIG_PATH): $(AUDIT_WEBHOOK_BOOTSTRAP_SOURCE) | $(CS)/audit + cp "$(AUDIT_WEBHOOK_BOOTSTRAP_SOURCE)" "$@" + +$(CS)/ready: test/e2e/cluster/start-cluster.sh $(AUDIT_POLICY_PATH) $(AUDIT_WEBHOOK_CONFIG_PATH) | $(CS) export CLUSTER_NAME=$(CLUSTER_NAME) + export AUDIT_DIR_REL=$(AUDIT_ASSET_DIR) bash test/e2e/cluster/start-cluster.sh kubectl --context $(CTX) get ns >/dev/null touch $@ @@ -349,25 +379,8 @@ $(CS)/flux.installed: $(CS)/ready | $(CS) .SILENT: $(CS)/flux-setup.ready $(CS)/services.ready $(CS)/flux-setup.ready: $(FLUX_SETUP_READY_INPUTS) | $(CS) kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) - flux_ready_count=0 - echo "⏳ Waiting for Flux-managed installations to become ready..." - for kind in \ - helmreleases.helm.toolkit.fluxcd.io \ - kustomizations.kustomize.toolkit.fluxcd.io - do - resources="$$(kubectl --context $(CTX) get $$kind --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null)" - [ -z "$$resources" ] && continue - - resource_count="$$(printf '%s\n' "$$resources" | sed '/^$$/d' | wc -l | tr -d ' ')" - flux_ready_count="$$(($$flux_ready_count + $$resource_count))" - - printf '%s\n' "$$resources" | while read -r namespace name; do - [ -n "$$namespace" ] || continue - kubectl --context $(CTX) -n "$$namespace" wait "$$kind/$$name" --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT) - done - done - [ "$$flux_ready_count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e ready-check resources found" >&2; exit 1; } - echo "✓ Flux-managed installations ready: $$flux_ready_count" + CTX="$(CTX)" FLUX_SERVICES_WAIT_TIMEOUT="$(FLUX_SERVICES_WAIT_TIMEOUT)" \ + bash hack/e2e/wait-flux-services.sh touch $@ # Aggregate stamp for external E2E services required by tests. @@ -443,38 +456,23 @@ $(CS)/crds.applied: $(CS)/ready $$(CRD_INPUTS) | $(CS) touch $@ $(CS)/image.loaded: $(IS)/project-image.ready $(CS)/ready | $(CS) - if ! $(CONTAINER_TOOL) image inspect "$(PROJECT_IMAGE)" >/dev/null 2>&1; then - if [ -z "$(PROJECT_IMAGE_PROVIDED)" ]; then - echo "Local image $(PROJECT_IMAGE) missing; rebuilding..." - $(MAKE) $(IS)/controller.id - else - echo "ERROR: PROJECT_IMAGE=$(PROJECT_IMAGE) not found locally" >&2 - exit 2 - fi - fi - img_id="$$( $(CONTAINER_TOOL) inspect --format='{{.Id}}' $(PROJECT_IMAGE) )" - echo "Loading $(PROJECT_IMAGE) ($$img_id) into $(CTX)" - $(K3D) image import $(PROJECT_IMAGE) -c $(CLUSTER_NAME) - echo "$$img_id" > "$@" + CTX="$(CTX)" CLUSTER_NAME="$(CLUSTER_NAME)" PROJECT_IMAGE="$(PROJECT_IMAGE)" \ + PROJECT_IMAGE_PROVIDED="$(PROJECT_IMAGE_PROVIDED)" \ + IMAGE_DELIVERY_MODE="$(IMAGE_DELIVERY_MODE)" \ + CONTROLLER_ID_STAMP="$(IS)/controller.id" \ + CONTAINER_TOOL="$(CONTAINER_TOOL)" K3D="$(K3D)" STAMP_FILE="$@" \ + bash hack/e2e/load-image.sh $(CS)/$(NAMESPACE)/controller.deployed: $(CS)/$(NAMESPACE)/$(INSTALL_MODE)/install.yaml $(CS)/image.loaded | $(CS)/$(NAMESPACE) - ctx="$(CTX)" - ns="$(NAMESPACE)" - img="$(PROJECT_IMAGE)" - container="$(CONTROLLER_CONTAINER)" - selector="$(CONTROLLER_DEPLOY_SELECTOR)" - [ -n "$$img" ] || { echo "ERROR: PROJECT_IMAGE must be non-empty" >&2; exit 2; } - [ -n "$$container" ] || { echo "ERROR: CONTROLLER_CONTAINER must be non-empty" >&2; exit 2; } - count="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$selector" --no-headers 2>/dev/null | wc -l | tr -d ' ')" - [ "$$count" -eq 1 ] || { - echo "ERROR: Expected exactly 1 Deployment matching '$$selector' in namespace '$$ns', found $$count" >&2 - kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$selector" -o wide || true - exit 1 - } - deploy="$$(kubectl --context "$$ctx" -n "$$ns" get deploy -l "$$selector" -o jsonpath='{.items[0].metadata.name}')" - echo "Setting deployment/$$deploy container '$$container' to image '$$img'" - kubectl --context "$$ctx" -n "$$ns" set image "deployment/$$deploy" "$$container=$$img" - kubectl --context "$$ctx" -n "$$ns" rollout status "deployment/$$deploy" --timeout=180s + CTX="$(CTX)" NAMESPACE="$(NAMESPACE)" PROJECT_IMAGE="$(PROJECT_IMAGE)" \ + IMAGE_DELIVERY_MODE="$(IMAGE_DELIVERY_MODE)" \ + IMAGE_PULL_SECRET_NAME="$(IMAGE_PULL_SECRET_NAME)" \ + IMAGE_PULL_SECRET_REGISTRY="$(IMAGE_PULL_SECRET_REGISTRY)" \ + IMAGE_PULL_SECRET_USERNAME="$(IMAGE_PULL_SECRET_USERNAME)" \ + IMAGE_PULL_SECRET_PASSWORD="$(IMAGE_PULL_SECRET_PASSWORD)" \ + CONTROLLER_CONTAINER="$(CONTROLLER_CONTAINER)" \ + CONTROLLER_DEPLOY_SELECTOR="$(CONTROLLER_DEPLOY_SELECTOR)" \ + bash hack/e2e/deploy-controller.sh touch "$@" .PHONY: portforward-ensure @@ -512,6 +510,11 @@ define UPDATE_IF_CHANGED fi endef +define DO_ENSURE_VALKEY_AUTH + @CTX="$(CTX)" NAMESPACE="$(NAMESPACE)" E2E_VALKEY_PASSWORD="$(E2E_VALKEY_PASSWORD)" \ + bash hack/e2e/ensure-valkey-auth.sh +endef + define REQUIRE_FILE @[ -f "$(1)" ] || { \ echo "ERROR: missing required $(2) file '$(1)'" >&2; \ @@ -528,9 +531,14 @@ $(CS)/$(NAMESPACE)/helm/install.yaml: $(CS)/services.ready $(HELM_SYNC_OUTPUTS) if kubectl --context $(CTX) get crd $(INSTALL_CRDS) >/dev/null 2>&1; then skip_crds_arg="--skip-crds" fi + $(DO_ENSURE_VALKEY_AUTH) $(HELM) --kube-context $(CTX) upgrade --install $(INSTALL_NAME) "$(HELM_CHART_SOURCE)" \ --namespace $(NAMESPACE) \ --create-namespace \ + --set queue.redis.addr=$(E2E_AUDIT_REDIS_ADDR) \ + --set queue.redis.auth.existingSecret=valkey-auth \ + --set queue.redis.auth.existingSecretKey=password \ + --set service.clusterIP=$(E2E_CONTROLLER_SERVICE_CLUSTER_IP) \ $$skip_crds_arg tmp_manifest="$(@D)/.$(@F).tmp" $(HELM) --kube-context $(CTX) get manifest $(INSTALL_NAME) \ @@ -542,37 +550,21 @@ CONFIG_DIR_INPUTS = $(shell find config -type f) $(CS)/$(NAMESPACE)/config-dir/install.yaml: $(CS)/services.ready $(MANIFEST_OUTPUTS) $$(CONFIG_DIR_INPUTS) | $(CS)/$(NAMESPACE)/config-dir $(DO_CLEANUP_INSTALLS) mkdir -p "$(@D)" # keep: cleanup script can delete this directory during the same recipe - tmpdir="$$(mktemp -d)" - trap 'rm -rf "$$tmpdir"' EXIT - cp -R config "$$tmpdir/config" - (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set namespace "$(NAMESPACE)") - (cd "$$tmpdir/config" && $(KUSTOMIZE) edit set image gitops-reverser="$(PROJECT_IMAGE)") - (cd "$$tmpdir/config" && $(KUSTOMIZE) build .) \ - | tee "$$tmpdir/install.yaml" \ - | $(KUBECTL) --context "$(CTX)" apply -f - - $(call UPDATE_IF_CHANGED,$$tmpdir/install.yaml) + CTX="$(CTX)" NAMESPACE="$(NAMESPACE)" PROJECT_IMAGE="$(PROJECT_IMAGE)" \ + E2E_VALKEY_PASSWORD="$(E2E_VALKEY_PASSWORD)" \ + KUSTOMIZE="$(KUSTOMIZE)" KUBECTL="$(KUBECTL)" STAMP_FILE="$@" \ + bash hack/e2e/install-config-dir.sh $(CS)/$(NAMESPACE)/plain-manifests-file/install.yaml: $(CS)/services.ready dist/install.yaml | $(CS)/$(NAMESPACE)/plain-manifests-file $(DO_CLEANUP_INSTALLS) mkdir -p "$(@D)" # keep: cleanup script can delete this directory during the same recipe - ctx="$(CTX)" - ns="$(NAMESPACE)" - tmpdir="$$(mktemp -d)" - trap 'rm -rf "$$tmpdir"' EXIT - cp dist/install.yaml "$$tmpdir/install.yaml" - printf '%s\n' \ - 'apiVersion: kustomize.config.k8s.io/v1beta1' \ - 'kind: Kustomization' \ - 'resources:' \ - '- install.yaml' \ - > "$$tmpdir/kustomization.yaml" - ( - cd "$$tmpdir" - $(KUSTOMIZE) edit set namespace "$$ns" >/dev/null - $(KUSTOMIZE) build . - ) | tee "$$tmpdir/rendered-install.yaml" \ - | $(KUBECTL) --context "$$ctx" apply -f - - $(call UPDATE_IF_CHANGED,$$tmpdir/rendered-install.yaml) + $(DO_ENSURE_VALKEY_AUTH) + CTX="$(CTX)" NAMESPACE="$(NAMESPACE)" \ + DEFAULT_AUDIT_REDIS_ADDR="$(DEFAULT_AUDIT_REDIS_ADDR)" \ + E2E_AUDIT_REDIS_ADDR="$(E2E_AUDIT_REDIS_ADDR)" \ + E2E_CONTROLLER_SERVICE_CLUSTER_IP="$(E2E_CONTROLLER_SERVICE_CLUSTER_IP)" \ + KUSTOMIZE="$(KUSTOMIZE)" KUBECTL="$(KUBECTL)" STAMP_FILE="$@" \ + bash hack/e2e/install-plain-manifests.sh .PHONY: test-e2e-quickstart-helm test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) diff --git a/README.md b/README.md index 805ad6b52..f794f8c22 100644 --- a/README.md +++ b/README.md @@ -12,51 +12,41 @@ GitOps Reverser is a Kubernetes operator that turns live Kubernetes API activity into clean, versioned YAML in Git. It gives API-first teams a Git audit trail and a deployable repo without forcing every change through Git first. -> [!IMPORTANT] -> Speaking at KubeCon: the Reverse GitOps pattern and this project will be presented on -> [March 23, 2026 at 15:55 in Amsterdam](https://sched.co/2DY82). - -
GitOps Reverser Demo
- -Want proof? See this [example commit](https://github.com/ConfigButler/example-audit/commit/800a51e5a8edcccbc85c94d5fef7ef7cc8381b7b) -in [ConfigButler/example-audit](https://github.com/ConfigButler/example-audit). - The broader pattern behind this project is described at [reversegitops.dev](https://reversegitops.dev). -This repo contains a concrete operator that is implementing this. - -## Kubecon Adam -> I would love your feedback! -I am at Kubecon Amsterdam 2026: +
Demo: kubectl apply triggers a sanitized Git commit within seconds
-* Feel free to [book a meeting](https://calendar.app.google/FgYaoZog1dFpRG9Z7) to share your thoughts. -* Take a look at the [manifesto](https://reversegitops.dev/) ⭐ or the [concrete open source implemention](https://github.com/ConfigButler/gitops-reverser) ⭐. -* Let me know what you think, I'm on [LinkedIn](https://www.linkedin.com/in/simonkoudijs/). +Want proof? See this [example commit](https://github.com/ConfigButler/example-audit/commit/800a51e5a8edcccbc85c94d5fef7ef7cc8381b7b) in [ConfigButler/example-audit](https://github.com/ConfigButler/example-audit). ## Why Today teams often choose between workflows: -- Pure GitOps: safe and auditable, but less friendly for users and tools that work through the Kubernetes API. -- API-first: fast and interactive, but it usually does not come with strong review, rollout safety, or change history out of the box. +- **Pure GitOps**: safe and auditable, but less friendly for users and tools that work through the Kubernetes API. +- **API-first**: fast and interactive, but it usually does not come with strong review, rollout safety, or change history. GitOps Reverser bridges that gap: write to the Kubernetes API, and let the operator write the result to Git. -![](docs/images/overview.excalidraw.svg) +![Overview diagram showing how API events flow through the operator into Git](docs/images/overview.excalidraw.svg) ## How it works -- Capture Kubernetes API activity. -- Sanitize objects into stable YAML (intent only). -- Queue and batch writes safely. -- Commit and push the result to Git with useful metadata (username and e-mail that made the change). +Kubernetes has a built-in audit feature that streams every API server event to a webhook. GitOps Reverser receives these events, strips runtime noise from the object, and pushes a clean YAML commit within seconds, including the username of whoever made the change. + +1. Receive the API event via the Kubernetes audit webhook. +2. Sanitize the object into stable YAML (intent only, no runtime fields). +3. Queue and batch writes safely via Valkey/Redis. +4. Commit and push to Git with useful metadata. + +> **Note:** This works best on clusters you control: `k3s`, `k3d`, Talos, Kamaji. Managed platforms often do not expose enough access to configure the audit webhook. ## Status -🚨 Early stage software. CRDs and behavior may change, and it is not recommended for production yet. +🚨 Early stage software. CRDs and behavior may change. Not recommended for production yet. -- Single pod only today (`replicas=1`); multi-pod/HA is not (yet!) supported yet. -- Runtime behavior is deterministic. The operator does not use AI or heuristics at runtime. -- Current roadmap includes signed commits, HA support, and more edge-case hardening. +- Single pod only (`replicas=1`); HA is not yet supported. +- Tests run against Kubernetes 1.35. Other versions may work but are not tested. +- Runtime behavior is deterministic; no AI or heuristics at runtime. ## Recommended usage @@ -67,58 +57,71 @@ GitOps Reverser bridges that gap: write to the Kubernetes API, and let the opera | Split ownership | API-first app resources, Git-first infra | Avoids shared write ownership | | Controlled bi-directional | Shared paths that truly need both sides | Requires explicit coordination | -For shared resources, the recommended model is: - -- write through the Kubernetes API -- let GitOps Reverser publish to Git -- let GitOps controllers acknowledge those commits in a controlled way +For shared resources, write through the Kubernetes API, let GitOps Reverser publish to Git, then let GitOps controllers acknowledge those commits in a controlled way. Do **not** let GitOps Reverser and Flux or Argo CD continuously reconcile the same resources. That creates loops and extra commits. -Do not let GitOps Reverser and Flux or Argo CD continuously reconcile the same resources in two always-on -loops. That creates stale replays, extra commits, and possible loops. +See [`docs/bi-directional.md`](docs/bi-directional.md) for the coordination guide. -For Flux on shared paths, the current direction is manual coordination: +## Quick start -- keep the source object -- suspend the `Kustomization` for the shared path -- refresh and reconcile on demand -- wait until Flux reports the expected revision +**Prerequisites** -See [`docs/bi-directional.md`](docs/bi-directional.md) for the user guide and -[`docs/design/bi-directional-flux-manual-handshake-plan.md`](docs/design/bi-directional-flux-manual-handshake-plan.md) -for the implementation direction. +- Kubernetes cluster with `kubectl` configured +- cert-manager for TLS certificate management +- Admin access to your kube-apiserver configuration (you need to configure the [audit webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend)) + - Managed platforms (e.g. EKS, GKE, AKS) don't allow this. + - See [`docs/design/audit-webhook-api-server-connectivity.md`](docs/design/audit-webhook-api-server-connectivity.md) for more information. -## Quick start -Prerequisites: +**1. Install cert-manager** -- A Kubernetes cluster with `kubectl` configured -- `cert-manager` for webhook certificates -- A Git repository with write access +```bash +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml +kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s +``` -Install `cert-manager` if needed: +**2. Install Valkey with auth** ```bash -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml +kubectl create namespace gitops-reverser +kubectl create secret generic valkey-auth \ + --namespace gitops-reverser \ + --from-literal=password="$(openssl rand -base64 32)" + +helm repo add valkey https://valkey.io/valkey-helm/ && helm repo update +helm install valkey valkey/valkey --version 0.9.3 --namespace gitops-reverser \ + --set auth.enabled=true \ + --set auth.usersExistingSecret=valkey-auth \ + --set auth.aclUsers.default.passwordKey=password \ + --set "auth.aclUsers.default.permissions=~* &* +@all" ``` -**1. Install GitOps Reverser** +**3. Install GitOps Reverser** ```bash -kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/latest/download/install.yaml +helm install gitops-reverser \ + oci://ghcr.io/configbutler/charts/gitops-reverser \ + --namespace gitops-reverser \ + --create-namespace ``` -**2. Create Git credentials** +**4. Configure kube-apiserver audit delivery** + +After the install is up, read the Helm post-install notes. -Generate an SSH key and add the public key to your Git provider: +If you missed the post-install output, you can print it again with: ```bash -ssh-keygen -t ed25519 -C "gitops-reverser@cluster" -f /tmp/gitops-reverser-key -N "" -# Add /tmp/gitops-reverser-key.pub to your Git provider as a deploy key +helm get notes gitops-reverser -n gitops-reverser ``` -Create a Secret with the private key: +Follow the post-install instructions carefully: you will be able to configure your Kubernetes API server to forward audit events. + +**5. Create Git credentials** ```bash +ssh-keygen -t ed25519 -C "gitops-reverser@cluster" -f /tmp/gitops-reverser-key -N "" +# Add /tmp/gitops-reverser-key.pub to your Git provider as a deploy key + kubectl create secret generic git-creds \ --from-file=ssh-privatekey=/tmp/gitops-reverser-key \ --from-literal=known_hosts="$(ssh-keyscan github.com)" \ @@ -127,110 +130,62 @@ kubectl create secret generic git-creds \ See [`docs/GITHUB_SETUP_GUIDE.md`](docs/GITHUB_SETUP_GUIDE.md) for a full setup guide. -**3. Create a `GitProvider`, `GitTarget`, and `WatchRule`** - -![](docs/images/config-basics.excalidraw.svg) +**6. Enable the starter Git configuration** -```bash -# NOTE: Edit the line with YOUR_USERNAME to match your repository -cat </.git ``` -If `age.recipients.generateWhenMissing: true` is enabled, GitOps Reverser can create the age key Secret -automatically. Back up the generated `*.agekey` immediately. If you lose that private key, existing encrypted -`*.sops.yaml` files are unrecoverable. After backup, remove the warning annotation: +Then check that the starter resources become ready: ```bash -kubectl annotate secret sops-age-key -n default configbutler.ai/backup-warning- +kubectl get gitprovider,gittarget,watchrule -n default ``` -**4. Test it** +See [`docs/configuration.md`](docs/configuration.md) for the quickstart values, names, and follow-up configuration +options. + +**7. Test it** ```bash kubectl create configmap test-config --from-literal=key=value -n default ``` -You should now see a new commit in your Git repository. +You should see a new commit in your Git repository within seconds. -For cluster-wide resources or watching multiple namespaces, use -[`ClusterWatchRule`](config/samples/clusterwatchrule.yaml). More examples live in [`config/samples/`](config/samples/). +> **Troubleshooting:** If no commit appears, check `kubectl logs -n gitops-reverser deploy/gitops-reverser`. ## Secrets and encryption -`Secret` resources go through the same pipeline, but sensitive values are encrypted before -commit. Secret manifests are written as `*.sops.yaml`. +`Secret` resources go through the same pipeline, but sensitive values are encrypted before commit. Secret manifests are written as `*.sops.yaml`. See [`docs/SOPS_AGE_GUIDE.md`](docs/SOPS_AGE_GUIDE.md) for working with Secrets, SOPS, age keys, bootstrap, and rotation. ## Known limitations -- GitOps Reverser currently supports only a single controller pod. -- Shared-resource bi-directional sync requires explicit coordination; it is not "set both sides to automatic and walk away." -- Avoid multiple GitProvider configurations pointing at the same repository. +- Single controller pod only; HA not yet supported. +- Shared-resource bi-directional sync requires explicit coordination. +- Avoid multiple `GitProvider` configurations pointing at the same repository. ## Other tools to consider | Tool | What it does | How GitOps Reverser differs | |---|---|---| -| [RichardoC/kube-audit-rest](https://github.com/RichardoC/kube-audit-rest) | Receives audit events and exposes them over a REST API | GitOps Reverser consumes the change and writes it to Git | | [robusta-dev/robusta](https://github.com/robusta-dev/robusta) | Broad observability and automation platform | GitOps Reverser is narrower and focused on Git write-back | +| [RichardoC/kube-audit-rest](https://github.com/RichardoC/kube-audit-rest) | Captures (partial!) cluster activity without access to Kubernetes API configuration | This tool might be usable for managed cluster. But only if it's possible to enable audit webhook forwarding | | [bpineau/katafygio](https://github.com/bpineau/katafygio) | Periodically snapshots cluster resources into Git | GitOps Reverser is event-driven and commit-oriented | ## Development -### DevContainer - This project includes a DevContainer for consistent development environments. [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/ConfigButler/gitops-reverser) @@ -238,8 +193,6 @@ This project includes a DevContainer for consistent development environments. - Linux/macOS: works with Docker Desktop or Docker Engine. - Windows: see [`docs/ci/WINDOWS_DEVCONTAINER_SETUP.md`](docs/ci/WINDOWS_DEVCONTAINER_SETUP.md). -### Running tests - ```bash make test make test-e2e @@ -250,6 +203,11 @@ make lint Contributions, issues, and discussions are welcome. +## Get in touch + +- Read the [Reverse GitOps manifesto](https://reversegitops.dev/) ⭐ for the broader pattern behind this work. +- Connect on [LinkedIn](https://www.linkedin.com/in/simonkoudijs/); feedback and ideas always welcome. + ## License Apache 2.0 diff --git a/TODO-tls.md b/TODO-tls.md new file mode 100644 index 000000000..085e9673a --- /dev/null +++ b/TODO-tls.md @@ -0,0 +1,96 @@ +https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ + +--audit-log-batch-buffer-size int Default: 10000 +The size of the buffer to store events before batching and writing. Only used in batch mode. + +--audit-log-batch-max-size int Default: 1 +The maximum size of a batch. Only used in batch mode. + +--audit-log-batch-max-wait duration +The amount of time to wait before force writing the batch that hadn't reached the max size. Only used in batch mode. + +--audit-log-batch-throttle-burst int +Maximum number of requests sent at the same moment if ThrottleQPS was not utilized before. Only used in batch mode. + +--audit-log-batch-throttle-enable +Whether batching throttling is enabled. Only used in batch mode. + +--audit-log-batch-throttle-qps float +Maximum average number of batches per second. Only used in batch mode. + +--audit-log-compress +If set, the rotated log files will be compressed using gzip. + +--audit-log-format string Default: "json" +Format of saved audits. "legacy" indicates 1-line text format for each event. "json" indicates structured json format. Known formats are legacy,json. + +--audit-log-maxage int +The maximum number of days to retain old audit log files based on the timestamp encoded in their filename. + +--audit-log-maxbackup int +The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files. + +--audit-log-maxsize int +The maximum size in megabytes of the audit log file before it gets rotated. + +--audit-log-mode string Default: "blocking" +Strategy for sending audit events. Blocking indicates sending events should block server responses. Batch causes the backend to buffer and write events asynchronously. Known modes are batch,blocking,blocking-strict. + +--audit-log-path string +If set, all requests coming to the apiserver will be logged to this file. '-' means standard out. + +--audit-log-truncate-enabled +Whether event and batch truncating is enabled. + +--audit-log-truncate-max-batch-size int Default: 10485760 +Maximum size of the batch sent to the underlying backend. Actual serialized size can be several hundreds of bytes greater. If a batch exceeds this limit, it is split into several batches of smaller size. + +--audit-log-truncate-max-event-size int Default: 102400 +Maximum size of the audit event sent to the underlying backend. If the size of an event is greater than this number, first request and response are removed, and if this doesn't reduce the size enough, event is discarded. + +--audit-log-version string Default: "audit.k8s.io/v1" +API group and version used for serializing audit events written to log. + +--audit-policy-file string +Path to the file that defines the audit policy configuration. + +--audit-webhook-batch-buffer-size int Default: 10000 +The size of the buffer to store events before batching and writing. Only used in batch mode. + +--audit-webhook-batch-max-size int Default: 400 +The maximum size of a batch. Only used in batch mode. + +--audit-webhook-batch-max-wait duration Default: 30s +The amount of time to wait before force writing the batch that hadn't reached the max size. Only used in batch mode. + +--audit-webhook-batch-throttle-burst int Default: 15 +Maximum number of requests sent at the same moment if ThrottleQPS was not utilized before. Only used in batch mode. + +--audit-webhook-batch-throttle-enable Default: true +Whether batching throttling is enabled. Only used in batch mode. + +--audit-webhook-batch-throttle-qps float Default: 10 +Maximum average number of batches per second. Only used in batch mode. + +--audit-webhook-config-file string +Path to a kubeconfig formatted file that defines the audit webhook configuration. + +--audit-webhook-initial-backoff duration Default: 10s +The amount of time to wait before retrying the first failed request. + +--audit-webhook-mode string Default: "batch" +Strategy for sending audit events. Blocking indicates sending events should block server responses. Batch causes the backend to buffer and write events asynchronously. Known modes are batch,blocking,blocking-strict. + +--audit-webhook-truncate-enabled +Whether event and batch truncating is enabled. + +--audit-webhook-truncate-max-batch-size int Default: 10485760 +Maximum size of the batch sent to the underlying backend. Actual serialized size can be several hundreds of bytes greater. If a batch exceeds this limit, it is split into several batches of smaller size. + +--audit-webhook-truncate-max-event-size int Default: 102400 +Maximum size of the audit event sent to the underlying backend. If the size of an event is greater than this number, first request and response are removed, and if this doesn't reduce the size enough, event is discarded. + + +--- + +The today here is that we should come up with some advise on how to self host these: which values matter? And why or why not set them? Especially focus on not getting to many errors during setup I guess :-) diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index 9c3031e2a..96976d527 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -5,43 +5,16 @@ This Helm chart provides a production-ready single-pod deployment. ## Quick Start -```bash -# 1. Install cert-manager (if not already installed) -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml - -# 2. Wait for cert-manager to be ready -kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s - -# 3. Install GitOps Reverser -helm install gitops-reverser \ - oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system \ - --create-namespace - -# 4. Verify installation -kubectl get pods -n gitops-reverser-system -``` +The official end-to-end quickstart lives in the repository root README: -Controller install is complete. Next, create a minimal provider/target/rule to validate first commit flow: +- [`README.md`](../../README.md) -```bash -kubectl apply -f config/samples/quickstart-gitprovider.yaml -kubectl apply -f config/samples/quickstart-gittarget.yaml -kubectl apply -f config/samples/quickstart-watchrule.yaml -``` +Use that guide for cert-manager, Valkey, Helm install, kube-apiserver audit configuration, Git credentials, starter +`GitProvider`/`GitTarget`/`WatchRule` resources, and first-commit verification. -The quickstart target uses: -- `spec.path` -- `spec.encryption.provider: sops` -- `spec.encryption.age.recipients.generateWhenMissing: true` +This chart README stays focused on chart-specific installation, configuration, and operations. -When the encryption Secret is auto-generated, back up the generated `*.agekey` entry immediately. -If you lose it, existing encrypted `*.sops.yaml` files are unrecoverable. -After backup verification, remove warning annotation: - -```bash -kubectl annotate secret sops-age-key -n default configbutler.ai/backup-warning- -``` +For the chart's optional starter `quickstart` block, see [`docs/configuration.md`](../../docs/configuration.md). ## Features @@ -58,6 +31,7 @@ kubectl annotate secret sops-age-key -n default configbutler.ai/backup-warning- - Kubernetes 1.28+ - Helm 3.8+ - cert-manager 1.13+ (for webhook TLS certificates) +- Valkey or Redis reachable from the controller, with auth enabled (`queue.redis.auth.existingSecret`) ## Installation @@ -68,28 +42,26 @@ Install the latest version: ```bash helm install gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system \ + --namespace gitops-reverser \ --create-namespace ``` +After installation, print the chart's post-install instructions with: + +```bash +helm get notes gitops-reverser --namespace gitops-reverser +``` + Install a specific version: ```bash helm install gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ --version 0.3.0 \ - --namespace gitops-reverser-system \ + --namespace gitops-reverser \ --create-namespace ``` -### From GitHub Releases - -You can also install using the single YAML manifest: - -```bash -kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/latest/download/install.yaml -``` - ## Architecture ### Deployment Topology @@ -139,7 +111,7 @@ affinity: {} ```bash helm install gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system \ + --namespace gitops-reverser \ --create-namespace \ --values minimal-values.yaml ``` @@ -197,28 +169,45 @@ webhook: | `image.repository` | Container image repository | `ghcr.io/configbutler/gitops-reverser` | | `webhook.validating.failurePolicy` | Webhook failure policy (Ignore/Fail) | `Ignore` | | `servers.admission.tls.enabled` | Serve admission webhook with TLS (disable only for local/testing) | `true` | -| `servers.admission.tls.secretName` | Secret name for admission TLS cert/key | `-admission-server-cert` | +| `servers.admission.tls.secretNameOverride` | Override Secret name for admission TLS cert/key | `-admission-server-cert` | | `servers.audit.port` | Audit container port | `9444` | | `servers.audit.tls.enabled` | Serve audit ingress with TLS | `true` | | `servers.audit.maxRequestBodyBytes` | Max accepted audit request size | `10485760` | | `servers.audit.timeouts.read` | Audit-server read timeout | `15s` | | `servers.audit.timeouts.write` | Audit-server write timeout | `30s` | | `servers.audit.timeouts.idle` | Audit-server idle timeout | `60s` | -| `servers.audit.tls.secretName` | Secret name for audit TLS cert/key | `-audit-server-cert` | -| `queue.redis.enabled` | Enqueue accepted audit events to Redis Streams | `false` | -| `queue.redis.addr` | Redis endpoint for durable audit queueing | `""` | +| `servers.audit.tls.secretNameOverride` | Override Secret name for audit TLS cert/key | `-audit-server-cert` | +| `auditService.type` | Service type for the dedicated audit Service | `NodePort` | +| `auditService.nodePort` | Fixed NodePort for the audit Service when `auditService.type=NodePort` | `30444` | +| `auditService.clusterIP` | Optional fixed ClusterIP for the dedicated audit Service | `""` | +| `quickstart.enabled` | Create starter `GitProvider`, `GitTarget`, and `WatchRule` resources | `false` | +| `quickstart.namespace` | Namespace for the starter quickstart resources | `default` | +| `quickstart.gitProvider.url` | Repository URL used by the starter `GitProvider` | `""` | +| `quickstart.gitProvider.secretRef.name` | Existing Secret name used by the starter `GitProvider` | `git-creds` | +| `quickstart.gitTarget.path` | Repository path used by the starter `GitTarget` | `live-cluster` | +| `quickstart.watchRule.rules` | Rules used by the starter `WatchRule` | `configmaps create/update/delete` | +| `queue.redis.addr` | Redis endpoint (`host:port`) for required durable audit queueing | `valkey:6379` | +| `queue.redis.auth.existingSecret` | Name of a pre-created Secret holding the Redis password | `valkey-auth` | +| `queue.redis.auth.existingSecretKey` | Key within the Secret that holds the password | `password` | +| `queue.redis.auth.username` | Optional Redis ACL username | `""` | | `queue.redis.stream` | Redis stream name for audit events | `gitopsreverser.audit.events.v1` | | `queue.redis.maxLen` | Approximate stream max length (`0` disables trim) | `0` | | `queue.redis.tls.enabled` | Enable TLS for Redis connection | `false` | | `servers.metrics.bindAddress` | Metrics listener bind address | `:8080` | | `servers.metrics.tls.enabled` | Serve metrics with TLS | `false` | | `servers.metrics.tls.certPath` | Metrics TLS certificate mount path | `/tmp/k8s-metrics-server/metrics-server-certs` | -| `servers.metrics.tls.secretName` | Secret name for metrics TLS cert/key | `-metrics-server-cert` | +| `servers.metrics.tls.secretNameOverride` | Override Secret name for metrics TLS cert/key | `-metrics-server-cert` | | `service.clusterIP` | Optional fixed ClusterIP for single controller Service | `""` | | `service.ports.admission` | Service port for admission webhook | `9443` | | `service.ports.audit` | Service port for audit ingress | `9444` | | `service.ports.metrics` | Service port for metrics | `8080` | | `certificates.certManager.enabled` | Use cert-manager for certificates | `true` | +| `certificates.audit.enabled` | Let the chart manage audit TLS Secrets via cert-manager | `true` | +| `certificates.audit.rootCA.secretNameOverride` | Override Secret name for the audit root CA | `-audit-root-ca` | +| `certificates.audit.client.duration` | Lifetime of the kube-apiserver audit client cert; long by default to avoid repeated manual kube-apiserver reconfiguration | `87600h` | +| `certificates.audit.client.renewBefore` | Renew the kube-apiserver audit client cert before expiry; shorter values increase control-plane maintenance frequency | `720h` | +| `certificates.audit.client.secretNameOverride` | Override Secret name for the kube-apiserver audit client cert | `-audit-client-cert` | +| `auditKubeconfig.insecureSkipTLSVerify` | Render Helm notes with `insecure-skip-tls-verify: true` for the generated audit kubeconfig example | `false` | | `podDisruptionBudget.enabled` | Enable PodDisruptionBudget | `true` | | `resources.requests.cpu` | CPU request | `10m` | | `resources.requests.memory` | Memory request | `64Mi` | @@ -260,13 +249,13 @@ kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watch > ⚠️ **Warning**: Deleting CRDs will also delete all custom resources of those types! -## Verification & Usage +## Verification ### Verify Installation ```bash # Check pods (should see 1 replica) -kubectl get pods -n gitops-reverser-system +kubectl get pods -n gitops-reverser # Check CRDs kubectl get crd | grep configbutler @@ -276,18 +265,20 @@ kubectl get validatingwebhookconfiguration -l app.kubernetes.io/name=gitops-reve ``` +For first-run GitOps Reverser usage, follow the root quickstart instead of duplicating those steps here. + ### View Logs ```bash # All pods -kubectl logs -n gitops-reverser-system -l app.kubernetes.io/name=gitops-reverser -f +kubectl logs -n gitops-reverser -l app.kubernetes.io/name=gitops-reverser -f ``` ### Access Metrics ```bash -kubectl port-forward -n gitops-reverser-system svc/gitops-reverser 8080:8080 +kubectl port-forward -n gitops-reverser svc/gitops-reverser 8080:8080 curl http://localhost:8080/metrics # If metrics TLS is enabled: # curl -k https://localhost:8080/metrics @@ -300,7 +291,7 @@ curl http://localhost:8080/metrics ```bash helm upgrade gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system + --namespace gitops-reverser ``` ### Upgrade with New Values @@ -308,7 +299,7 @@ helm upgrade gitops-reverser \ ```bash helm upgrade gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system \ + --namespace gitops-reverser \ --values new-values.yaml ``` @@ -325,10 +316,10 @@ If upgrading from earlier chart versions: ```bash # Uninstall chart -helm uninstall gitops-reverser --namespace gitops-reverser-system +helm uninstall gitops-reverser --namespace gitops-reverser # Delete namespace (optional) -kubectl delete namespace gitops-reverser-system +kubectl delete namespace gitops-reverser # Delete CRDs (optional, but removes all custom resources) kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai @@ -341,8 +332,8 @@ kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watch Check certificate status: ```bash -kubectl get certificate -n gitops-reverser-system -kubectl describe certificate gitops-reverser-admission-server-cert -n gitops-reverser-system +kubectl get certificate -n gitops-reverser +kubectl describe certificate gitops-reverser-admission-server-cert -n gitops-reverser ``` If cert-manager is not working: @@ -366,32 +357,15 @@ kubectl get nodes # If you have only 1 node, keep a single replica or disable affinity helm upgrade gitops-reverser \ oci://ghcr.io/configbutler/charts/gitops-reverser \ - --namespace gitops-reverser-system \ + --namespace gitops-reverser \ --set replicaCount=1 \ --set affinity=null ``` -### Git Authentication Issues - -Ensure your Git credentials secret exists: - -```bash -# For HTTPS with token -kubectl create secret generic git-credentials \ - --from-literal=username=git \ - --from-literal=password=YOUR_TOKEN \ - -n gitops-reverser-system - -# For SSH -kubectl create secret generic git-credentials \ - --from-file=ssh-privatekey=~/.ssh/id_rsa \ - -n gitops-reverser-system -``` - ### View Controller Events ```bash -kubectl get events -n gitops-reverser-system --sort-by='.lastTimestamp' +kubectl get events -n gitops-reverser --sort-by='.lastTimestamp' ``` ## Advanced Configuration @@ -415,7 +389,7 @@ Create certificate secret manually: kubectl create secret tls gitops-reverser-admission-server-cert \ --cert=path/to/tls.crt \ --key=path/to/tls.key \ - -n gitops-reverser-system + -n gitops-reverser ``` ### Custom Resource Limits @@ -454,7 +428,7 @@ curl -s https://api.github.com/repos/ConfigButler/gitops-reverser/releases | jq ## Support & Contributing - **Issues**: [GitHub Issues](https://github.com/ConfigButler/gitops-reverser/issues) -- **Documentation**: [Project README](https://github.com/ConfigButler/gitops-reverser) +- **Documentation**: [Project README and quickstart](https://github.com/ConfigButler/gitops-reverser) - **Contributing**: See [CONTRIBUTING.md](https://github.com/ConfigButler/gitops-reverser/blob/main/CONTRIBUTING.md) ## License diff --git a/charts/gitops-reverser/templates/NOTES.txt b/charts/gitops-reverser/templates/NOTES.txt new file mode 100644 index 000000000..7da000031 --- /dev/null +++ b/charts/gitops-reverser/templates/NOTES.txt @@ -0,0 +1,129 @@ +{{- $fullname := include "gitops-reverser.fullname" . -}} +{{- $auditServiceName := include "gitops-reverser.auditServiceName" . -}} +{{- $auditRootCASecret := include "gitops-reverser.auditRootCASecretName" . -}} +{{- $auditClientSecret := include "gitops-reverser.auditClientSecretName" . -}} +{{- $auditServerPort := int .Values.service.ports.audit -}} +{{- $auditNodePort := .Values.auditService.nodePort -}} +{{- $auditClusterIP := .Values.auditService.clusterIP -}} +{{- $tlsServerName := printf "%s.%s.svc" $auditServiceName .Release.Namespace -}} +GitOps Reverser is installed in namespace `{{ .Release.Namespace }}`. + +What the chart did for you: +- deployed the controller and audit receiver +- created the audit Service (`{{ $auditServiceName }}`) +{{- if and .Values.servers.audit.tls.enabled .Values.certificates.certManager.enabled .Values.certificates.audit.enabled }} +- created the audit TLS materials + - root CA Secret: `{{ $auditRootCASecret }}` + - kube-apiserver client cert Secret: `{{ $auditClientSecret }}` +{{- else if .Values.servers.audit.tls.enabled }} +- expects audit TLS materials to exist with these Secret names + - root CA Secret: `{{ $auditRootCASecret }}` + - kube-apiserver client cert Secret: `{{ $auditClientSecret }}` +{{- else }} +- configured the audit listener without TLS (`servers.audit.tls.enabled=false`) +{{- end }} +{{- if .Values.quickstart.enabled }} +- created quickstart resources in namespace `{{ .Values.quickstart.namespace }}` + - GitProvider: `{{ .Values.quickstart.gitProvider.name }}` + - GitTarget: `{{ .Values.quickstart.gitTarget.name }}` + - WatchRule: `{{ .Values.quickstart.watchRule.name }}` +{{- end }} + +What the chart did NOT do for you: +- it did NOT write `/etc/kubernetes/...` files on your control-plane node(s) +- it did NOT create your audit policy file +- it did NOT edit kube-apiserver flags or manifests +- it did NOT restart kube-apiserver + +Install GitOps Reverser first. After that, generate the kube-apiserver audit webhook kubeconfig, copy it to every +control-plane node, configure kube-apiserver, and restart or reload kube-apiserver once. + +Recommended kube-apiserver settings: +- use `--audit-webhook-mode=batch` +- do NOT start with `blocking` or `blocking-strict` + +{{- if .Values.servers.audit.tls.enabled }} +Suggested audit webhook server URL from the control-plane node: +{{- if eq .Values.auditService.type "NodePort" }} +{{- if $auditNodePort }} +- `https://127.0.0.1:{{ $auditNodePort }}/audit-webhook/` +{{- else }} +- `https://127.0.0.1:/audit-webhook/` + Query the assigned nodePort with: + `kubectl get svc {{ $auditServiceName }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}'` +{{- end }} +{{- else if eq .Values.auditService.type "LoadBalancer" }} +- `https://:{{ $auditServerPort }}/audit-webhook/` + Query the address with: + `kubectl get svc {{ $auditServiceName }} -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'` +{{- else if $auditClusterIP }} +- `https://{{ $auditClusterIP }}:{{ $auditServerPort }}/audit-webhook/` +- Use this only if your control-plane node can route to the service CIDR. +{{- else }} +- Choose an address reachable from the control-plane node for service `{{ $auditServiceName }}`. +- `ClusterIP` alone is usually NOT enough unless your control-plane host can route to the service CIDR. +{{- end }} + +Copy-paste example to generate `audit-webhook.kubeconfig` after the Secrets exist: + +```bash +export NAMESPACE={{ .Release.Namespace }} +export AUDIT_ROOT_CA_SECRET={{ $auditRootCASecret }} +export AUDIT_CLIENT_SECRET={{ $auditClientSecret }} +export AUDIT_TLS_SERVER_NAME={{ $tlsServerName }} +export AUDIT_WEBHOOK_SERVER_URL={{- if and (eq .Values.auditService.type "NodePort") $auditNodePort }}https://127.0.0.1:{{ $auditNodePort }}/audit-webhook/{{- else if $auditClusterIP }}https://{{ $auditClusterIP }}:{{ $auditServerPort }}/audit-webhook/{{- else }}https:///audit-webhook/{{- end }} + +cat > audit-webhook.kubeconfig </audit-webhook/ +insecure-skip-tls-verify: true +``` + +{{- end }} + +After generating the kubeconfig: +1. Copy `audit-webhook.kubeconfig` to every control-plane node. +2. Create or update your audit policy file. +3. Start kube-apiserver with: + - `--audit-policy-file=/path/to/audit-policy.yaml` + - `--audit-webhook-config-file=/path/to/audit-webhook.kubeconfig` + - `--audit-webhook-mode=batch` +4. Restart or reload kube-apiserver once. + +Further guidance: +- `https://github.com/ConfigButler/gitops-reverser/blob/main/docs/design/audit-webhook-api-server-connectivity.md` +- `https://github.com/ConfigButler/gitops-reverser/blob/main/docs/design/audit-webhook-tls-design.md` diff --git a/charts/gitops-reverser/templates/_helpers.tpl b/charts/gitops-reverser/templates/_helpers.tpl index b0b20256a..d62eb2c3a 100644 --- a/charts/gitops-reverser/templates/_helpers.tpl +++ b/charts/gitops-reverser/templates/_helpers.tpl @@ -64,3 +64,31 @@ Create the name of the service account to use {{- fail "serviceAccount.name must be set when serviceAccount.create=false" }} {{- end }} {{- end }} + +{{/* +Create the name of the audit service. +*/}} +{{- define "gitops-reverser.auditServiceName" -}} +{{- printf "%s-audit" (include "gitops-reverser.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Default Secret name for the audit root CA. +*/}} +{{- define "gitops-reverser.auditRootCASecretName" -}} +{{- .Values.certificates.audit.rootCA.secretNameOverride | default (printf "%s-audit-root-ca" (include "gitops-reverser.fullname" .)) -}} +{{- end }} + +{{/* +Default Secret name for the audit server certificate. +*/}} +{{- define "gitops-reverser.auditServerSecretName" -}} +{{- .Values.servers.audit.tls.secretNameOverride | default (printf "%s-audit-server-cert" (include "gitops-reverser.fullname" .)) -}} +{{- end }} + +{{/* +Default Secret name for the kube-apiserver audit client certificate. +*/}} +{{- define "gitops-reverser.auditClientSecretName" -}} +{{- .Values.certificates.audit.client.secretNameOverride | default (printf "%s-audit-client-cert" (include "gitops-reverser.fullname" .)) -}} +{{- end }} diff --git a/charts/gitops-reverser/templates/admission-webhook.yaml b/charts/gitops-reverser/templates/admission-webhook.yaml index ec2b6a973..f6a23114b 100644 --- a/charts/gitops-reverser/templates/admission-webhook.yaml +++ b/charts/gitops-reverser/templates/admission-webhook.yaml @@ -1,4 +1,3 @@ -{{- if .Values.webhook.validating.enabled }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -17,32 +16,20 @@ webhooks: name: {{ include "gitops-reverser.fullname" . }} namespace: {{ .Release.Namespace }} port: {{ .Values.service.ports.admission }} - path: /process-validating-webhook + path: /validate-configbutler-ai-v1alpha1-gittarget {{- if not .Values.certificates.certManager.enabled }} caBundle: {{ .Values.webhook.caBundle | b64enc }} {{- end }} - failurePolicy: {{ .Values.webhook.validating.failurePolicy }} - name: gitops-reverser.configbutler.ai + failurePolicy: Fail + name: vgittarget.kb.io rules: - {{- range .Values.webhook.validating.rules }} - - operations: {{ .operations | toJson }} - apiGroups: {{ .apiGroups | toJson }} - apiVersions: {{ .apiVersions | toJson }} - resources: {{ .resources | toJson }} - {{- if .scope }} - scope: {{ .scope }} - {{- end }} - {{- end }} - sideEffects: {{ .Values.webhook.validating.sideEffects }} - {{- with .Values.webhook.validating.namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.webhook.validating.objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.webhook.validating.timeoutSeconds }} - timeoutSeconds: {{ .Values.webhook.validating.timeoutSeconds }} - {{- end }} -{{- end }} + - apiGroups: + - configbutler.ai + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - gittargets + sideEffects: None diff --git a/charts/gitops-reverser/templates/audit-certificates.yaml b/charts/gitops-reverser/templates/audit-certificates.yaml new file mode 100644 index 000000000..06178cf36 --- /dev/null +++ b/charts/gitops-reverser/templates/audit-certificates.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.certificates.certManager.enabled .Values.certificates.audit.enabled .Values.servers.audit.tls.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "gitops-reverser.fullname" . }}-audit-root-ca + namespace: {{ .Release.Namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} +spec: + commonName: {{ include "gitops-reverser.fullname" . }}-audit-root-ca + duration: 87600h + isCA: true + issuerRef: + kind: {{ .Values.certificates.certManager.issuer.kind }} + name: {{ .Values.certificates.certManager.issuer.name }} + privateKey: + algorithm: ECDSA + rotationPolicy: Always + size: 256 + secretName: {{ include "gitops-reverser.auditRootCASecretName" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "gitops-reverser.fullname" . }}-audit-ca-issuer + namespace: {{ .Release.Namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} +spec: + ca: + secretName: {{ include "gitops-reverser.auditRootCASecretName" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "gitops-reverser.fullname" . }}-audit-server-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "gitops-reverser.auditServiceName" . }}.{{ .Release.Namespace }}.svc + - {{ include "gitops-reverser.auditServiceName" . }}.{{ .Release.Namespace }}.svc.cluster.local + duration: 2160h + issuerRef: + kind: Issuer + name: {{ include "gitops-reverser.fullname" . }}-audit-ca-issuer + renewBefore: 360h + secretName: {{ include "gitops-reverser.auditServerSecretName" . }} + usages: + - digital signature + - key encipherment + - server auth + privateKey: + rotationPolicy: Always +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "gitops-reverser.fullname" . }}-audit-client-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} +spec: + commonName: kube-apiserver + duration: {{ .Values.certificates.audit.client.duration | quote }} + issuerRef: + kind: Issuer + name: {{ include "gitops-reverser.fullname" . }}-audit-ca-issuer + renewBefore: {{ .Values.certificates.audit.client.renewBefore | quote }} + secretName: {{ include "gitops-reverser.auditClientSecretName" . }} + usages: + - digital signature + - client auth + privateKey: + rotationPolicy: Always +{{- end }} diff --git a/charts/gitops-reverser/templates/certificates.yaml b/charts/gitops-reverser/templates/certificates.yaml index 8e6ad918e..f685adce6 100644 --- a/charts/gitops-reverser/templates/certificates.yaml +++ b/charts/gitops-reverser/templates/certificates.yaml @@ -26,41 +26,13 @@ spec: issuerRef: kind: {{ .Values.certificates.certManager.issuer.kind }} name: {{ .Values.certificates.certManager.issuer.name }} - secretName: {{ .Values.servers.admission.tls.secretName | default (printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .)) }} + secretName: {{ .Values.servers.admission.tls.secretNameOverride | default (printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .)) }} usages: - digital signature - key encipherment - server auth privateKey: rotationPolicy: Always -{{- if .Values.servers.audit.tls.enabled }} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "gitops-reverser.fullname" . }}-audit-server-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "gitops-reverser.labels" . | nindent 4 }} -spec: -{{- if .Values.service.clusterIP }} - ipAddresses: - - {{ .Values.service.clusterIP }} -{{- end }} - dnsNames: - - {{ include "gitops-reverser.fullname" . }}.{{ .Release.Namespace }}.svc - - {{ include "gitops-reverser.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - issuerRef: - kind: {{ .Values.certificates.certManager.issuer.kind }} - name: {{ .Values.certificates.certManager.issuer.name }} - secretName: {{ .Values.servers.audit.tls.secretName | default (printf "%s-audit-server-cert" (include "gitops-reverser.fullname" .)) }} - usages: - - digital signature - - key encipherment - - server auth - privateKey: - rotationPolicy: Always -{{- end }} {{- if .Values.servers.metrics.tls.enabled }} --- apiVersion: cert-manager.io/v1 @@ -77,7 +49,7 @@ spec: issuerRef: kind: {{ .Values.certificates.certManager.issuer.kind }} name: {{ .Values.certificates.certManager.issuer.name }} - secretName: {{ .Values.servers.metrics.tls.secretName | default (printf "%s-metrics-server-cert" (include "gitops-reverser.fullname" .)) }} + secretName: {{ .Values.servers.metrics.tls.secretNameOverride | default (printf "%s-metrics-server-cert" (include "gitops-reverser.fullname" .)) }} usages: - digital signature - key encipherment diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index cd9a4daff..c81fe0d52 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -7,7 +7,7 @@ metadata: {{- include "gitops-reverser.labels" . | nindent 4 }} app.kubernetes.io/component: manager app.kubernetes.io/part-of: gitops-reverser - control-plane: controller-manager + control-plane: gitops-reverser spec: replicas: {{ .Values.replicaCount }} strategy: @@ -24,6 +24,7 @@ spec: {{- end }} labels: {{- include "gitops-reverser.selectorLabels" . | nindent 8 }} + control-plane: gitops-reverser spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -60,6 +61,8 @@ spec: - --audit-cert-path={{ .Values.servers.audit.tls.certPath }} - --audit-cert-name={{ .Values.servers.audit.tls.certName }} - --audit-cert-key={{ .Values.servers.audit.tls.certKey }} + - --audit-client-ca-path={{ .Values.servers.audit.tls.clientCAPath }} + - --audit-client-ca-name={{ .Values.servers.audit.tls.clientCAName }} {{- else }} - --audit-insecure {{- end }} @@ -69,18 +72,16 @@ spec: - --audit-read-timeout={{ .Values.servers.audit.timeouts.read }} - --audit-write-timeout={{ .Values.servers.audit.timeouts.write }} - --audit-idle-timeout={{ .Values.servers.audit.timeouts.idle }} - {{- if .Values.queue.redis.enabled }} - - --audit-redis-enabled - --audit-redis-addr={{ .Values.queue.redis.addr }} - - --audit-redis-username={{ .Values.queue.redis.username | quote }} - - --audit-redis-password={{ .Values.queue.redis.password | quote }} + {{- with .Values.queue.redis.auth.username }} + - --audit-redis-username={{ . | quote }} + {{- end }} - --audit-redis-db={{ .Values.queue.redis.db }} - --audit-redis-stream={{ .Values.queue.redis.stream }} - --audit-redis-max-len={{ .Values.queue.redis.maxLen }} {{- if .Values.queue.redis.tls.enabled }} - --audit-redis-tls {{- end }} - {{- end }} {{- if .Values.logging.level }} - --zap-log-level={{ .Values.logging.level }} {{- end }} @@ -115,6 +116,13 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- if .Values.queue.redis.auth.existingSecret }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.queue.redis.auth.existingSecret }} + key: {{ .Values.queue.redis.auth.existingSecretKey | default "password" }} + {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -153,6 +161,9 @@ spec: - name: audit-cert mountPath: {{ .Values.servers.audit.tls.certPath }} readOnly: true + - name: audit-client-ca + mountPath: {{ .Values.servers.audit.tls.clientCAPath }} + readOnly: true {{- end }} {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} @@ -167,19 +178,23 @@ spec: {{- if .Values.servers.admission.tls.enabled }} - name: admission-cert secret: - secretName: {{ .Values.servers.admission.tls.secretName | default (printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .)) }} + secretName: {{ .Values.servers.admission.tls.secretNameOverride | default (printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .)) }} defaultMode: 420 {{- end }} {{- if .Values.servers.metrics.tls.enabled }} - name: metrics-cert secret: - secretName: {{ .Values.servers.metrics.tls.secretName | default (printf "%s-metrics-server-cert" (include "gitops-reverser.fullname" .)) }} + secretName: {{ .Values.servers.metrics.tls.secretNameOverride | default (printf "%s-metrics-server-cert" (include "gitops-reverser.fullname" .)) }} defaultMode: 420 {{- end }} {{- if .Values.servers.audit.tls.enabled }} - name: audit-cert secret: - secretName: {{ .Values.servers.audit.tls.secretName | default (printf "%s-audit-server-cert" (include "gitops-reverser.fullname" .)) }} + secretName: {{ include "gitops-reverser.auditServerSecretName" . }} + defaultMode: 420 + - name: audit-client-ca + secret: + secretName: {{ include "gitops-reverser.auditRootCASecretName" . }} defaultMode: 420 {{- end }} {{- with .Values.volumes }} diff --git a/charts/gitops-reverser/templates/quickstart.yaml b/charts/gitops-reverser/templates/quickstart.yaml new file mode 100644 index 000000000..2b449bcc8 --- /dev/null +++ b/charts/gitops-reverser/templates/quickstart.yaml @@ -0,0 +1,57 @@ +{{- if .Values.quickstart.enabled }} +--- +apiVersion: configbutler.ai/v1alpha1 +kind: GitProvider +metadata: + name: {{ .Values.quickstart.gitProvider.name }} + namespace: {{ .Values.quickstart.namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} + app.kubernetes.io/component: quickstart +spec: + url: {{ .Values.quickstart.gitProvider.url | quote }} + allowedBranches: + {{- toYaml .Values.quickstart.gitProvider.allowedBranches | nindent 4 }} + secretRef: + name: {{ .Values.quickstart.gitProvider.secretRef.name }} + push: + interval: {{ .Values.quickstart.gitProvider.push.interval | quote }} + maxCommits: {{ .Values.quickstart.gitProvider.push.maxCommits }} +--- +apiVersion: configbutler.ai/v1alpha1 +kind: GitTarget +metadata: + name: {{ .Values.quickstart.gitTarget.name }} + namespace: {{ .Values.quickstart.namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} + app.kubernetes.io/component: quickstart +spec: + providerRef: + name: {{ .Values.quickstart.gitProvider.name }} + branch: {{ .Values.quickstart.gitTarget.branch | quote }} + path: {{ .Values.quickstart.gitTarget.path | quote }} + encryption: + provider: {{ .Values.quickstart.gitTarget.encryption.provider | quote }} + age: + enabled: {{ .Values.quickstart.gitTarget.encryption.age.enabled }} + recipients: + extractFromSecret: true + generateWhenMissing: {{ .Values.quickstart.gitTarget.encryption.age.generateWhenMissing }} + secretRef: + name: {{ .Values.quickstart.gitTarget.encryption.secretRef.name }} +--- +apiVersion: configbutler.ai/v1alpha1 +kind: WatchRule +metadata: + name: {{ .Values.quickstart.watchRule.name }} + namespace: {{ .Values.quickstart.namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} + app.kubernetes.io/component: quickstart +spec: + targetRef: + name: {{ .Values.quickstart.gitTarget.name }} + rules: + {{- toYaml .Values.quickstart.watchRule.rules | nindent 4 }} +{{- end }} diff --git a/charts/gitops-reverser/templates/services.yaml b/charts/gitops-reverser/templates/services.yaml index 063c787d6..0bba03ec1 100644 --- a/charts/gitops-reverser/templates/services.yaml +++ b/charts/gitops-reverser/templates/services.yaml @@ -19,13 +19,35 @@ spec: port: {{ .Values.service.ports.admission }} targetPort: {{ .Values.servers.admission.port }} protocol: TCP - - name: audit - port: {{ .Values.service.ports.audit }} - targetPort: {{ .Values.servers.audit.port }} - protocol: TCP - name: metrics port: {{ .Values.service.ports.metrics }} targetPort: {{ .Values.servers.metrics.port }} protocol: TCP selector: {{- include "gitops-reverser.selectorLabels" . | nindent 4 }} + control-plane: gitops-reverser +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gitops-reverser.auditServiceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "gitops-reverser.labels" . | nindent 4 }} + app.kubernetes.io/component: audit +spec: + type: {{ .Values.auditService.type }} + {{- if .Values.auditService.clusterIP }} + clusterIP: {{ .Values.auditService.clusterIP }} + {{- end }} + ports: + - name: audit + port: {{ .Values.service.ports.audit }} + targetPort: {{ .Values.servers.audit.port }} + protocol: TCP + {{- if and (eq .Values.auditService.type "NodePort") .Values.auditService.nodePort }} + nodePort: {{ .Values.auditService.nodePort }} + {{- end }} + selector: + {{- include "gitops-reverser.selectorLabels" . | nindent 4 }} + control-plane: gitops-reverser diff --git a/charts/gitops-reverser/templates/validate-quickstart.yaml b/charts/gitops-reverser/templates/validate-quickstart.yaml new file mode 100644 index 000000000..c3168abae --- /dev/null +++ b/charts/gitops-reverser/templates/validate-quickstart.yaml @@ -0,0 +1,23 @@ +{{- if .Values.quickstart.enabled -}} +{{- if eq (trim .Values.quickstart.namespace) "" -}} +{{- fail "quickstart.namespace must not be empty when quickstart.enabled=true" -}} +{{- end -}} +{{- if eq (trim .Values.quickstart.gitProvider.url) "" -}} +{{- fail "quickstart.gitProvider.url must be set when quickstart.enabled=true" -}} +{{- end -}} +{{- if eq (trim .Values.quickstart.gitProvider.secretRef.name) "" -}} +{{- fail "quickstart.gitProvider.secretRef.name must be set when quickstart.enabled=true" -}} +{{- end -}} +{{- if eq (trim .Values.quickstart.gitTarget.branch) "" -}} +{{- fail "quickstart.gitTarget.branch must be set when quickstart.enabled=true" -}} +{{- end -}} +{{- if eq (trim .Values.quickstart.gitTarget.path) "" -}} +{{- fail "quickstart.gitTarget.path must be set when quickstart.enabled=true" -}} +{{- end -}} +{{- if eq (trim .Values.quickstart.gitTarget.encryption.secretRef.name) "" -}} +{{- fail "quickstart.gitTarget.encryption.secretRef.name must be set when quickstart.enabled=true" -}} +{{- end -}} +{{- if not .Values.quickstart.watchRule.rules -}} +{{- fail "quickstart.watchRule.rules must contain at least one rule when quickstart.enabled=true" -}} +{{- end -}} +{{- end -}} diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index b346ef7b8..6dd502786 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -69,7 +69,8 @@ servers: certPath: "/tmp/k8s-admission-server/admission-server-certs" certName: "tls.crt" certKey: "tls.key" - secretName: "" + # Leave empty to use the chart-generated Secret name. + secretNameOverride: "" audit: listenAddress: 0.0.0.0 @@ -80,7 +81,10 @@ servers: certPath: "/tmp/k8s-audit-server/audit-server-certs" certName: "tls.crt" certKey: "tls.key" - secretName: "" + clientCAPath: "/tmp/k8s-audit-server/audit-client-ca" + clientCAName: "tls.crt" + # Leave empty to use the chart-generated Secret name. + secretNameOverride: "" timeouts: read: "15s" write: "30s" @@ -96,47 +100,33 @@ servers: certPath: "/tmp/k8s-metrics-server/metrics-server-certs" certName: "tls.crt" certKey: "tls.key" - secretName: "" + # Leave empty to use the chart-generated Secret name. + secretNameOverride: "" # Webhook behavior webhook: audit: # Set to true if you want to write events to /var/run/audit-dumps debugDumps: false - - # Validating webhook configuration - validating: - enabled: true - failurePolicy: Ignore - sideEffects: None - admissionReviewVersions: ["v1", "v1beta1"] - # Rules for validating webhook (by default watching all resources) - rules: - - operations: ["CREATE", "UPDATE", "DELETE"] - apiGroups: ["*"] - apiVersions: ["*"] - resources: ["*"] - # Namespace selector for validating webhook - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kube-public - - kube-node-lease - # Object selector for validating webhook - objectSelector: {} + # CA bundle used when cert-manager is disabled (base64-encoded PEM). + # Required only when certificates.certManager.enabled=false. + caBundle: "" # Durable audit queue configuration. queue: redis: - # Enable Redis Stream enqueueing for accepted audit events. - enabled: false - # Redis endpoint in host:port format. - addr: "" - username: "" - password: "" + # Redis endpoint in host:port format. GitOps Reverser requires a reachable Valkey/Redis service. + addr: "valkey:6379" + auth: + # Name of a pre-existing Secret in the same namespace that holds the Redis password. + # The Secret must be created before installing this chart. + # Required for production deployments. You may leave it empty only for dev/local environments + # where Valkey runs without authentication. + existingSecret: "valkey-auth" + # Key within the Secret that holds the password value. + existingSecretKey: "password" + # Optional Redis username for ACL-based auth (Redis 6+). Not sensitive; stays as a plain value. + username: "" db: 0 stream: "gitopsreverser.audit.events.v1" # Approximate max stream length; 0 means no trimming. @@ -153,6 +143,26 @@ certificates: name: selfsigned-issuer kind: Issuer create: true + audit: + # When true, the chart creates audit root CA, audit server cert, and + # kube-apiserver client cert resources via cert-manager. Set to false if + # you want to provide those Secrets yourself. + enabled: true + rootCA: + # Leave empty to use the chart-generated Secret name. + secretNameOverride: "" + client: + # Keep the kube-apiserver client cert long-lived by default so cluster admins + # are protected from frequent manual audit-webhook kubeconfig updates and + # kube-apiserver reconfiguration. cert-manager durations use Go-style units, + # so years must be expressed in hours (10y = 87600h). + duration: "87600h" + # Renew 30 days before expiry. Shortening this cadence increases control-plane + # maintenance because kube-apiserver embeds this client credential in its + # node-local audit webhook kubeconfig. + renewBefore: "720h" + # Leave empty to use the chart-generated Secret name. + secretNameOverride: "" # RBAC configuration rbac: @@ -215,6 +225,66 @@ service: audit: 9444 metrics: 8080 +auditService: + type: NodePort + nodePort: 30444 + # Optional fixed ClusterIP for the dedicated audit Service. Useful for + # "Option 1" bootstrap models where kube-apiserver talks directly to a pinned + # Service IP and the control-plane host can route to the service CIDR. + clusterIP: "" + +# Guidance-only values used by Helm NOTES.txt for kube-apiserver follow-up. +auditKubeconfig: + # When true, the generated example kubeconfig in Helm notes uses + # insecure-skip-tls-verify: true instead of a CA bundle. This is useful for + # local/bootstrap scenarios only and is not recommended for production. + insecureSkipTLSVerify: false + +# Optional starter resources for first-commit validation. +# Disabled by default so the chart does not create example Git objects unless +# you explicitly opt in. +quickstart: + enabled: false + # Namespace for the starter GitProvider, GitTarget, and WatchRule. + # This namespace must already exist. + namespace: default + gitProvider: + name: example-provider + # Required when quickstart.enabled=true. + url: "" + allowedBranches: + - "*" + secretRef: + # Required when quickstart.enabled=true. + name: git-creds + push: + interval: "5s" + maxCommits: 10 + gitTarget: + name: example-target + branch: main + path: live-cluster + encryption: + provider: sops + age: + enabled: true + generateWhenMissing: true + secretRef: + name: sops-age-key + watchRule: + name: example-watchrule + rules: + - operations: + - CREATE + - UPDATE + - DELETE + apiGroups: + - "" + apiVersions: + - v1 + resources: + - configmaps + # Logging configuration logging: level: info diff --git a/cmd/main.go b/cmd/main.go index 8e0aa38e8..441393cc3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ package main import ( "context" "crypto/tls" + "crypto/x509" "errors" "flag" "fmt" @@ -46,11 +47,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" configbutleraiv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" "github.com/ConfigButler/gitops-reverser/internal/controller" - "github.com/ConfigButler/gitops-reverser/internal/correlation" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/reconcile" @@ -67,9 +66,6 @@ var ( ) const ( - // Correlation store configuration. - correlationMaxEntries = 10000 - correlationTTL = 5 * time.Minute flagParseFailureExitCode = 2 defaultAuditPort = 9444 defaultAuditMaxBodyBytes = int64(10 * 1024 * 1024) @@ -91,6 +87,14 @@ func main() { cfg := parseFlags() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&cfg.zapOpts))) + if cfg.auditRedisPassword == "" { + setupLog.Info( + "no Redis password configured — "+ + "connecting to Valkey without authentication is not recommended for production deployments", + "hint", "set REDIS_PASSWORD env var via queue.redis.auth.existingSecret in the Helm chart", + ) + } + // Log metrics configuration for debugging setupLog.Info("Metrics configuration", "metrics-bind-address", cfg.metricsAddr, @@ -128,15 +132,6 @@ func main() { fatalIfErr(mgr.Add(workerManager), "unable to add worker manager to manager") setupLog.Info("WorkerManager initialized and added to manager") - // Initialize correlation store for webhook→watch enrichment - correlationStore := correlation.NewStore(correlationTTL, correlationMaxEntries) - correlationStore.SetEvictionCallback(func() { - telemetry.KVEvictionsTotal.Add(context.Background(), 1) - }) - setupLog.Info("Correlation store initialized", - "ttl", correlationTTL, - "maxEntries", correlationMaxEntries) - // Create ReconcilerManager (will be set up as ControlEventEmitter) reconcilerManager := reconcile.NewReconcilerManager( nil, // eventRouter will be set after EventRouter is created @@ -146,11 +141,11 @@ func main() { // Watch ingestion manager (placeholder, will get EventRouter set later) watchMgr := &watch.Manager{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("watch"), - RuleStore: ruleStore, - EventRouter: nil, // Will be set below - CorrelationStore: correlationStore, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("watch"), + RuleStore: ruleStore, + EventRouter: nil, // Will be set below + AuditLiveEventsEnabled: true, } // Initialize EventRouter with all dependencies @@ -183,46 +178,54 @@ func main() { WatchManager: watchMgr, }).SetupWithManager(mgr), "unable to create controller", "controller", "ClusterWatchRule") - // Webhook handler (correlation storage only - stores ALL resources, watch filters by rules) - eventHandler := &webhookhandler.EventHandler{ - Client: mgr.GetClient(), - CorrelationStore: correlationStore, - } - - // Create and inject decoder for generic Kubernetes resource handling - decoder := admission.NewDecoder(scheme) - fatalIfErr(eventHandler.InjectDecoder(&decoder), "unable to inject decoder into webhook handler") - setupLog.Info("Generic unstructured decoder injected - ready to handle all Kubernetes resource types") - - // Register event correlation webhook - validatingWebhook := &admission.Webhook{Handler: eventHandler} - mgr.GetWebhookServer().Register("/process-validating-webhook", validatingWebhook) - setupLog.Info("Event correlation webhook handler registered", "path", "/process-validating-webhook") - // Register GitTarget validator webhook (prevents duplicate targets) fatalIfErr(webhookhandler.SetupGitTargetValidatorWebhook(mgr), "unable to setup GitTarget validator webhook") setupLog.Info("GitTarget validator webhook registered - enforcing uniqueness constraint") - // Register experimental audit webhook for metrics collection - var auditQueue webhookhandler.AuditEventQueue - if cfg.auditRedisEnabled { - auditQueue, err = queue.NewRedisAuditQueue(queue.RedisAuditQueueConfig{ + // Register audit webhook receiver for metrics collection + auditQueue, err := queue.NewRedisAuditQueue(queue.RedisAuditQueueConfig{ + Addr: cfg.auditRedisAddr, + Username: cfg.auditRedisUsername, + AuthValue: cfg.auditRedisPassword, + DB: cfg.auditRedisDB, + Stream: cfg.auditRedisStream, + MaxLen: cfg.auditRedisMaxLen, + TLSEnabled: cfg.auditRedisTLS, + }) + fatalIfErr(err, "unable to initialize audit redis queue") + setupLog.Info("Audit Redis queue configured", + "redis-address", cfg.auditRedisAddr, + "stream", cfg.auditRedisStream, + "db", cfg.auditRedisDB, + "tls-enabled", cfg.auditRedisTLS) + + // Register the audit stream consumer. It shares the same Redis config as the + // producer but uses a dedicated consumer group and ID (pod-name-scoped so that + // multiple replicas do not share a consumer identity). + consumerID := os.Getenv("POD_NAME") + if consumerID == "" { + consumerID = "gitopsreverser-consumer-0" + } + auditConsumer, consumerErr := queue.NewAuditConsumer( + queue.AuditConsumerConfig{ Addr: cfg.auditRedisAddr, Username: cfg.auditRedisUsername, AuthValue: cfg.auditRedisPassword, DB: cfg.auditRedisDB, Stream: cfg.auditRedisStream, - MaxLen: cfg.auditRedisMaxLen, TLSEnabled: cfg.auditRedisTLS, - }) - fatalIfErr(err, "unable to initialize audit redis queue") - setupLog.Info("Audit Redis queue enabled", - "redis-address", cfg.auditRedisAddr, - "stream", cfg.auditRedisStream, - "db", cfg.auditRedisDB, - "tls-enabled", cfg.auditRedisTLS) - } + ConsumerID: consumerID, + }, + ruleStore, + eventRouter, + ctrl.Log.WithName("audit-consumer"), + ) + fatalIfErr(consumerErr, "unable to initialize audit stream consumer") + fatalIfErr(mgr.Add(auditConsumer), "unable to add audit stream consumer to manager") + setupLog.Info("Audit stream consumer registered", + "stream", cfg.auditRedisStream, + "consumer-id", consumerID) auditHandler, err := webhookhandler.NewAuditHandler(webhookhandler.AuditHandlerConfig{ DumpDir: cfg.auditDumpPath, @@ -306,12 +309,13 @@ type appConfig struct { auditCertPath string auditCertName string auditCertKey string + auditClientCAPath string + auditClientCAName string auditInsecure bool auditMaxRequestBodyBytes int64 auditReadTimeout time.Duration auditWriteTimeout time.Duration auditIdleTimeout time.Duration - auditRedisEnabled bool auditRedisAddr string auditRedisUsername string auditRedisPassword string @@ -359,6 +363,10 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "IP address for the dedicated audit ingress HTTPS server.") fs.IntVar(&cfg.auditPort, "audit-port", defaultAuditPort, "Port for the dedicated audit ingress HTTPS server.") bindServerCertFlags(fs, "audit", "audit ingress TLS", &cfg.auditCertPath, &cfg.auditCertName, &cfg.auditCertKey) + fs.StringVar(&cfg.auditClientCAPath, "audit-client-ca-path", "/tmp/k8s-audit-server/audit-client-ca", + "Directory that contains the audit client CA certificate used to verify kube-apiserver client certificates.") + fs.StringVar(&cfg.auditClientCAName, "audit-client-ca-name", "tls.crt", + "File name of the audit client CA certificate used to verify kube-apiserver client certificates.") fs.BoolVar(&cfg.auditInsecure, "audit-insecure", false, "If set, the audit ingress endpoint is served via HTTP instead of HTTPS.") fs.Int64Var(&cfg.auditMaxRequestBodyBytes, "audit-max-request-body-bytes", defaultAuditMaxBodyBytes, @@ -369,14 +377,16 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "Write timeout for the dedicated audit ingress HTTPS server.") fs.DurationVar(&cfg.auditIdleTimeout, "audit-idle-timeout", defaultAuditIdleTimeout, "Idle timeout for the dedicated audit ingress HTTPS server.") - fs.BoolVar(&cfg.auditRedisEnabled, "audit-redis-enabled", false, - "If set, accepted audit events are enqueued to Redis Streams for durable buffering.") - fs.StringVar(&cfg.auditRedisAddr, "audit-redis-addr", "", + fs.StringVar(&cfg.auditRedisAddr, "audit-redis-addr", "valkey:6379", "Redis server address (:) for audit event queueing.") fs.StringVar(&cfg.auditRedisUsername, "audit-redis-username", "", "Optional Redis username for audit event queueing.") - fs.StringVar(&cfg.auditRedisPassword, "audit-redis-password", "", - "Optional Redis password for audit event queueing.") + fs.StringVar( + &cfg.auditRedisPassword, + "audit-redis-password", + os.Getenv("REDIS_PASSWORD"), + "Redis password for audit event queueing. Prefer setting via REDIS_PASSWORD env var from a Kubernetes Secret.", + ) fs.IntVar(&cfg.auditRedisDB, "audit-redis-db", 0, "Redis database index for audit event queueing.") fs.StringVar(&cfg.auditRedisStream, "audit-redis-stream", queue.DefaultRedisAuditStream, @@ -445,8 +455,11 @@ func validateAuditConfig(cfg appConfig) error { if cfg.auditIdleTimeout <= 0 { return fmt.Errorf("audit-idle-timeout must be > 0, got %s", cfg.auditIdleTimeout) } - if cfg.auditRedisEnabled && strings.TrimSpace(cfg.auditRedisAddr) == "" { - return errors.New("audit-redis-addr is required when audit-redis-enabled is true") + if strings.TrimSpace(cfg.auditRedisAddr) == "" { + return errors.New("audit-redis-addr is required") + } + if !cfg.auditInsecure && strings.TrimSpace(cfg.auditClientCAPath) == "" { + return errors.New("audit-client-ca-path is required when audit TLS is enabled") } if cfg.auditRedisDB < 0 { return fmt.Errorf("audit-redis-db must be >= 0, got %d", cfg.auditRedisDB) @@ -549,7 +562,7 @@ func (r *auditServerRunnable) Start(ctx context.Context) error { go func() { defer close(shutdownDone) <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), defaultAuditShutdownTimeout) + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultAuditShutdownTimeout) defer cancel() if err := r.server.Shutdown(shutdownCtx); err != nil { setupLog.Error(err, "Failed to shutdown dedicated audit ingress server") @@ -584,7 +597,10 @@ func initAuditServerRunnable( var serverTLS *tls.Config if tlsEnabled { - serverTLS = buildServerTLSConfig(tlsOpts) + serverTLS, err = buildAuditServerTLSConfig(cfg, tlsOpts) + if err != nil { + return nil, nil, err + } } else { setupLog.Info("Audit ingress TLS disabled; serving plain HTTP for audit ingress") } @@ -626,6 +642,20 @@ func buildServerTLSConfig(tlsOpts []func(*tls.Config)) *tls.Config { return serverTLS } +func buildAuditServerTLSConfig(cfg appConfig, tlsOpts []func(*tls.Config)) (*tls.Config, error) { + serverTLS := buildServerTLSConfig(tlsOpts) + + clientCAPool, err := loadCertPoolFromPEMFile(filepath.Join(cfg.auditClientCAPath, cfg.auditClientCAName)) + if err != nil { + return nil, fmt.Errorf("failed to load audit client CA: %w", err) + } + + serverTLS.ClientAuth = tls.RequireAndVerifyClientCert + serverTLS.ClientCAs = clientCAPool + + return serverTLS, nil +} + func buildTLSRuntime( tlsEnabled bool, requireCert bool, @@ -680,6 +710,20 @@ func newCertWatcher(certPath, certName, certKey string) (*certwatcher.CertWatche ) } +func loadCertPoolFromPEMFile(path string) (*x509.CertPool, error) { + pemData, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read certificate bundle %q: %w", path, err) + } + + pool := x509.NewCertPool() + if ok := pool.AppendCertsFromPEM(pemData); !ok { + return nil, fmt.Errorf("parse certificate bundle %q: no certificates found", path) + } + + return pool, nil +} + // newManager creates a new controller-runtime Manager with common options. func newManager( metricsOptions metricsserver.Options, diff --git a/cmd/main_audit_server_test.go b/cmd/main_audit_server_test.go index 05a7add02..a197efc5d 100644 --- a/cmd/main_audit_server_test.go +++ b/cmd/main_audit_server_test.go @@ -17,9 +17,19 @@ package main import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "flag" + "math/big" "net/http" "net/http/httptest" + "os" + "path/filepath" "testing" "time" @@ -38,12 +48,13 @@ func TestParseFlagsWithArgs_Defaults(t *testing.T) { assert.False(t, cfg.auditInsecure) assert.Equal(t, "0.0.0.0", cfg.auditListenAddress) assert.Equal(t, 9444, cfg.auditPort) + assert.Equal(t, "/tmp/k8s-audit-server/audit-client-ca", cfg.auditClientCAPath) + assert.Equal(t, "tls.crt", cfg.auditClientCAName) assert.Equal(t, int64(10485760), cfg.auditMaxRequestBodyBytes) assert.Equal(t, 15*time.Second, cfg.auditReadTimeout) assert.Equal(t, 30*time.Second, cfg.auditWriteTimeout) assert.Equal(t, 60*time.Second, cfg.auditIdleTimeout) - assert.False(t, cfg.auditRedisEnabled) - assert.Empty(t, cfg.auditRedisAddr) + assert.Equal(t, "valkey:6379", cfg.auditRedisAddr) assert.Equal(t, "gitopsreverser.audit.events.v1", cfg.auditRedisStream) assert.Equal(t, int64(0), cfg.auditRedisMaxLen) assert.False(t, cfg.auditRedisTLS) @@ -70,11 +81,12 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { "--audit-cert-path=/tmp/audit-certs", "--audit-cert-name=cert.pem", "--audit-cert-key=key.pem", + "--audit-client-ca-path=/tmp/audit-ca", + "--audit-client-ca-name=ca.pem", "--audit-max-request-body-bytes=2048", "--audit-read-timeout=5s", "--audit-write-timeout=8s", "--audit-idle-timeout=13s", - "--audit-redis-enabled", "--audit-redis-addr=127.0.0.1:6379", "--audit-redis-username=user", "--audit-redis-password=pass", @@ -92,11 +104,12 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { assert.Equal(t, "/tmp/audit-certs", cfg.auditCertPath) assert.Equal(t, "cert.pem", cfg.auditCertName) assert.Equal(t, "key.pem", cfg.auditCertKey) + assert.Equal(t, "/tmp/audit-ca", cfg.auditClientCAPath) + assert.Equal(t, "ca.pem", cfg.auditClientCAName) assert.Equal(t, int64(2048), cfg.auditMaxRequestBodyBytes) assert.Equal(t, 5*time.Second, cfg.auditReadTimeout) assert.Equal(t, 8*time.Second, cfg.auditWriteTimeout) assert.Equal(t, 13*time.Second, cfg.auditIdleTimeout) - assert.True(t, cfg.auditRedisEnabled) assert.Equal(t, "127.0.0.1:6379", cfg.auditRedisAddr) assert.Equal(t, "user", cfg.auditRedisUsername) assert.Equal(t, "pass", cfg.auditRedisPassword) @@ -130,13 +143,17 @@ func TestParseFlagsWithArgs_InvalidAuditSettings(t *testing.T) { name: "invalid body size", args: []string{"--audit-max-request-body-bytes=0"}, }, + { + name: "missing audit client ca path", + args: []string{"--audit-client-ca-path=", "--audit-insecure=false"}, + }, { name: "invalid read timeout", args: []string{"--audit-read-timeout=0s"}, }, { - name: "redis enabled without address", - args: []string{"--audit-redis-enabled"}, + name: "empty redis address", + args: []string{"--audit-redis-addr="}, }, { name: "invalid redis db", @@ -184,3 +201,66 @@ func TestBuildAuditServerAddress(t *testing.T) { assert.Equal(t, "0.0.0.0:9444", buildAuditServerAddress("0.0.0.0", 9444)) assert.Equal(t, ":9444", buildAuditServerAddress("", 9444)) } + +func TestBuildAuditServerTLSConfig_RequiresClientCertificates(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + caPath := filepath.Join(tempDir, "tls.crt") + require.NoError(t, os.WriteFile(caPath, []byte(mustMakeTestRootCA(t)), 0o600)) + + cfg := appConfig{ + auditClientCAPath: tempDir, + auditClientCAName: "tls.crt", + } + + serverTLS, err := buildAuditServerTLSConfig(cfg, []func(*tls.Config){ + func(c *tls.Config) { + c.MinVersion = tls.VersionTLS13 + }, + }) + require.NoError(t, err) + require.NotNil(t, serverTLS.ClientCAs) + + assert.Equal(t, tls.RequireAndVerifyClientCert, serverTLS.ClientAuth) + assert.Equal(t, uint16(tls.VersionTLS13), serverTLS.MinVersion) + expectedPool, err := loadCertPoolFromPEMFile(caPath) + require.NoError(t, err) + assert.True(t, expectedPool.Equal(serverTLS.ClientCAs)) +} + +func TestLoadCertPoolFromPEMFile_InvalidPEM(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + caPath := filepath.Join(tempDir, "invalid.pem") + require.NoError(t, os.WriteFile(caPath, []byte("not-a-cert"), 0o600)) + + _, err := loadCertPoolFromPEMFile(caPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "no certificates found") +} + +func mustMakeTestRootCA(t *testing.T) string { + t.Helper() + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "gitops-reverser-test-root", + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, privateKey.Public(), privateKey) + require.NoError(t, err) + + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})) +} diff --git a/config/audit-service.yaml b/config/audit-service.yaml new file mode 100644 index 000000000..62a1fb067 --- /dev/null +++ b/config/audit-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + name: gitops-reverser-audit + namespace: sut +spec: + type: NodePort + ports: + - name: audit + nodePort: 30444 + port: 9444 + protocol: TCP + targetPort: 9444 + selector: + app.kubernetes.io/name: gitops-reverser + control-plane: gitops-reverser diff --git a/config/certs/certificates.yaml b/config/certs/certificates.yaml index c580689cb..148346ae4 100644 --- a/config/certs/certificates.yaml +++ b/config/certs/certificates.yaml @@ -19,6 +19,39 @@ spec: --- apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + name: gitops-reverser-audit-root-ca + namespace: sut +spec: + commonName: gitops-reverser-audit-root-ca + duration: 87600h + isCA: true + issuerRef: + kind: Issuer + name: gitops-reverser-selfsigned-issuer + privateKey: + algorithm: ECDSA + rotationPolicy: Always + size: 256 + secretName: gitops-reverser-audit-root-ca +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + name: gitops-reverser-audit-ca-issuer + namespace: sut +spec: + ca: + secretName: gitops-reverser-audit-root-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -27,11 +60,39 @@ metadata: namespace: sut spec: dnsNames: - - gitops-reverser-service.sut.svc - - gitops-reverser-service.sut.svc.cluster.local + - gitops-reverser-audit.sut.svc + - gitops-reverser-audit.sut.svc.cluster.local + duration: 2160h issuerRef: kind: Issuer - name: gitops-reverser-selfsigned-issuer + name: gitops-reverser-audit-ca-issuer privateKey: rotationPolicy: Always + renewBefore: 360h secretName: audit-server-cert + usages: + - digital signature + - key encipherment + - server auth +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: gitops-reverser + name: gitops-reverser-audit-client-cert + namespace: sut +spec: + commonName: kube-apiserver + duration: 87600h + issuerRef: + kind: Issuer + name: gitops-reverser-audit-ca-issuer + privateKey: + rotationPolicy: Always + renewBefore: 720h + secretName: audit-client-cert + usages: + - digital signature + - client auth diff --git a/config/deployment.yaml b/config/deployment.yaml index f5a49fd93..5dc7fb182 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -29,7 +29,7 @@ spec: - --health-probe-bind-address=:8081 - --webhook-cert-path=/tmp/k8s-admission-server/admission-server-certs - --audit-cert-path=/tmp/k8s-audit-server/audit-server-certs - - --audit-redis-enabled + - --audit-client-ca-path=/tmp/k8s-audit-server/audit-client-ca - --audit-redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 - --audit-redis-stream=gitopsreverser.audit.events.v1 command: @@ -43,6 +43,11 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-auth + key: password image: gitops-reverser:latest livenessProbe: httpGet: @@ -89,6 +94,9 @@ spec: - mountPath: /tmp/k8s-audit-server/audit-server-certs name: audit-webhook-certs readOnly: true + - mountPath: /tmp/k8s-audit-server/audit-client-ca + name: audit-client-ca + readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -104,3 +112,6 @@ spec: - name: audit-webhook-certs secret: secretName: audit-server-cert + - name: audit-client-ca + secret: + secretName: gitops-reverser-audit-root-ca diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 1ff22d995..35dfffd8c 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -5,6 +5,7 @@ resources: - crd - rbac - service.yaml +- audit-service.yaml - deployment.yaml - certs - webhook.yaml diff --git a/config/service.yaml b/config/service.yaml index a6669d063..9ad157d47 100644 --- a/config/service.yaml +++ b/config/service.yaml @@ -7,16 +7,12 @@ metadata: name: gitops-reverser-service namespace: sut spec: - clusterIP: 10.43.200.200 # k3d/k3s service CIDR; kube-apiserver needs a fixed address before CoreDNS is available + type: ClusterIP ports: - name: admission port: 9443 protocol: TCP targetPort: 9443 - - name: audit - port: 9444 - protocol: TCP - targetPort: 9444 - name: metrics port: 8443 protocol: TCP diff --git a/config/webhook.yaml b/config/webhook.yaml index 8d3afdb44..7162bf66d 100644 --- a/config/webhook.yaml +++ b/config/webhook.yaml @@ -12,19 +12,17 @@ webhooks: name: gitops-reverser-service namespace: sut port: 9443 - path: /process-validating-webhook - failurePolicy: Ignore - name: gitops-reverser.configbutler.ai + path: /validate-configbutler-ai-v1alpha1-gittarget + failurePolicy: Fail + name: vgittarget.kb.io rules: - apiGroups: - - '*' + - configbutler.ai apiVersions: - - '*' + - v1alpha1 operations: - CREATE - UPDATE - - DELETE resources: - - '*' + - gittargets sideEffects: None - diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml deleted file mode 100644 index 26cdf5e56..000000000 --- a/config/webhook/manifests.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /process-validating-webhook - failurePolicy: Ignore - name: gitops-reverser.configbutler.ai - rules: - - apiGroups: - - '*' - apiVersions: - - '*' - operations: - - CREATE - - UPDATE - - DELETE - resources: - - '*' - sideEffects: None diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..ff2539695 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,97 @@ +# Configuration Guide + +This guide covers the chart's starter configuration after the main install from the +[root README](../README.md). + +## Recommended Path + +For first-time setup, use the Helm chart's optional `quickstart` block. It creates: + +- a starter `GitProvider` +- a starter `GitTarget` +- a starter `WatchRule` + +That keeps the quickstart on one path instead of asking operators to hand-apply separate sample YAML files. + +## Minimal Quickstart Values + +Create a small values file like this: + +```yaml +quickstart: + enabled: true + namespace: default + gitProvider: + url: git@github.com:/.git + secretRef: + name: git-creds + gitTarget: + path: live-cluster + watchRule: + rules: + - operations: [CREATE, UPDATE, DELETE] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["configmaps"] +``` + +Apply it with: + +```bash +helm upgrade gitops-reverser \ + oci://ghcr.io/configbutler/charts/gitops-reverser \ + --namespace gitops-reverser \ + --reuse-values \ + --values quickstart-values.yaml +``` + +## Defaults + +If you only set `quickstart.enabled=true` plus `quickstart.gitProvider.url`, the chart uses: + +- namespace: `default` +- `GitProvider`: `example-provider` +- `GitTarget`: `example-target` +- `WatchRule`: `example-watchrule` +- Git credentials Secret: `git-creds` +- target branch: `main` +- target path: `live-cluster` +- encryption: SOPS with an auto-generated age key Secret `sops-age-key` + +## Verify + +Check the starter resources: + +```bash +kubectl get gitprovider,gittarget,watchrule -n default +``` + +You should see them settle to `Ready=True` once: + +- the Git credentials Secret exists +- the repository URL is correct +- the operator can connect and push + +Then create a test object: + +```bash +kubectl create configmap test-config --from-literal=key=value -n default +``` + +That should produce a commit in the configured repository within seconds. + +## When To Customize Further + +Use chart values when you want to change: + +- starter resource names +- namespace +- branch or repository path +- Git credentials Secret name +- watched resources and operations + +For more advanced setups, create your own `GitProvider`, `GitTarget`, `WatchRule`, or `ClusterWatchRule` resources +instead of relying on the starter block. + +The chart value reference for the quickstart keys lives in +[charts/gitops-reverser/README.md](../charts/gitops-reverser/README.md). diff --git a/docs/design/audit-consumer-username-attribution-gap.md b/docs/design/audit-consumer-username-attribution-gap.md new file mode 100644 index 000000000..78d5169be --- /dev/null +++ b/docs/design/audit-consumer-username-attribution-gap.md @@ -0,0 +1,139 @@ +# Audit Consumer: Username Attribution Gap + +## What happened + +The correlation webhook removal was completed (all Go code, tests, manifests, Helm chart). +All unit tests and linter pass cleanly. The e2e suite mostly passes — but one test fails: + +``` +[FAILED] Manager Manager should create Git commit when ConfigMap is added via WatchRule +Expected to contain substring "jane@acme.com" +``` + +The test creates a ConfigMap `--as=jane@acme.com` via `kubectl` and asserts that the +resulting git commit has `jane@acme.com` as the author. This is **correct behaviour** — +the audit consumer is supposed to provide exactly this. The test should not be weakened. + +--- + +## Root cause + +The audit consumer's `Start` method is called and the consumer is running. However, every +`XREADGROUP` call fails with: + +``` +XREADGROUP failed: NOGROUP No such key 'gitopsreverser.audit.events.v1' or consumer group +'gitopsreverser-consumer' in XREADGROUP with GROUP option +``` + +Running `XINFO GROUPS gitopsreverser.audit.events.v1` on the live Valkey pod confirms: +**the consumer group does not exist**, despite the stream having 1310 messages and +`ensureConsumerGroup` logging "Consumer group ready" without error. + +### Suspected cause: `NeedLeaderElection() = true` on a single-replica deployment + +The controller-runtime manager only calls `Start` on Runnables that need leader election +**after** the leader lease is acquired. In a single-pod e2e deployment this normally works, +but if the lease is held by a previous pod (from the `prepare-e2e` reinstall) and the new +pod hasn't acquired it yet, the consumer won't start. When it does start, the stream exists +but the group hasn't been created on this connection. + +A cleaner hypothesis: `ensureConsumerGroup` succeeds (creates the group) but the group is +created in the **same startup context that gets cancelled** when the leader lease is +re-acquired or the pod restarts — wiping the in-memory group. The stream persists in +Valkey but the group is lost if Valkey is restarted (Valkey may not be using persistent +storage in the e2e cluster). + +### What to verify in a new session + +1. **Is Valkey persistent?** + ``` + kubectl get deploy -n valkey-e2e valkey -o yaml | grep -A5 storage + kubectl exec -n valkey-e2e deploy/valkey -- valkey-cli CONFIG GET save + kubectl exec -n valkey-e2e deploy/valkey -- valkey-cli CONFIG GET appendonly + ``` + If Valkey uses no persistence and is restarted between test setup and pod startup, + the group creation done by `ensureConsumerGroup` would be lost. + +2. **Does the group get created at all?** + After `prepare-e2e` and pod start, immediately run: + ``` + kubectl exec -n valkey-e2e deploy/valkey -- valkey-cli XINFO GROUPS gitopsreverser.audit.events.v1 + ``` + If the group appears during pod startup but disappears later, Valkey is losing state. + +3. **Does `ensureConsumerGroup` return an error that is being swallowed?** + The `isAlreadyExistsErr` check only looks for `BUSYGROUP`. If Valkey returns a + different error (e.g. auth, ACL, or connection error) for `XGroupCreateMkStream` that + is NOT `BUSYGROUP` but is also not nil, the current code would return it as a fatal + error from `Start`. But the log shows the consumer IS running (producing NOGROUP errors + repeatedly), which means `ensureConsumerGroup` returned nil — so it logged "Consumer + group ready" but the group doesn't actually exist. + + The most likely explanation: `XGROUP CREATE ... MKSTREAM` succeeds (returns OK), Valkey + confirms the group, but then Valkey is restarted or the data is flushed, and by the + time XREADGROUP runs, the group is gone. + +4. **Is the audit handler actually writing to the same Valkey instance?** + The producer (audit handler) and consumer use the same `--audit-redis-addr`. Check + that the `config/deployment.yaml` address (`valkey.valkey-e2e.svc.cluster.local:6379`) + resolves to the same pod that `XINFO` is run against. + +--- + +## State of the removal at the time of pausing + +### Completed +| Item | Status | +|---|---| +| Delete `internal/correlation/` package (6 files) | ✅ | +| Delete `internal/webhook/event_handler.go` + `_test.go` | ✅ | +| `cmd/main.go`: remove import, constants, store init, event handler, webhook registration | ✅ | +| `internal/watch/manager.go`: remove `CorrelationStore` field, `tryEnrichFromCorrelation`, unused imports | ✅ | +| `internal/watch/informers.go`: replace correlation call with `git.UserInfo{}` | ✅ | +| `internal/telemetry/exporter.go`: remove 4 correlation metrics | ✅ | +| `config/webhook.yaml`: replace correlation stanza with GitTarget validator | ✅ | +| `config/webhook/manifests.yaml`: same | ✅ | +| `charts/gitops-reverser/templates/admission-webhook.yaml`: GitTarget validator only | ✅ | +| `charts/gitops-reverser/values.yaml`: remove `webhook.validating`, add `webhook.caBundle` | ✅ | +| `test/e2e/e2e_test.go`: remove 3 correlation webhook tests | ✅ | +| `make lint` passes | ✅ | +| `make test` (unit tests) passes | ✅ | + +### Remaining blocker +The `jane@acme.com` author assertion in the WatchRule e2e test fails because the audit +consumer's `XREADGROUP` can't find its consumer group. The consumer group is either not +being created persistently or is lost between Valkey restarts. This needs investigation +before the e2e suite can pass end-to-end. + +The `test/e2e/e2e_test.go` author assertion at line ~811 has been **restored** to its +original form (it was temporarily removed during investigation but then put back). The +test is correct and should not be weakened. + +--- + +## What the fix is likely to be + +Either: +- Enable Valkey persistence in the e2e cluster (`appendonly yes` or RDB snapshots), **or** +- Make `ensureConsumerGroup` retry with backoff (in case the group is lost after a Valkey + restart mid-session), **or** +- Move `ensureConsumerGroup` into the read loop so the group is re-created whenever + `NOGROUP` is encountered (self-healing approach): + +```go +func (c *AuditConsumer) readAndProcessBatch(ctx context.Context) error { + streams, err := c.client.XReadGroup(ctx, ...).Result() + if err != nil { + if strings.Contains(err.Error(), "NOGROUP") { + // Group was lost (Valkey restart?); recreate it. + _ = c.ensureConsumerGroup(ctx) + } + return fmt.Errorf("XREADGROUP failed: %w", err) + } + ... +} +``` + +The self-healing approach is safest for production resilience regardless of whether the +e2e Valkey is persistent. diff --git a/docs/design/audit-webhook-api-server-connectivity.md b/docs/design/audit-webhook-api-server-connectivity.md new file mode 100644 index 000000000..3618e87a2 --- /dev/null +++ b/docs/design/audit-webhook-api-server-connectivity.md @@ -0,0 +1,320 @@ +# Audit Webhook API Server Connectivity + +GitOps Reverser relies on the Kubernetes [audit webhook backend](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/#webhook-backend): the kube-apiserver POSTs audit events to an HTTPS endpoint using a kubeconfig file written on the control-plane node(s). This works best on clusters you fully control — k3s, k3d, Talos, Kamaji. Managed platforms (EKS, GKE, AKS) restrict access to apiserver configuration; running on them requires switching to a self-managed control plane or a platform that does expose it. + +## The problem + +This creates a connectivity challenge that is easy to underestimate: + +- The audit webhook kubeconfig must be a static file on disk, present before the apiserver starts. +- The endpoint must be reachable from the control-plane node, not just from within the cluster. +- TLS is required for any real deployment; the apiserver must trust the server certificate. + +The apiserver will start fine even if the endpoint is unreachable — audit event delivery is non-fatal. Events will be dropped or retried depending on the configured batch mode, but the cluster comes up regardless. + +This is not a simple configuration step. It is a network and TLS design decision. The right answer depends on how your cluster is hosted. + +## DNS + +The kube-apiserver resolves names using the host's DNS, not CoreDNS. This means in-cluster service names (`*.svc.cluster.local`) do not resolve — those only exist inside CoreDNS, which the host node does not use. However, any hostname resolvable from the control-plane node (internal DNS, a LoadBalancer hostname, etc.) works fine. If you have a stable hostname reachable from the node, DNS is a valid approach. + +This is why the current e2e setup uses a node-local `127.0.0.1:` address instead. + +## Why TLS is non-trivial + +The kube-apiserver validates the TLS certificate of the audit webhook backend. Options are: + +1. Provide a trusted CA bundle in the webhook kubeconfig via `certificate-authority-data`. +2. Skip verification with `insecure-skip-tls-verify: true` (only acceptable for local development). + +The complication: cert-manager, which issues the controller's TLS certificate, is installed *after* the apiserver starts. You cannot reference a cert-manager-issued certificate in the audit webhook kubeconfig at cluster bootstrap time — at least not without a two-phase setup. + +## Audit webhook backend best practices + +The connectivity model is only half of the design. The other half is how kube-apiserver should behave when the audit +backend is slow, unavailable, or receives unusually large events. + +For the full flag reference, see the Kubernetes [`kube-apiserver` command reference](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). + +The safest baseline for GitOps Reverser is: + +- use `--audit-webhook-mode=batch` +- keep batching throttling enabled +- enable webhook truncation +- keep the default initial backoff unless you have a measured reason to change it +- tune batch latency only after the network path and TLS path are already stable + +### Strong recommendation: do not use `blocking` or `blocking-strict` + +Kubernetes documents three webhook modes: + +- `batch`: buffer events and send them asynchronously +- `blocking`: block API server responses while each audit event is sent +- `blocking-strict`: same as `blocking`, but a failure at `RequestReceived` can fail the API request itself + +For GitOps Reverser, `blocking` and especially `blocking-strict` are poor defaults: + +- they couple normal API request latency directly to the health of the audit receiver +- they turn a slow or unavailable audit sink into a control-plane performance problem +- `blocking-strict` can reject otherwise valid user requests during audit backend failures + +So the practical rule is simple: **never start with `blocking` or `blocking-strict` for this integration**. If someone +ever wants to use them, that should be a conscious, high-trust, explicitly tested choice rather than a default. + +### Recommended starting flags + +For self-hosted clusters, the most sensible starting point is: + +```text +--audit-webhook-mode=batch +--audit-webhook-batch-throttle-enable=true +--audit-webhook-truncate-enabled=true +``` + +Then leave these at Kubernetes defaults unless you have measurements showing a real need: + +- `--audit-webhook-initial-backoff=10s` +- `--audit-webhook-batch-buffer-size=10000` +- `--audit-webhook-batch-max-size=400` +- `--audit-webhook-batch-throttle-qps=10` +- `--audit-webhook-batch-throttle-burst=15` + +Those defaults are designed to keep the audit path asynchronous and bounded without making the apiserver overly eager +to retry or overrun the backend. + +### What to tune first, and what not to tune first + +If operators want faster event delivery, the first knob to consider is `--audit-webhook-batch-max-wait`. + +- lower `batch-max-wait` if event freshness matters more than batching efficiency +- leave buffer size, throttle QPS, and burst alone at first +- only increase buffer or throttle after observing real dropped events or backlog pressure + +This ordering matters because `batch-max-wait` changes freshness without immediately increasing failure blast radius, +while aggressive throttle or buffer changes can hide a backend sizing problem rather than solving it. + +### Truncation should be enabled in real deployments + +Kubernetes leaves webhook truncation disabled by default. That is not a great production default for this use case. + +Large audit events do happen, especially when: + +- request bodies are large +- patch payloads are verbose +- request/response objects include a lot of data + +Enabling truncation is safer than letting oversized events destabilize delivery behavior. The tradeoff is that very +large events may lose request/response bodies before export, but that is still usually better than failed delivery or +excessive pressure on the backend. + +### The e2e values are intentionally not production advice + +Our e2e cluster uses a much more aggressive batch profile so tests observe audit events quickly: + +- low `--audit-webhook-batch-max-wait` +- small `--audit-webhook-batch-max-size` + +That is a test feedback optimization, not a universal best practice. Production operators should start from `batch` +mode plus truncation, validate the end-to-end path, and then tune for freshness only if needed. + +## Options + +### Option 1: Pinned ClusterIP + +Pin the Kubernetes Service ClusterIP to a known value via `spec.clusterIP`. Write the webhook kubeconfig with that IP address before the cluster starts. + +```yaml +# Service with fixed ClusterIP +spec: + clusterIP: 10.43.200.200 + +# webhook kubeconfig +server: https://10.43.200.200:9444/audit-webhook +``` + +**TLS**: Use `insecure-skip-tls-verify: true` for development. For production, pre-generate a certificate with a SAN for this IP and include its CA in `certificate-authority-data`. + +**When to use**: Local/dev clusters (k3s, k3d) where you control the CIDR and can reliably pin an IP. Not suitable for clusters where the service CIDR is unpredictable. + +**Tradeoffs**: +- Simple and works without extra infrastructure. +- The fixed ClusterIP will conflict if something else claims it. +- `insecure-skip-tls-verify` is not acceptable for production. + +--- + +### Option 2: Node-local endpoint (hostNetwork / hostPort) + +Run the audit consumer with `hostNetwork: true` or a `hostPort`, so the apiserver can reach it at a loopback or node IP address. + +```yaml +server: https://127.0.0.1:9444/audit-webhook +``` + +**TLS**: A certificate with `localhost` or the node IP as a SAN is straightforward to provision. + +**When to use**: Clusters where you control scheduling on control-plane nodes, and you can tolerate a hostPort or hostNetwork binding. + +**Tradeoffs**: +- Avoids the DNS and ClusterIP pinning problems entirely. +- Requires the pod to be scheduled on the control-plane node (toleration + nodeSelector or DaemonSet on control-plane nodes). +- hostNetwork gives the pod broad network access — consider security implications. + +--- + +### Option 3: External / stable URL + +Route audit events to a stable external URL: a LoadBalancer service, a NodePort, or a tunnel (e.g. Cloudflare Tunnel, ngrok). + +```yaml +server: https://audit.example.com/audit-webhook +``` + +**TLS**: Standard — use a publicly trusted certificate or any cert your apiserver trusts. + +**When to use**: When you want the cleanest separation, or when operating on a managed cluster where you cannot configure the service CIDR or pin ClusterIPs. + +**Tradeoffs**: +- Most robust and portable. +- Requires external DNS, a stable IP or tunnel, and certificate management outside the cluster. +- Adds a dependency on external infrastructure. + +--- + +### Option 4: Bootstrapped CA with pre-provisioned cert + +Pre-generate a CA and issue a certificate for the service IP or hostname before the cluster starts. Write the CA into the webhook kubeconfig as `certificate-authority-data`. Install the same CA and cert into the cluster as a Secret during bootstrap (before cert-manager runs). + +**When to use**: Production k3s/Talos/Kamaji clusters where you want proper TLS without an external endpoint. + +**Tradeoffs**: +- Proper TLS without `insecure-skip-tls-verify`. +- Requires a small bootstrap script to generate and distribute the CA and cert. +- Certificate rotation requires updating the kubeconfig on control-plane nodes and restarting the apiserver — operationally heavier than cert-manager rotation. + +--- + +### Option 5: NodePort reachable from the node itself + +A NodePort service binds to all node interfaces (`0.0.0.0`). Because kube-apiserver runs on the same node, it can reach the NodePort at `127.0.0.1:` — no ClusterIP pinning, no hostNetwork needed. + +```yaml +# Service +spec: + type: NodePort + ports: + - port: 9444 + nodePort: 30444 + +# webhook kubeconfig +server: https://127.0.0.1:30444/audit-webhook +``` + +**TLS**: Issue a cert with `localhost` or the node IP as a SAN. Works naturally with cert-manager if you provision the cert before writing the kubeconfig (two-phase setup), or use a pre-generated cert. + +**When to use**: Single-node or multi-node clusters where you want to avoid IP pinning and hostNetwork, and the apiserver runs directly on a node (i.e. not a managed control plane). + +**Tradeoffs**: +- Clean — uses a standard Kubernetes service primitive. +- NodePort range is typically 30000–32767, which is a slightly unusual port for a webhook. +- On multi-control-plane clusters the NodePort is reachable from every node, so you need to ensure all control-plane nodes can reach it. + +--- + +### Option 6: Static Pod on the control-plane node + +Run the audit receiver as a [static Pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) — a manifest placed in `/etc/kubernetes/manifests/` (or the kubelet's configured staticPodPath). Kubelet starts static pods independently of the API server, before the rest of the cluster is up. With `hostNetwork: true` the pod binds directly to the node's network stack at `127.0.0.1`. + +```yaml +server: https://127.0.0.1:9444/audit-webhook +``` + +**TLS**: The pod can mount a cert from a host path, provisioned during cluster bootstrap alongside the other control-plane certs. + +**When to use**: Clusters where you want the audit receiver to be co-located with the control plane as a first-class component — similar to how etcd or the scheduler itself runs. Fits naturally on bare-metal k8s, Talos, or any setup where you already manage control-plane static pods. + +**Tradeoffs**: +- Very tight coupling between the audit receiver and the control-plane node — both by design. +- Starts before the API server; no dependency on cluster scheduling. +- Updating the pod requires writing a new manifest to the node, not a `kubectl rollout`. +- Not practical if your control-plane is managed (you cannot write to its manifest directory). + +--- + +### Option 7: Node-local proxy as a systemd unit + +Run a minimal reverse proxy on the control-plane node as a systemd service. The proxy forwards traffic from a stable local address to wherever the cluster service happens to be. The audit webhook kubeconfig always points at `127.0.0.1` and never needs to change when the cluster-side service moves. + +```bash +# Example using socat (for illustration — use nginx or a purpose-built proxy for production) +socat TCP-LISTEN:9444,fork TCP:10.43.200.200:9444 +``` + +**TLS**: The proxy handles TLS termination with a cert for `localhost`, decoupling the webhook's TLS from the cluster's internal cert infrastructure. + +**When to use**: When the cluster-side service IP or port is likely to change, or when you want a clean abstraction between the apiserver config and the cluster topology. Also useful as a temporary bridge while migrating between options. + +**Tradeoffs**: +- Adds an extra component to manage on the control-plane node. +- The proxy config must be updated if the backend changes — but the webhook kubeconfig does not. +- `socat` is fine for a quick test; for production use something that handles TLS, connection pooling, and health checks. + +--- + +### Option 8: Kamaji tenant control planes + +[Kamaji](https://kamaji.clastix.io/) runs tenant control planes as pods inside a parent cluster. The tenant kube-apiserver is itself a workload, and its audit webhook kubeconfig can point directly at any service in the parent cluster using standard in-cluster DNS — because the parent cluster's CoreDNS *is* available to the tenant apiserver pod. + +```yaml +server: https://gitops-reverser.gitops-reverser.svc.cluster.local:9444/audit-webhook +``` + +**TLS**: Normal in-cluster TLS with cert-manager. The parent cluster's service infrastructure handles it. + +**When to use**: If you are already running Kamaji for multi-tenancy. The connectivity and TLS problems described in this document largely disappear because the control plane is just another workload. + +**Tradeoffs**: +- Only applicable if you are running Kamaji (or a similar hosted-control-plane system like vcluster). +- Adds a significant operational layer if adopted only to solve this problem. +- Arguably the cleanest long-term architecture if you need multiple clusters. + +--- + +## Which option for which environment + +| Environment | Recommended option | Notes | +|---|---|---| +| Local dev / devcontainer | Option 1 (pinned ClusterIP) + `insecure-skip-tls-verify` | Acceptable for local use only | +| k3s / k3d on a real server | Option 5 (NodePort) or Option 4 | NodePort avoids IP pinning; Option 4 for proper TLS | +| Single-node bare metal | Option 6 (static pod) | First-class control-plane component, no scheduler dependency | +| Talos | Option 4 or Option 6 | Talos machine config makes bootstrap certs and static pods feasible | +| Kamaji | Option 8 | Tenant control plane is a pod; DNS and TLS just work | +| Self-hosted with ingress | Option 3 | Cleanest for multi-node or HA setups | +| Managed cluster (EKS, GKE, AKS) | Requires self-managed control plane | No access to apiserver audit webhook config on managed platforms | + +## What the current e2e setup does + +The e2e test cluster uses **Option 5**: + +- kube-apiserver connects to `https://127.0.0.1:30444/...` +- the audit Service is exposed through a `NodePort` +- the tracked bootstrap kubeconfig source lives at `test/e2e/cluster/audit/webhook-config.yaml` +- the mounted runtime copy is generated under `.stamps/cluster//audit/webhook-config.yaml` + +At cluster bootstrap, that generated file uses `insecure-skip-tls-verify: true` so kube-apiserver can start before +cert-manager issues the real audit TLS materials. After install, the e2e flow rewrites the mounted `.stamps` copy with +the final CA-trusting mTLS kubeconfig and restarts kube-apiserver once. + +## Cluster setup reference + +For k3s-specific setup steps (config file location, policy file, restart procedure), see [`docs/audit-setup/cluster/readme.md`](../audit-setup/cluster/readme.md). + +## Summary + +There is no single correct answer for audit webhook connectivity. Before installing GitOps Reverser, decide: + +1. **How will the kube-apiserver reach the webhook?** (IP, hostname, or tunnel) +2. **How will TLS be handled?** (skip verify, pinned CA, or public cert) +3. **Is this a dev cluster or a production cluster?** + +Option 1 is the path of least resistance for dev clusters and is what the quick start uses. For anything beyond that, plan the network design before you install. diff --git a/docs/design/audit-webhook-tls-design.md b/docs/design/audit-webhook-tls-design.md new file mode 100644 index 000000000..f760eb39f --- /dev/null +++ b/docs/design/audit-webhook-tls-design.md @@ -0,0 +1,428 @@ +# Audit Webhook TLS Design + +## The two operator questions + +This design needs to answer two practical questions before anything else: + +1. What has to happen first? +2. How often do we need to touch kube-apiserver after the initial setup? + +The short answer is: + +- **Yes, audit webhooks always require one control-plane step** because kube-apiserver reads a static + `--audit-webhook-config-file` from the node filesystem. +- **No, we should not require admins to reconfigure kube-apiserver every time cert-manager rotates a leaf cert.** + That would be too operationally expensive. +- **The design goal is one-time enablement, then rare planned maintenance only.** + +The rest of this document is organized around those answers. + +## Why admission and audit are different + +GitOps Reverser has two webhook servers that need TLS: + +1. **Admission webhook** on port `9443` +2. **Audit webhook** on port `9444` + +They look similar, but the trust distribution path is completely different. + +### Admission webhook: zero-touch after install + +The admission webhook is configured through a `ValidatingWebhookConfiguration`, which is a Kubernetes API object. +That object has a `caBundle` field. cert-manager's CA injector can patch that field automatically when the serving +certificate is issued or renewed. + +That is why admission webhook TLS is operationally easy: + +- cert-manager issues or renews the serving cert +- cert-manager injects the CA bundle into the webhook object +- kube-apiserver uses the updated `caBundle` +- no node-level file changes are needed + +### Audit webhook: node-level bootstrap is unavoidable + +The audit webhook is configured with `--audit-webhook-config-file`, which points to a kubeconfig-like file on the +control-plane node filesystem. kube-apiserver reads: + +- the webhook URL from `clusters[].cluster.server` +- the server trust bundle from `certificate-authority-data` +- the expected TLS name from `tls-server-name` +- optional client credentials from `users[]` + +That file is **not** a Kubernetes API object, so there is no audit equivalent of admission webhook CA injection. + +This means there is an unavoidable control-plane bootstrap step: + +- generate the audit webhook kubeconfig +- copy it onto every control-plane node +- ensure kube-apiserver is started with `--audit-webhook-config-file=...` + +That is the first important ordering rule: **the API server integration point is static, even if the webhook +implementation inside the cluster is dynamic.** + +## What must happen first, and why + +There are two separate kinds of kube-apiserver work here: + +1. **Enable audit webhook delivery at all** +2. **Teach kube-apiserver what server certificate and client credential to trust** + +Those have different frequencies. + +### One-time enablement + +A cluster that does not already use the audit webhook backend needs: + +- an audit policy file +- an audit webhook kubeconfig file on the node +- kube-apiserver flags pointing at those files + +This is a control-plane configuration task. GitOps Reverser cannot remove that requirement, because Kubernetes exposes +audit webhooks as a kube-apiserver feature, not as an in-cluster API object. + +### Post-install trust bootstrap + +Once the cluster can send audit events to a webhook, we still need proper TLS. The key point is that we do **not** need +the final TLS assets before kube-apiserver first starts. Audit webhook delivery is non-fatal. The apiserver can start, +the operator can install cert-manager-managed certificates, and then the final kubeconfig can be generated and copied +onto the node. + +That leads to the correct ordering: + +1. Pick a connectivity model that kube-apiserver can actually reach. +2. Install GitOps Reverser and cert-manager resources. +3. Wait for the audit trust root and certificates to exist. +4. Generate the final audit webhook kubeconfig from those artifacts. +5. Copy that file to the control-plane node(s). +6. Restart or otherwise reload kube-apiserver once for the final configuration. + +This order matters because the audit webhook kubeconfig depends on artifacts that do not exist until after the +in-cluster deployment is running. + +## Recommended operating model + +The design should optimize for this operator experience: + +- one initial control-plane enablement step +- one TLS bootstrap step after install +- no kube-apiserver changes for normal serving cert rotation +- rare planned maintenance only when the root CA or the apiserver client credential is intentionally rotated + +In other words, **the API server should trust a stable identity, not a rotating leaf certificate.** + +## Why the current self-signed leaf design is not enough + +Today the audit serving certificate is issued directly from a `selfSigned` issuer. In that setup, the `ca.crt` bundled +in the Secret is effectively the same object as the serving cert. + +That creates the bad lifecycle: + +1. cert-manager renews the serving cert +2. `ca.crt` changes with it +3. kube-apiserver still trusts the old `certificate-authority-data` +4. audit delivery breaks until the node-side kubeconfig is rewritten +5. kube-apiserver must be restarted or reloaded to pick up the change + +That is exactly the admin burden we want to avoid. + +This is not an audit-webhook problem by itself. It is a trust-anchor problem. The fix is to separate: + +- a long-lived root CA that kube-apiserver trusts +- short-lived serving certs signed by that CA + +## The design decision + +Use a small CA hierarchy for the audit webhook: + +1. A long-lived audit root CA +2. A CA issuer backed by that root +3. A short-lived audit serving cert signed by that CA +4. A distinct apiserver client credential for mTLS + +That gives us the lifecycle we actually want: + +- serving cert rotation is automatic and invisible to kube-apiserver +- only root CA rotation requires updating `certificate-authority-data` + +## Connectivity choice + +This document focuses on TLS, but the TLS design still depends on a reachable endpoint. + +For the current k3d/k3s e2e path, the design assumes: + +- audit Service exposed as `NodePort` +- kube-apiserver connects to `https://127.0.0.1:30444/...` +- kubeconfig sets `tls-server-name` to the Service DNS name + +This works in our e2e environment, but should be treated as a **k3d/k3s-specific convenience**, not a universal +Kubernetes guarantee. The broader connectivity tradeoffs stay in +`docs/design/audit-webhook-api-server-connectivity.md`. + +The important TLS point is that `tls-server-name` lets us decouple transport address from certificate identity: + +- kube-apiserver can connect to `127.0.0.1:30444` +- the certificate can still be issued only for + `gitops-reverser-audit.gitops-reverser.svc` + +That means the serving cert does not need IP SANs for `127.0.0.1`. + +## Recommended certificate layout + +### 1. Long-lived audit root CA + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gitops-reverser-audit-root-ca + namespace: gitops-reverser +spec: + isCA: true + commonName: gitops-reverser-audit-root-ca + secretName: gitops-reverser-audit-root-ca + duration: 87600h # 10 years + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: gitops-reverser-selfsigned-issuer + kind: Issuer +``` + +### 2. CA issuer backed by that root + +```yaml +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: gitops-reverser-audit-ca-issuer + namespace: gitops-reverser +spec: + ca: + secretName: gitops-reverser-audit-root-ca +``` + +### 3. Short-lived serving cert + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gitops-reverser-audit-server-cert + namespace: gitops-reverser +spec: + secretName: audit-server-cert + dnsNames: + - gitops-reverser-audit.gitops-reverser.svc + - gitops-reverser-audit.gitops-reverser.svc.cluster.local + usages: + - digital signature + - key encipherment + - server auth + issuerRef: + name: gitops-reverser-audit-ca-issuer + kind: Issuer + duration: 2160h # 90 days + renewBefore: 360h # 15 days + privateKey: + rotationPolicy: Always +``` + +### 4. Long-lived apiserver client cert + +The audit endpoint should require client authentication, but this certificate has a different operational lifecycle than +the serving cert. Because kube-apiserver is the client and its credentials live on the control-plane node, rotating this +credential is a control-plane maintenance event. + +So the client cert should be long-lived as well. It should **not** rotate on the same 90-day cadence as the serving +cert. + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gitops-reverser-audit-client-cert + namespace: gitops-reverser +spec: + secretName: audit-client-cert + commonName: kube-apiserver + usages: + - digital signature + - client auth + issuerRef: + name: gitops-reverser-audit-ca-issuer + kind: Issuer + duration: 87600h # 10 years + renewBefore: 720h # 30 days + privateKey: + rotationPolicy: Always +``` + +The intent is to protect sysadmins from repeated manual kube-apiserver audit-webhook kubeconfig updates and control-plane +reconfiguration. The key design point is that **serving cert rotation should be frequent and automatic; apiserver client +credential rotation should be infrequent and explicit.** + +If an operator wants a shorter lifetime, that should remain configurable, but it should come with a clear warning: +shorter client-cert rotation increases the frequency of node-side kubeconfig redistribution and kube-apiserver +maintenance. + +## The kubeconfig kube-apiserver should trust + +With the stable root CA in place, the node-side file should look like this: + +```yaml +apiVersion: v1 +kind: Config +clusters: +- name: audit-webhook + cluster: + server: https://127.0.0.1:30444/audit-webhook/my-cluster + certificate-authority-data: + tls-server-name: gitops-reverser-audit.gitops-reverser.svc +contexts: +- name: audit-webhook + context: + cluster: audit-webhook + user: apiserver +current-context: audit-webhook +users: +- name: apiserver + user: + client-certificate-data: + client-key-data: +``` + +The important lifecycle property is: + +- `certificate-authority-data` comes from the root CA Secret, so it stays stable across serving cert renewal + +## How often kube-apiserver must be changed + +This is the most important operational section. + +| Event | Node-side update needed? | Control-plane maintenance needed? | +|---|---|---| +| Initial audit webhook enablement | Yes | Yes | +| First TLS bootstrap after install | Yes | Yes | +| Normal serving cert rotation | No | No | +| Root CA rotation | Yes | Yes | +| Client certificate rotation | Yes | Yes | + +### Always required + +These are unavoidable control-plane tasks: + +- enabling the audit webhook backend in kube-apiserver +- placing an audit policy file on the node +- placing an audit webhook kubeconfig on the node + +### Required once at install time + +After GitOps Reverser and cert-manager issue the initial root CA and certificates, the generated kubeconfig must be +copied to the control-plane node(s) and kube-apiserver must pick it up. + +For this design, that should happen once after installation. + +### Not required for normal serving cert rotation + +If kube-apiserver trusts the long-lived root CA rather than the serving leaf cert: + +- cert-manager can renew the serving cert every 90 days +- the webhook pod can reload the new cert +- kube-apiserver keeps working without any kubeconfig edit + +This is the main design win. + +### Required for root CA rotation + +If the root CA is intentionally rotated, then the node-side `certificate-authority-data` must be updated on every +control-plane node. This should be rare and planned. + +### Required for client certificate rotation + +If the client certificate embedded in the kubeconfig is rotated, then the kubeconfig must be regenerated and +redistributed. That is why the client credential needs a much slower rotation cadence than the serving cert. + +If we later prove that a specific Kubernetes distribution safely reloads file-based client credentials without a full +apiserver restart, we can optimize this. For now, the conservative design assumption should be: + +- changes to the audit webhook kubeconfig or its embedded credentials are control-plane maintenance + +## Why mTLS still matters + +Without client authentication, anything that can reach the audit endpoint can POST forged audit events. + +That risk is lower in local e2e, but it is not acceptable as a production design. So the audit server should require and +verify client certificates against the same root CA: + +- server presents a cert signed by `gitops-reverser-audit-root-ca` +- kube-apiserver presents a client cert signed by that same CA +- audit server uses `tls.RequireAndVerifyClientCert` + +That keeps the endpoint closed to unauthenticated callers even if the NodePort is reachable more broadly than intended. + +## Current e2e state + +The current e2e path is intentionally a bridge state, not the end-state design. + +Today it does this: + +1. Start with `insecure-skip-tls-verify: true` +2. Wait for `audit-server-cert` +3. Extract `ca.crt` from that Secret +4. Rewrite `test/e2e/cluster/audit/webhook-config.yaml` +5. Restart the k3d server node + +That proves the transport works, but it still has two production problems: + +- trust is tied to the rotating serving cert +- no client authentication is configured + +So the e2e mechanism is useful as proof of connectivity, but it should not be the final operator story. + +## Proposed operator story + +The installation flow should be documented like this: + +1. Install GitOps Reverser and cert-manager resources. +2. Wait for `gitops-reverser-audit-root-ca`, `audit-server-cert`, and `audit-client-cert`. +3. Run a helper that prints the final webhook kubeconfig. +4. Copy that kubeconfig to the control-plane node(s). +5. Restart or reload kube-apiserver once. +6. Leave serving cert rotation entirely to cert-manager. + +That answers the original ordering question cleanly: + +- **what goes first?** + The root CA must exist before we generate the final kubeconfig, because kube-apiserver needs a stable trust anchor. +- **why?** + Because the on-disk kubeconfig is where apiserver trust lives, and we want that trust to outlive normal serving cert + rotation. + +## Service split recommendation + +The current single `NodePort` Service exposes admission, audit, and metrics together. Only the audit path needs +node-level reachability. + +The cleaner topology is: + +| Service | Type | Ports | Purpose | +|---|---|---|---| +| `gitops-reverser` | `ClusterIP` | admission `9443`, metrics `8443` | In-cluster only | +| `gitops-reverser-audit` | `NodePort` | audit `9444` -> `30444` | Reachable by kube-apiserver | + +This keeps the node-reachable surface area as small as possible. + +## Implementation checklist + +- [ ] Split the audit Service from the in-cluster admission and metrics Service +- [ ] Add a long-lived audit root CA Certificate +- [ ] Add a CA Issuer backed by that root +- [ ] Issue the audit serving cert from the CA Issuer +- [ ] Issue a distinct audit client cert for kube-apiserver +- [ ] Remove `127.0.0.1` IP SAN dependency and rely on `tls-server-name` +- [ ] Update the audit server to require and verify client certificates +- [ ] Add tooling that prints the final audit webhook kubeconfig after install +- [ ] Update e2e TLS injection to read the stable root CA instead of the serving cert Secret +- [ ] Document that serving cert rotation is automatic, but root CA and client credential rotation are planned + control-plane maintenance events diff --git a/docs/design/e2e-runtime-state-vs-stamps.md b/docs/design/e2e-runtime-state-vs-stamps.md new file mode 100644 index 000000000..4fb27d78a --- /dev/null +++ b/docs/design/e2e-runtime-state-vs-stamps.md @@ -0,0 +1,509 @@ +# E2E Runtime State vs Stamp State + +## Summary + +This note explains a local-only e2e failure mode that appeared while validating the +`reverse-gitops` changes, why the existing stamp mechanism did not fully protect us, what +I think the right solution shape is, and what the tradeoffs are. + +The short version is: + +- the failure was real +- it was not the audit webhook listener IP +- it was caused by a mismatch between **stamp state** and **live cluster runtime state** +- the mismatch is most relevant on **reused local k3d clusters** +- the most defensible fix is to keep stamps for deterministic setup work, but add a small + amount of **explicit runtime validation** where stamps cannot be authoritative + +I do **not** think this means the stamp mechanism is wrong. I do think it was asked to +represent a class of state that it cannot reliably represent on its own. + +## What Was Happening + +The local failure looked like this: + +- `make test-e2e` entered `BeforeSuite` +- `prepare-e2e` ran +- install artifacts were applied successfully +- controller rollout timed out +- the pod ended up in `ImagePullBackOff` + +On inspection, the key facts were: + +- the local image `gitops-reverser:e2e-local` existed in Docker +- `.stamps/image/controller.id` matched that local image +- `.stamps/cluster//image.loaded` also matched that local image +- but the current k3d nodes did **not** actually have that image in containerd +- kubelet then tried to pull `docker.io/library/gitops-reverser:e2e-local`, which of course failed + +So the precise mismatch was: + +- Make believed the image had already been loaded into the cluster +- the actual cluster runtime no longer had that image available on the node + +That is why rollout failed even though the stamp graph looked correct. + +## Why This Can Happen + +The stamp mechanism is good at tracking outputs that are: + +- deterministic +- created by our own recipes +- still valid as long as their inputs did not change + +Examples: + +- generated manifests +- rendered install YAML +- built local image ID +- an applied secret manifest + +But node-local image availability is different. It is affected by runtime behavior outside +the Make graph, for example: + +- node-local container image garbage collection +- manual cluster deletion/recreation +- node replacement +- partial cluster corruption +- external cleanup in Docker or k3d internals + +In other words, `image.loaded` is not purely a build artifact. It is a claim about the +current state of another system. + +That claim can become false without any Make input changing. + +## Why The Stamp Was Not Enough + +The existing stamp said: + +> At some point, this image was loaded into this cluster context. + +What the controller rollout actually needs is stronger: + +> Right now, the node that may schedule this pod can start it from the expected image. + +Those are not the same statement. + +This is why the failure looked surprising: + +- from the Makefile perspective, nothing was stale +- from Kubernetes' perspective, the deployment was not runnable + +That gap is the central issue. + +## Why This Is Mostly a Local Problem + +This is mainly a reused-local-cluster problem. + +CI usually starts from a fresh environment: + +- new container runtime state +- new cluster +- no long-lived node image cache history +- no previously accumulated local drift + +Local development is different: + +- the cluster lives longer +- nodes may hit disk pressure +- images may be GC'd +- the same cluster name and stamp directory may be reused for a long time + +That is why this showed up locally but not on the build server. + +## Why The Webhook Listener IP Was Not The Root Cause + +The original suspicion was that the audit webhook listener might be bound to the wrong IP. + +That did not match the evidence: + +- the controller args still showed `--audit-listen-address=0.0.0.0` +- once the controller pod actually started, audit traffic worked +- the blocking failure happened before that, at image start time + +So the webhook IP was a plausible hypothesis, but it was not the root cause of the local +rollout failure we observed. + +## What Is Actually Needed + +I think we need to distinguish two classes of truth: + +### 1. Stamp truth + +This should continue to cover deterministic setup work: + +- cluster bootstrap prerequisites +- rendered manifests +- install outputs +- generated secrets +- local image build identity + +### 2. Runtime truth + +This has to be checked live because it can drift: + +- is the cluster healthy enough to trust? +- is the expected image available to the current nodes? +- did the deployment actually roll out? +- is the running controller the expected build? + +The important design point is: + +**Runtime truth should not replace stamps. It should sit next to them only where stamps are +not authoritative.** + +## Recommended Solution + +My preferred solution is this: + +### A. Fail hard on unhealthy reused clusters + +If `start-cluster.sh` finds an existing cluster that is unhealthy, it should stop and tell +the user to clean it up explicitly. + +Reason: + +- once the cluster is already unhealthy, we cannot confidently infer which cluster-scoped + stamps are still valid +- auto-recreating the cluster is convenient, but it is also guessy +- explicit cleanup is safer and more honest + +This is especially true because cluster state and `.stamps/cluster/` are coupled by +convention, not by a strong transactional boundary. + +### B. Keep `controller.deployed` self-sufficient + +The deployment recipe should still ensure the image is really available before rollout. + +Reason: + +- if `controller.deployed` runs, it must be able to succeed on its own +- relying only on `image.loaded` is not enough when node runtime state may have drifted +- the deployment recipe is exactly the place that knows what the controller pod needs next + +### C. Keep runtime revalidation explicit + +If we want an always-run check for reused clusters, it should be a clearly named step such as +`ensure-runtime-ready`, not an accidental side effect hidden behind a stale-looking stamp. + +That keeps the model understandable: + +- stamps represent reproducible outputs +- ensure steps validate live state + +### D. Add a build-info endpoint + +This is a very good complement. + +It would let the e2e suite verify: + +- the controller is running +- the running controller is the expected build + +For that endpoint I would expose: + +- `version` +- `gitCommit` +- `buildDate` +- optionally `dirty` +- optionally image digest if available + +I would validate on `gitCommit` or digest, not `buildDate` alone. + +## Why A Build-Info Endpoint Helps + +A build-info endpoint solves a different but important problem: + +- stamps can say what we built +- rollout can say something is running +- build-info can say the thing that is running is the one we expected + +That is stronger than today's implicit assumption. + +It is especially useful for: + +- local reuse of an existing deployment +- accidental drift between install modes +- debugging "did I actually run the new binary?" questions + +## Implementation Revision: Trusting the Stamp via Containerd Pinning + +After implementing the hybrid model above, we reconsidered the `cluster_has_project_image` +node probe in `load-image.sh` and the `ensure-runtime-ready` phony target. + +The node-probe approach ran `crictl images` inside every k3d node container on each +`prepare-e2e` call to verify the image was still present in containerd. The justification was +that node-local GC could silently invalidate the stamp. We initially dismissed this as +overcomplicated — the cluster is developer-owned, k3d nodes do not arbitrarily lose images, +and if the cluster is broken `start-cluster.sh` would catch it. + +**That bet was wrong.** We observed the exact failure on the next run: + +- the stamp was valid — the image had been loaded into the nodes +- a code change we made invalidated the `config-dir/install.yaml` stamp, triggering an + automatic cleanup that deleted the deployment and namespace +- with no running containers referencing the image, containerd's kubelet-driven image GC + evicted it from all nodes within minutes +- on the next `prepare-e2e`, Make skipped the import (stamp still matched), the deploy ran, + and rollout failed with `ImagePullBackOff` + +The root cause is: **containerd's CRI plugin GCs unreferenced images on behalf of the kubelet**. +This is not a node restart or manual intervention — it is normal Kubernetes image lifecycle +behaviour. The stamp cannot know about it. + +### Why we did not restore the runtime node probe + +The per-node `crictl images` probe on every `prepare-e2e` call is the safe but expensive fix. +We looked for a cheaper alternative that makes the stamp itself reliable. + +### The fix: pin the image after import + +Containerd's CRI plugin checks for the label `io.cri-containerd.pinned=pinned` before evicting +any image. If the label is set, the image is skipped unconditionally by the GC. + +After `k3d image import`, `load-image.sh` now runs: + +``` +ctr -n k8s.io images label io.cri-containerd.pinned=pinned +``` + +on every node. This is done once, at import time. The stamp remains the sole fast-path check +on subsequent runs, and is now reliable: if the stamp says the image is loaded, containerd +still has it because it is pinned. + +We verified this in the cluster by inspecting the image labels after import: + +``` +docker.io/library/gitops-reverser:e2e-local ... io.cri-containerd.image=managed,io.cri-containerd.pinned=pinned +``` + +The simplification that was also applied at this point and is still in effect: + +- removed `ensure-runtime-ready` from the `prepare-e2e` flow entirely +- kept `deploy-controller.sh` idempotent (checks spec image + available replicas before + issuing `kubectl set image`) to avoid the unconditional 180s rollout wait on no-op runs + +## What A Build-Info Endpoint Does Not Solve + +It does **not** replace rollout or image-availability checks. + +If the image has been GC'd and the pod cannot start: + +- there is no running process +- there is no endpoint to query + +So it cannot be the only guard. + +It is a valuable verification layer, not a startup-recovery mechanism. + +## Alternative Approaches + +## Option 1: Trust stamps only + +Description: + +- keep current stamp graph +- do not perform live runtime validation + +Pros: + +- fastest in the no-op case +- simplest Makefile mental model +- no extra runtime checks or imports + +Cons: + +- vulnerable to exactly the class of local drift we observed +- difficult to debug when runtime state diverges from cached assumptions +- rollout failures become surprising instead of explainable + +Assessment: + +- acceptable only if we believe node-local image drift is effectively impossible +- the observed failure shows that belief is too strong + +## Option 2: Always self-heal runtime state + +Description: + +- always re-check runtime state +- re-import image if missing +- possibly auto-recreate unhealthy clusters + +Pros: + +- robust local developer experience +- fewer manual cleanup steps +- reduced chance of confusing local failures + +Cons: + +- can undermine the spirit of stamps if done too broadly +- adds time to no-op or mostly-no-op flows +- may hide underlying local environment problems +- auto-repairing unhealthy clusters can guess incorrectly about which stamps remain valid + +Assessment: + +- useful in moderation +- too aggressive if it starts replacing stamp invalidation logic with runtime probing everywhere + +## Option 3: Fail fast on any runtime mismatch + +Description: + +- if node image is missing, fail and instruct user to clean/reset +- if cluster is unhealthy, fail immediately + +Pros: + +- very clean model +- preserves stamp meaning +- surfaces environmental problems directly + +Cons: + +- more manual recovery for developers +- repeated local friction for recoverable situations +- harder to use in long-lived dev clusters + +Assessment: + +- strong from a correctness perspective +- maybe too punishing for day-to-day local iteration + +## Option 4: Hybrid model + +Description: + +- fail hard on unhealthy cluster reuse +- keep `controller.deployed` self-sufficient for image availability +- optionally add one explicit runtime ensure step for reused-cluster no-op paths +- add build-info verification + +Pros: + +- clean separation between stamp truth and runtime truth +- targeted protection exactly where stamps are weak +- avoids pretending the stamp graph can represent external runtime perfectly +- easier to reason about than broad self-healing + +Cons: + +- still adds some runtime overhead +- slightly more complex than pure-stamp logic +- can duplicate checks if wired poorly + +Assessment: + +- this is the best balance in my view + +## My Recommended Shape + +If I were shaping this for the long term, I would recommend: + +1. `start-cluster.sh` fails hard on unhealthy existing clusters +2. `controller.deployed` ensures image availability before rollout +3. `prepare-e2e` does not broadly bypass the stamp graph +4. if we keep an always-run runtime check, make it explicit and narrow +5. add a build-info endpoint and assert the running binary identity in e2e + +That keeps the system honest: + +- stamps are still the primary mechanism +- runtime checks cover only what stamps cannot know + +## Build-Time / Runtime Cost + +This definitely has a cost. + +The main cost comes from image import when the runtime check decides the node image is missing. + +From the observed local runs, a re-import was roughly: + +- about 7 to 12 seconds for the import itself +- plus rollout time after that + +If the check is wired in the wrong place, that cost can be paid more than once in one e2e +preparation path. That is real overhead and should be minimized. + +So I would separate the cost discussion like this: + +### Cheap path + +When the image is already present and the check exits fast: + +- low overhead +- mostly process startup plus a few `crictl images` calls + +### Expensive path + +When the image must be re-imported: + +- noticeable local cost +- but that cost is still cheaper than a confusing `ImagePullBackOff` failure followed by manual debugging + +## How Sure Am I That This Adds Value? + +### On the existence of the problem + +High confidence. + +We directly observed: + +- matching local image + stamp state +- missing node-local image +- `ImagePullBackOff` +- rollout recovery immediately after forced re-import + +That part is not speculative. + +### On the need for some runtime validation + +High confidence. + +The failure class is exactly one that stamps cannot fully represent. + +### On the exact final shape + +Medium confidence. + +I am confident in the principle: + +- fail hard on unhealthy clusters +- keep deployment recipes self-sufficient +- verify running build identity explicitly + +I am less confident that the current arrangement of checks is the final ergonomic optimum. +There is still room to reduce duplicate imports/checks on cold rebuilds. + +### On overall value + +Moderate to high value for local developer correctness. + +The value is highest when: + +- the same local k3d cluster is reused often +- developers rely on e2e repeatedly during iteration +- debugging time matters more than a few extra seconds + +The value is lower if: + +- clusters are always torn down fresh +- local e2e is rare +- CI is the only trusted validation path + +## Final Position + +I do not think we should abandon the stamp mechanism. + +I do think we should stop expecting stamps to prove live runtime facts that belong to the +cluster and node runtime. + +So my final position is: + +- stamps remain the primary mechanism +- unhealthy existing clusters should fail hard +- deployment/runtime checks should cover only the live state that stamps cannot guarantee +- a build-info endpoint would be an excellent next improvement to verify that the running + controller is actually the intended build diff --git a/docs/design/final-redis-required-plan.md b/docs/design/final-redis-required-plan.md new file mode 100644 index 000000000..99514bde6 --- /dev/null +++ b/docs/design/final-redis-required-plan.md @@ -0,0 +1,486 @@ +# Final Plan: Redis/Valkey as a Hard Requirement + +This document captures the concrete changes needed to treat Valkey/Redis as a first-class external +dependency, require authenticated connections, rename the default install namespace, and clean up +the leftover design-era artifacts that have accumulated now that the consumer is fully implemented +and all e2e tests pass. + +--- + +## 1. Current State + +Valkey is already a hard runtime requirement. The controller fails to start without a reachable Redis +endpoint (`audit-redis-addr` is validated at startup and the consumer/producer both call `fatalIfErr` +on connection setup). There is no code path that skips Redis. + +What has not caught up to that reality: + +- The quick start in `charts/gitops-reverser/README.md` installs Valkey **without auth**, directly + teaching operators to run an unauthenticated queue that carries audit payloads and secret material. +- The password is threaded through as a CLI argument (`--audit-redis-password`), making it visible in + `kubectl describe pod` output. +- `values.yaml` exposes `queue.redis.username` and `queue.redis.password` as plain text fields, + which end up in Helm release secrets and command args rather than in a proper Kubernetes Secret. +- The default install namespace throughout all documentation is `gitops-reverser-system`. The + `-system` suffix is a convention for Kubernetes infrastructure (e.g. `kube-system`). For an + operator that users install by name, `gitops-reverser` is shorter, cleaner, and more consistent + with how cert-manager itself uses `cert-manager` rather than `cert-manager-system`. +- Several design documents written during the "should we adopt Valkey?" and "should the consumer be + optional?" phases are now fully superseded, but they still sit in `docs/design/` and create + confusion about what is settled. + +--- + +## 2. Why Auth Is Non-Negotiable + +Every audit event that flows through the system — including those for Secrets, all user activity, +and all mutating operations — passes through Valkey before it reaches Git. An unauthenticated Valkey +instance allows any process in the cluster to: + +- Read every audit payload (including raw Secret `data` when the audit policy level allows it). +- Inject arbitrary events into the audit stream, causing fake commits. +- Delete or corrupt pending entries, breaking at-least-once delivery. + +This is not a theoretical risk. Valkey is explicitly deployed in the controller's namespace in the +standard quick start, meaning any pod in that namespace could attack it without credentials. + +Auth is therefore not an optional hardening step — it is the minimum acceptable security posture +for a queue that carries Kubernetes audit data. + +--- + +## 3. Dependency Model: Follow the cert-manager Pattern + +Valkey/Redis should be documented and installed the same way as cert-manager: **install it first, +point us at it, we require it to be there**. + +What this means in practice: + +- The chart does **not** bundle a Valkey subchart (the `valkey-adoption-guide.md` recommended this + approach, and we already follow it — no change needed here). +- The prerequisites section of the README lists Valkey/Redis alongside cert-manager and Kubernetes. +- The quick start shows a minimal but **auth-enabled** Valkey install before the controller install. +- The configuration reference documents `queue.redis.auth.existingSecret` as the canonical auth + pattern, not inline plaintext credentials. + +There is no need for a `valkey.enabled` toggle or embedded subchart. If a user wants to run a +quick dev environment, they can install Valkey via Helm with one command; they do not need us to +do it for them any more than they need us to install cert-manager. + +--- + +## 4. Changes Required + +### 4.1 Default Namespace: `gitops-reverser` + +Every occurrence of `gitops-reverser-system` in documentation, README examples, config README, and +any other user-facing text should change to `gitops-reverser`. + +The `-system` suffix is a Kubernetes convention for infrastructure namespaces that ship with the +platform (`kube-system`, `kube-public`). Third-party operators typically do not follow that pattern +— cert-manager uses `cert-manager`, not `cert-manager-system`. Using just `gitops-reverser` is +shorter, easier to type, and consistent with the project name. + +This is a documentation-only change. The chart itself uses `{{ .Release.Namespace }}` throughout +and has no hardcoded namespace. Users who already installed into `gitops-reverser-system` are +unaffected; the new default is what the quick start and README suggest going forward. + +Files to update: + +- `charts/gitops-reverser/README.md` — all example commands +- `config/README.md` — install path examples +- Any other docs that use `gitops-reverser-system` as an example namespace + +--- + +### 4.2 Valkey Chart Source: Official valkey-io Chart + +**Decision: use the official `valkey-io` Helm chart, not Bitnami.** + +The current quick start uses: + +```bash +helm install valkey oci://registry-1.docker.io/bitnamicharts/valkey \ + --set auth.enabled=false +``` + +Replace with the official chart from the Valkey project: + +```bash +helm repo add valkey https://valkey.io/valkey-helm/ +helm repo update +``` + +**Why the official chart over Bitnami:** + +| | Official (valkey-io) | Bitnami | +|---|---|---| +| Maintained by | The Valkey project itself | Broadcom/Bitnami | +| Auth model | Native ACL users (`usersExistingSecret`) | Wrapped flag (`auth.existingSecret`) | +| Abstraction layer | Minimal — config maps almost directly to `valkey.conf` | Heavy — many init scripts, non-standard entrypoints | +| Dependency on Bitnami | None | Bitnami base image, scripts, helpers | +| User experience | What you configure is what Valkey does | Bitnami layer can surprise users debugging issues | + +The Bitnami chart is widely used and works fine, but it wraps Valkey in its own init scripts and +base images. When something goes wrong, users end up debugging Bitnami's layer rather than Valkey +itself. For a project that positions Valkey as a first-class required dependency that operators +need to understand and maintain, pointing to the official chart removes that indirection. + +The official chart is actively maintained (0.9.3, January 2025), has 243 GitHub stars, and is +published through the official Valkey project at `https://valkey.io/valkey-helm/`. + +**Auth configuration for the official chart:** + +The official chart uses ACL-based auth. The `default` user must be defined when auth is enabled. +The secret key name is set per-user via `passwordKey`. + +```bash +# 3. Create the namespace and auth secret +kubectl create namespace gitops-reverser +kubectl create secret generic valkey-auth \ + --namespace gitops-reverser \ + --from-literal=password="$(openssl rand -base64 32)" + +# 4. Install Valkey using the official chart with auth enabled +helm repo add valkey https://valkey.io/valkey-helm/ +helm repo update +helm install valkey valkey/valkey \ + --version 0.9.3 \ + --namespace gitops-reverser \ + --set auth.enabled=true \ + --set auth.usersExistingSecret=valkey-auth \ + --set auth.aclUsers.default.passwordKey=password \ + --set auth.aclUsers.default.permissions="~* &* +@all" + +# 5. Install GitOps Reverser, referencing the same auth secret +helm install gitops-reverser \ + oci://ghcr.io/configbutler/charts/gitops-reverser \ + --namespace gitops-reverser \ + --create-namespace \ + --set queue.redis.addr="valkey.gitops-reverser.svc.cluster.local:6379" \ + --set queue.redis.auth.existingSecret=valkey-auth \ + --set queue.redis.auth.existingSecretKey=password +``` + +The `--create-namespace` on the GitOps Reverser install is a safety net; the namespace already +exists from step 3, so this is idempotent. + +> **Service hostname:** The official chart applies standard Helm deduplication: when the release +> name and chart name are both `valkey`, the fullname is just `valkey` (not `valkey-valkey`). +> So `helm install valkey valkey/valkey` produces a service called `valkey`, and the address +> is `valkey.gitops-reverser.svc.cluster.local:6379`. No `fullnameOverride` is needed. + +--- + +### 4.3 Replace Plaintext Password Fields with existingSecret Pattern + +**File:** `charts/gitops-reverser/values.yaml` + +Remove: + +```yaml +queue: + redis: + username: "" + password: "" +``` + +Add: + +```yaml +queue: + redis: + auth: + # Name of a pre-existing Secret in the same namespace that holds the Redis password. + # The secret must be created before installing this chart. + # Required for production deployments. Leave empty only for dev/local environments + # where Valkey runs without authentication. + existingSecret: "" + # Key within the Secret that holds the password value. + existingSecretKey: "password" + # Optional Redis username (for Redis 6+ ACL-based auth). + username: "" +``` + +The `username` field stays because Redis ACL setups use it, and it is not sensitive enough to +require a secret reference. If a project later wants to require username-in-secret too, it can +follow the same `existingSecret` extension. + +--- + +### 4.4 Remove Password from CLI Args, Use Env Var Instead + +**File:** `charts/gitops-reverser/templates/deployment.yaml` + +Remove: + +```yaml +{{- with .Values.queue.redis.username }} +- --audit-redis-username={{ . | quote }} +{{- end }} +{{- with .Values.queue.redis.password }} +- --audit-redis-password={{ . | quote }} +{{- end }} +``` + +Keep the username arg (it is not sensitive), but source the password through an env var: + +```yaml +{{- with .Values.queue.redis.auth.username }} +- --audit-redis-username={{ . | quote }} +{{- end }} +``` + +Add to the `env:` block: + +```yaml +{{- if .Values.queue.redis.auth.existingSecret }} +- name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.queue.redis.auth.existingSecret }} + key: {{ .Values.queue.redis.auth.existingSecretKey | default "password" }} +{{- end }} +``` + +**File:** `cmd/main.go` + +Change the flag default for `--audit-redis-password` from `""` to `os.Getenv("REDIS_PASSWORD")`: + +```go +fs.StringVar(&cfg.auditRedisPassword, "audit-redis-password", os.Getenv("REDIS_PASSWORD"), + "Redis password for audit event queueing. Can also be set via REDIS_PASSWORD env var.") +``` + +This means the password never appears in process args, `kubectl describe pod`, or Helm release +history. The env var is read at startup from the mounted secret volume. + +The same pattern should be applied to `--audit-redis-username` if ACL usernames are ever +considered sensitive in a given deployment, but that is optional since usernames are generally +not secrets. + +--- + +### 4.5 Single YAML Install (`install.yaml`) + +The `install.yaml` generated from the Helm chart is used for quick cluster installs without Helm. +With the `existingSecret` approach, users must create the Secret before applying the manifest. + +Document this as a prerequisite in the release page and in the config README: + +```bash +# 1. Create the namespace and Valkey auth secret +kubectl create namespace gitops-reverser +kubectl create secret generic valkey-auth \ + --namespace gitops-reverser \ + --from-literal=password="$(openssl rand -base64 32)" + +# 2. Install cert-manager (if not already installed) +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml +kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s + +# 3. Install Valkey (official chart, auth-enabled) +helm repo add valkey https://valkey.io/valkey-helm/ +helm repo update +helm install valkey valkey/valkey \ + --version 0.9.3 \ + --namespace gitops-reverser \ + --set auth.enabled=true \ + --set auth.usersExistingSecret=valkey-auth \ + --set auth.aclUsers.default.passwordKey=password \ + --set auth.aclUsers.default.permissions="~* &* +@all" + +# 4. Apply the install manifest +kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/latest/download/install.yaml +``` + +The `install.yaml` workflow does not change structurally. It already requires cert-manager to be +pre-installed; requiring a Valkey auth secret follows the same model. + +The `config/deployment.yaml` used for e2e testing currently connects to Valkey without auth. This +is acceptable for CI (the Valkey instance is cluster-internal and short-lived), but a TODO comment +should note that the e2e Valkey install should be updated to run with auth once the e2e environment +Helm values are updated. + +--- + +### 4.6 E2E Test Environment Auth + +The e2e cluster already uses the official Valkey chart (via Flux HelmRelease in `valkey-e2e` +namespace). Auth is currently disabled. It should be enabled — no exceptions — but with a fixed +known password committed to the repo. A hardcoded test secret is appropriate here: the e2e cluster +is ephemeral, local, and contains no real data. + +**Fixed password constant:** `e2e-valkey-password` + +This is committed openly and that is intentional. It is only ever used in CI and local dev clusters. + +#### Changes required + +**`test/e2e/setup/flux/values/valkey-values.yaml`** — enable auth, reference secret: + +```yaml +# Valkey Helm values for e2e testing. +# Standalone mode, no persistence, minimal resources, auth enabled. + +architecture: standalone + +primary: + persistence: + enabled: false + resources: + limits: + memory: "256Mi" + requests: + memory: "128Mi" + +auth: + enabled: true + usersExistingSecret: valkey-auth + aclUsers: + default: + permissions: "~* &* +@all" + passwordKey: password +``` + +**`test/e2e/setup/flux/kustomization.yaml`** — add a `secretGenerator` for the valkey-auth secret +in `valkey-e2e`. This ensures the secret exists before the HelmRelease reconciles: + +```yaml +secretGenerator: + - name: valkey-auth + namespace: valkey-e2e + literals: + - password=e2e-valkey-password + options: + disableNameSuffixHash: true +``` + +**`config/valkey-auth.yaml`** — new file: a fixed Secret for the controller namespace. Kustomize +will replace the `namespace` value with `$(NAMESPACE)` at apply time: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: valkey-auth + namespace: sut +type: Opaque +stringData: + password: e2e-valkey-password +``` + +Add `valkey-auth.yaml` to `config/kustomization.yaml` resources. + +**`config/deployment.yaml`** — add the `REDIS_PASSWORD` env var from the secret: + +```yaml +env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-auth + key: password +``` + +**`Makefile`** — add `E2E_VALKEY_PASSWORD` variable and wire auth into the helm install mode. +The `config-dir` mode picks it up from the committed secret file automatically. The `helm` mode +needs an explicit `kubectl create secret` before the chart install and the auth values passed to +`helm upgrade --install`: + +```makefile +E2E_VALKEY_PASSWORD ?= e2e-valkey-password +``` + +In the `$(CS)/$(NAMESPACE)/helm/install.yaml` target, before the `helm upgrade --install` call: + +```makefile +kubectl --context $(CTX) create namespace $(NAMESPACE) --dry-run=client -o yaml \ + | kubectl --context $(CTX) apply -f - +kubectl --context $(CTX) -n $(NAMESPACE) create secret generic valkey-auth \ + --from-literal=password=$(E2E_VALKEY_PASSWORD) \ + --dry-run=client -o yaml | kubectl --context $(CTX) apply -f - +``` + +And add to the `helm upgrade --install` call: + +```makefile +--set queue.redis.auth.existingSecret=valkey-auth \ +--set queue.redis.auth.existingSecretKey=password \ +``` + +> **`plain-manifests-file` mode:** The `dist/install.yaml` is generated without `existingSecret` +> set (empty default), so no `REDIS_PASSWORD` env var is rendered. The e2e plain-manifests-file +> path currently does not test auth. This is a known gap; address it in a follow-up by adding a +> `sed` patch alongside the existing redis-addr patch in the Makefile. + +--- + +## 5. Documents to Remove or Archive + +Now that the consumer is fully implemented and all e2e tests pass, several design documents are +either fully superseded or describe decisions that are no longer open questions. + +### Move to `docs/past/` + +| Document | Reason | +|---|---| +| `docs/design/audit-consumer-next-steps.md` | All steps are marked done. The doc's remaining value is tracking the implementation history, which belongs in past/. | +| `docs/design/audit-webhook-redis-queue-plan.md` | "Not implemented yet" section is now fully implemented. "Remaining work" is done. The authoritative current-state doc is `webhook-audit-pipeline-current-state.md`. | +| `docs/design/valkey-adoption-guide.md` | The adoption question is settled. Valkey is required and the architecture described is now live. The Phased Adoption Plan is complete through step 4. Move to past/ and keep as a reference, but remove it from the active design/ surface. | +| `docs/future/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md` | NATS was considered and explicitly rejected in favour of Valkey. This is historical context only. | + +### Update in Place + +| Document | What to Update | +|---|---| +| `docs/design/webhook-audit-pipeline-current-state.md` | Add a note at the top that this is the authoritative current-state document for the audit pipeline. Remove the "still broken" IceCreamOrder section — that was fixed in a subsequent commit. | +| `docs/design/webhook-audit-e2e-remediation-status.md` | The "What Still Fails" section describes a failure that was subsequently fixed. Either remove that section or annotate it as resolved. The doc is still useful for understanding the remediation history. | +| `charts/gitops-reverser/README.md` | Update the quick start (section 4.1), prerequisites list, and the configuration reference table to replace `queue.redis.username`/`password` rows with `queue.redis.auth.*` rows. | + +### Keep As-Is + +| Document | Reason | +|---|---| +| `docs/design/best-practices-webhook-ingress.md` | Still relevant for ingress hardening decisions. | +| `docs/design/webhook-audit-pipeline-current-state.md` | Current and authoritative once the small updates above are made. | + +--- + +## 6. Summary of Changes + +| File | Change | +|---|---| +| `charts/gitops-reverser/values.yaml` | Replace `queue.redis.username`/`password` with `queue.redis.auth.existingSecret`, `existingSecretKey`, `username` | +| `charts/gitops-reverser/templates/deployment.yaml` | Remove `--audit-redis-password` arg; add `REDIS_PASSWORD` env var from secret ref; keep `--audit-redis-username` arg | +| `cmd/main.go` | Change `--audit-redis-password` default to `os.Getenv("REDIS_PASSWORD")` | +| `charts/gitops-reverser/README.md` | Fix quick start: official Valkey chart, auth-enabled, new namespace; update prerequisites and config reference table | +| `config/README.md` | Add Valkey auth secret and official chart install as pre-install steps for the single-YAML path | +| All user-facing docs | Replace every `gitops-reverser-system` namespace example with `gitops-reverser` | +| `test/e2e/setup/flux/values/valkey-values.yaml` | Enable auth, reference `valkey-auth` secret | +| `test/e2e/setup/flux/kustomization.yaml` | Add `secretGenerator` for `valkey-auth` in `valkey-e2e` namespace | +| `config/valkey-auth.yaml` | New: fixed Secret for controller namespace (committed test credential) | +| `config/kustomization.yaml` | Add `valkey-auth.yaml` to resources | +| `config/deployment.yaml` | Add `REDIS_PASSWORD` env var from `valkey-auth` secret | +| `Makefile` | Add `E2E_VALKEY_PASSWORD`, create secret before helm install, pass auth values | + +--- + +## 7. What Not to Change + +- The `--audit-redis-addr`, `--audit-redis-db`, `--audit-redis-stream`, `--audit-redis-max-len`, + and `--audit-redis-tls` flags are not sensitive and stay as CLI args. +- The e2e `config/deployment.yaml` can keep running against an unauthenticated Valkey for now. + Add a TODO comment pointing at this plan. +- The `internal/queue/` package does not need changes — `RedisAuditQueueConfig.AuthValue` already + accepts the password value regardless of how it was sourced. The change is purely in how the + value reaches the process. +- The `AuditConsumerConfig.AuthValue` field follows the same pattern. diff --git a/docs/design/remove-correlation-webhook.md b/docs/design/remove-correlation-webhook.md new file mode 100644 index 000000000..219c383dc --- /dev/null +++ b/docs/design/remove-correlation-webhook.md @@ -0,0 +1,188 @@ +# Remove the Correlation Webhook + +## What this document is + +A concrete plan for deleting the validating correlation webhook, the in-memory correlation +store, and all related username-guessing code now that the audit stream consumer provides +direct user attribution from audit events. + +--- + +## Architectural verdict: yes, this can go + +The correlation webhook (`/process-validating-webhook`) is a pure side-effect webhook: it +intercepts every Kubernetes API call, extracts the username from the admission request, stores +it in an in-memory LRU cache keyed on a content hash, and **always returns `allowed: true`**. +It never blocks or mutates any resource. + +Its only purpose was to bridge a gap: the watch path (informers) sees resource changes but +cannot see who made them. The webhook saw the user but not the full change; the correlation +store joined the two together — imperfectly, via a content-hash race. + +The audit stream consumer eliminates that gap entirely: + +``` +auditv1.Event.User.Username // always present; no guessing required +auditv1.Event.ImpersonatedUser.Username // also handled, in resolveUserInfo() +``` + +The consumer already calls `resolveUserInfo` and populates `git.UserInfo.Username` before +routing any event. The correlation store is no longer needed. + +### The one webhook that stays + +There are **two** webhooks in the codebase. Only the correlation webhook is removed here. + +| Webhook | Path | Admission control? | Decision | +|---|---|---|---| +| Correlation / event webhook | `/process-validating-webhook` | No — always allowed | **DELETE** | +| GitTarget validator | registered via `SetupGitTargetValidatorWebhook` | **Yes** — blocks duplicate (repo, branch, path) tuples; validates encryption config | **KEEP** | + +The GitTarget validator is doing real work: it is the uniqueness constraint that prevents two +GitTargets from writing to the same path in the same repository. It has nothing to do with +correlation and must remain. + +--- + +## What will change when the correlation webhook is removed + +### Watch path loses username attribution + +The watch path (informers in `internal/watch/manager.go`) calls `tryEnrichFromCorrelation` +today. After the correlation store is deleted, watch-path events will emit git commits with +an empty `UserInfo.Username`. This is already the fallback behaviour — the code handles +empty strings. The audit consumer path still provides usernames for every event it sees. + +**Implication:** during any window where the audit consumer is not running (e.g. pod restart, +cluster without audit logging), git commits may have no author name. This was also true +before the correlation webhook existed for clusters that did not deploy the webhook, so this +is not a regression. + +### Three e2e tests are deleted + +These three tests exist only to verify the correlation webhook infrastructure. They have no +value after the webhook is removed: + +| Test | Location | Why deleted | +|---|---|---| +| `"should route webhook traffic to the running controller pod"` | `test/e2e/e2e_test.go` | Tests webhook routing to pod — infrastructure gone | +| `"should have webhook registration configured"` | `test/e2e/e2e_test.go` | Asserts `ValidatingWebhookConfiguration` exists — removed | +| `"should receive webhook calls and process them successfully"` | `test/e2e/e2e_test.go` | Verifies `gitopsreverser_events_received_total` counter — metric removed | + +All other e2e tests continue to pass. The audit consumer path and GitTarget validator tests +are unaffected. + +--- + +## Complete list of deletions and modifications + +### Files to delete entirely + +| File | Why | +|---|---| +| `internal/correlation/store.go` | The correlation store itself | +| `internal/correlation/store_test.go` | Tests for the store | +| `internal/webhook/event_handler.go` | The correlation webhook handler | +| `internal/webhook/event_handler_test.go` | Tests for the handler | + +If `internal/correlation/` becomes empty after the above, delete the directory. + +### `cmd/main.go` + +- Remove import: `"github.com/ConfigButler/gitops-reverser/internal/correlation"` +- Remove constants: `correlationMaxEntries`, `correlationTTL` +- Remove block: `correlationStore := correlation.NewStore(...)` and the callback wiring +- Remove field: `CorrelationStore: correlationStore` from the `watchMgr` initialisation +- Remove block: `eventHandler` creation and `/process-validating-webhook` registration + +Keep everything related to: +- The audit handler (`/audit-webhook/{clusterID}`) +- `SetupGitTargetValidatorWebhook` +- The audit consumer (`queue.NewAuditConsumer`) + +### `internal/watch/manager.go` + +- Remove field: `CorrelationStore *correlation.Store` from the `Manager` struct +- Remove method: `tryEnrichFromCorrelation` (the method that queries the store) +- Remove the call site: wherever `tryEnrichFromCorrelation` is called during event processing +- Remove import: `"github.com/ConfigButler/gitops-reverser/internal/correlation"` + +The watch manager will still produce git commits; it just will not populate `UserInfo.Username` +unless a future path (e.g. passing user context through informer events) provides it. + +### `internal/telemetry/` + +Remove the following metrics (and their `prometheus.MustRegister` calls and any helper +references): + +- `WebhookCorrelationsTotal` (or equivalent) — correlation store write counter +- `EnrichHitsTotal` — correlation lookup hit counter +- `EnrichMissesTotal` — correlation lookup miss counter +- The `gitopsreverser_events_received_total` counter incremented by the correlation webhook + handler (distinct from the audit events counter which stays) + +Verify the exact metric names by reading `internal/telemetry/exporter.go` and grepping for +uses in `internal/webhook/event_handler.go`. + +### Config / manifests + +- `config/webhook.yaml` or equivalent: remove the `ValidatingWebhookConfiguration` stanza + that matches `*/*` resources and routes to `/process-validating-webhook`. Leave any stanza + for the GitTarget validator. +- `config/webhook/manifests.yaml`: same — remove the correlation webhook entry. +- Any Helm chart values or CRD/RBAC manifests that reference + `gitops-reverser-validating-webhook-configuration` by that name should be reviewed; the + name may be shared with the GitTarget validator, so verify before deleting. + +### `test/e2e/e2e_test.go` + +Remove the three test cases identified above. If they are inside a `Describe` block that +contains only these three tests, remove the entire `Describe` block. If the block has other +surviving tests, remove only the `It` entries. + +--- + +## Tests that confirm nothing broke + +After the deletion, the following must all pass: + +``` +make test # unit tests — no correlation or event_handler tests remain +make test-e2e # full e2e suite excluding the three deleted tests +make test-e2e-audit-redis # producer + consumer path end-to-end +``` + +Pay particular attention to any e2e test that asserts a git commit contains a non-empty +author name — if such a test exists it needs to be driven through the audit consumer path, +not the watch path. + +--- + +## What to leave alone + +- `internal/watch/` — keep the watch manager and all informer logic. The watch path is still + how the controller discovers resources initially (List+Watch). It just will not have + usernames. +- `internal/sanitize/` — used by both paths; not affected. +- `internal/correlation/store.go` references in `internal/watch/manager.go` other than the + enrichment call — there should be none, but verify with a grep. +- RBAC for the admission webhook server — the GitTarget validator still needs the + `ValidatingWebhookConfiguration` RBAC. Do not remove webhook RBAC generically; remove + only the specific rule/entry that covers `*/*` resources for the correlation path. + +--- + +## Order of work + +1. **Delete** `internal/correlation/store.go` and `store_test.go`. +2. **Delete** `internal/webhook/event_handler.go` and `event_handler_test.go`. +3. **Fix** `cmd/main.go`: remove imports, constants, correlation store init, `CorrelationStore` + field, and the `/process-validating-webhook` registration block. +4. **Fix** `internal/watch/manager.go`: remove `CorrelationStore` field, `tryEnrichFromCorrelation` + method, and the call site. +5. **Fix** `internal/telemetry/`: remove the three metrics and their registration. +6. **Fix** config manifests: remove the correlation `ValidatingWebhookConfiguration` entry. +7. **Fix** `test/e2e/e2e_test.go`: remove the three correlation webhook tests. +8. Run `make lint` and fix any remaining references the compiler or linter surfaces. +9. Run `make test` to confirm unit tests pass. +10. Run `make test-e2e` to confirm e2e tests pass with the three tests gone. diff --git a/docs/design/webhook-audit-e2e-remediation-status.md b/docs/design/webhook-audit-e2e-remediation-status.md new file mode 100644 index 000000000..70b8a4eab --- /dev/null +++ b/docs/design/webhook-audit-e2e-remediation-status.md @@ -0,0 +1,190 @@ +# Webhook Audit E2E Remediation Status + +## Purpose + +This document captures: + +1. The original remediation plan for the webhook-audit migration regressions. +2. What was implemented. +3. What validated successfully. +4. What is still failing after the latest clean-cluster `make test-e2e` run. +5. The updated understanding of Secret handling in the audit pipeline. + +## Original Problem Statement + +The move toward webhook-driven audit ingestion introduced two visible regressions: + +1. Audit consumer instability around missing Redis consumer groups (`NOGROUP`). +2. Lost username attribution in Git commits because watch-path events with empty `UserInfo` could win the race against audit-path events that carried the real user. + +There was also a strong suspicion that the Valkey persistence explanation was incomplete, and that at least one data-mapping or ingestion-path mismatch existed in the new pipeline. + +## Original Plan + +The plan we implemented was: + +### 1. Make audit authoritative for live mutating events when audit Redis is enabled + +- Add an internal runtime switch on the watch side. +- Audit Redis queueing is now always enabled: + - keep watch informers for snapshot / reconcile behavior + - stop routing live mutating watch events into `GitTargetEventStream` +- Let the audit consumer be the source of truth for live mutation commits. + +### 2. Harden the audit consumer against lost Redis groups + +- Ensure the consumer group exists on startup. +- Recreate it if `XREADGROUP` returns `NOGROUP`. +- Recreate it if `XAUTOCLAIM` returns `NOGROUP`. + +### 3. Stop destructive stream cleanup in e2e + +- Remove stream deletion in the audit producer e2e. +- Use unique object names and filtered assertions instead of deleting the shared stream key. + +### 4. Strengthen e2e verification + +- Keep the existing `jane@acme.com` author-attribution expectation. +- Strengthen the audit-consumer e2e so it also verifies commit author attribution, not just file creation. + +### 5. Preserve watch-based snapshot / reconcile behavior + +- Watchers should still drive cluster-state snapshots and rule-change reconciliation. +- The migration should not break snapshot-oriented paths like cluster-scoped CRD export. + +## Implemented Changes + +### Wiring + +- [cmd/main.go](/workspaces/gitops-reverser/cmd/main.go) + - `watch.Manager` now runs with `AuditLiveEventsEnabled: true`. + +### Watch Manager + +- [internal/watch/manager.go](/workspaces/gitops-reverser/internal/watch/manager.go) + - Added `AuditLiveEventsEnabled bool`. + +- [internal/watch/informers.go](/workspaces/gitops-reverser/internal/watch/informers.go) + - Watch-path live routing is skipped when audit is authoritative, with two important exceptions: + - `secrets` + - cluster-scoped resources + - This is narrower than the original broad suppression because the broader version broke valid e2e behavior. + +### Audit Consumer + +- [internal/queue/redis_audit_consumer.go](/workspaces/gitops-reverser/internal/queue/redis_audit_consumer.go) + - Added `NOGROUP` self-healing for: + - `XREADGROUP` + - `XAUTOCLAIM` + - Added `isNoGroupErr`. + +### Audit E2E + +- [test/e2e/audit_redis_e2e_test.go](/workspaces/gitops-reverser/test/e2e/audit_redis_e2e_test.go) + - Removed destructive `DEL` of the shared stream. + - Added commit-author verification for the audit consumer path. + +### Unit Test Coverage + +- [internal/watch/informers_test.go](/workspaces/gitops-reverser/internal/watch/informers_test.go) + - Added a test proving watch live routing is skipped when audit is authoritative. + - Added a test proving cluster-scoped live routing still works when audit is authoritative. + +- [internal/queue/redis_audit_consumer_test.go](/workspaces/gitops-reverser/internal/queue/redis_audit_consumer_test.go) + - Added tests for consumer group recreation on `NOGROUP`. + +### Test Stability + +- [internal/git/git_operations_test.go](/workspaces/gitops-reverser/internal/git/git_operations_test.go) + - Public network tests now skip on DNS / timeout failures rather than failing the whole suite in an offline environment. + +## Important Design Correction: Secrets and Audit Policy + +An important finding emerged while testing: + +- SOPS protects what gets committed to Git. +- SOPS does not protect the raw audit payload sent by kube-apiserver to the audit webhook. +- The Redis producer stores audit payloads as `payload_json`. +- Our sanitizer preserves Secret `data` / `binaryData`. + +This means removing `secrets` from the audit exclusion policy would expose Secret contents in the audit webhook / Redis pipeline before SOPS encryption happens. + +So the current safe position is: + +- Keep `secrets` excluded in [policy.yaml](/workspaces/gitops-reverser/test/e2e/cluster/audit/policy.yaml). +- Continue handling Secret live mutations via the watch path for now. + +That is why the final code keeps a `secrets` exception in [internal/watch/informers.go](/workspaces/gitops-reverser/internal/watch/informers.go). + +## Validation Performed + +The following validations passed locally: + +- `make fmt` +- `make lint` +- `make test` +- `docker info` + +Focused package tests also passed: + +- `go test ./internal/watch ./internal/queue` + +## Clean-Cluster E2E Result + +After the audit-policy discussion, the cluster was fully reset with: + +```bash +make clean-cluster +``` + +Then `make test-e2e` was rerun from a clean baseline. + +### What Passed in That Clean Run + +The following important specs passed: + +- Audit Redis producer stream test +- Audit Redis consumer commit test +- Secret encryption via WatchRule +- Secret encryption with generated age recipient +- ConfigMap commit via WatchRule +- ConfigMap deletion via WatchRule +- ClusterWatchRule CRD installation commit + +This means the original migration regressions appear fixed: + +- `NOGROUP` self-healing is working well enough for the exercised audit path. +- Audit-path author attribution is working. +- The race between watch-path ConfigMap events and audit-path ConfigMap events is no longer breaking the tested commit attribution path. +- The earlier cluster-scoped CRD regression introduced by overly broad watch suppression was fixed by allowing cluster-scoped live routing. + +### What Still Fails + +> **Resolved.** The IceCreamOrder failure described here was fixed in commit `c08c7c4` +> (correct custom-resource API group mapping in the audit consumer — `objectRef.apiGroup` + +> `objectRef.apiVersion` were not being combined correctly, causing CRs to be misrouted as core +> `/v1` resources). All e2e suites now pass. + +~~The clean-cluster `make test-e2e` run still fails overall.~~ + +~~Current failing spec: `Manager should create Git commit when IceCreamOrder is added via WatchRule`~~ + +## Current Interpretation + +The original webhook-audit remediation is complete. All originally failing specs pass. + +### Fixed + +- Audit consumer group recovery +- ConfigMap author attribution through the audit path +- Audit producer e2e isolation +- Audit consumer e2e author verification +- Cluster-scoped CRD export in audit-enabled mode +- Namespaced custom resource live commit path (IceCreamOrder) — fixed by correct API group mapping + +## Current Bottom Line + +`make test-e2e` passes on a clean cluster. The remediation work described in this document is done. + +For current architectural state and remaining rough edges, see +[webhook-audit-pipeline-current-state.md](webhook-audit-pipeline-current-state.md). diff --git a/docs/design/webhook-audit-pipeline-current-state.md b/docs/design/webhook-audit-pipeline-current-state.md new file mode 100644 index 000000000..b1dd74388 --- /dev/null +++ b/docs/design/webhook-audit-pipeline-current-state.md @@ -0,0 +1,321 @@ +# Webhook Audit Pipeline: Current State, Remaining Hackiness, and Secret Handling + +> **This is the authoritative current-state document for the audit ingestion pipeline.** +> For implementation history and the migration journey, see `docs/past/`. + +## Purpose + +This document captures the current state of the webhook-audit ingestion pipeline after the recent fixes, with a focus on: + +1. what is now working +2. how the effective event flow works +3. where the implementation is still uneven or fragile +4. how Secret handling works today +5. what better alternatives exist for Secret handling + +This document reflects the code and test state validated on 2026-04-09. All e2e suites pass. + +## Executive Summary + +The system is now green again: + +- `make lint` passes +- `make test` passes +- `make test-e2e` passes +- `make test-e2e-quickstart-manifest` passes +- `make test-e2e-quickstart-helm` passes + +The webhook-audit migration is now functionally working. The architecture is operationally uniform: + +- audit is the single authority for all live mutating events +- watch is still used for snapshot/reconcile behavior only +- Secrets flow through the audit path the same as any other resource + +The accepted tradeoff: raw Secret payloads are persisted in Valkey as `payload_json` before Git-side SOPS encryption applies. This is a deliberate choice — uniformity over a separate Secret ingestion path. The audit queue must be secured with auth and TLS as a result (see `final-redis-required-plan.md`). + +## What Was Broken + +There were three distinct failure classes involved. + +### 1. Consumer-group loss in Redis / Valkey + +The audit consumer could fail with `NOGROUP` when the stream or consumer group disappeared. + +This was real, but it was not the full explanation. + +### 2. Username attribution loss + +Watch-path events had empty `UserInfo`. + +Audit-path events had the real user, for example `jane@acme.com`. + +Because both paths could emit the same logical mutation, the watch event could win the race and produce the Git commit first. Once the file content was already written, the later audit event often became a no-op, so the correct author never made it into Git history. + +### 3. CRD deletion path regression + +The remaining red e2e after the first fixes was: + +- `Manager should delete Git file when IceCreamOrder CRD is deleted via ClusterWatchRule` + +This turned out to be two separate problems stacked together: + +1. cluster-scoped watch events were still live-routing in audit mode, which recreated the CRD file after deletion +2. rule-change snapshot reconciliation only refreshed cluster state, not repo state, so it could diff against stale repo contents and incorrectly conclude there was nothing to delete + +## Final Fixes That Landed + +## Audit authority for live mutations + +Now that audit queueing is required: + +- live watch-path mutation routing is suppressed for all resources +- audit consumer is the live mutation source of truth + +Implemented in: + +- [cmd/main.go](../../cmd/main.go) +- [internal/watch/manager.go](../../internal/watch/manager.go) +- [internal/watch/informers.go](../../internal/watch/informers.go) + +### Why this matters + +This removes the duplicate-source race for live-mutating resources: + +- no more watch-vs-audit race for ConfigMaps +- no more watch-vs-audit race for CRDs +- no more empty-author watch event suppressing later audit attribution + +## Redis consumer self-healing + +The audit consumer now recreates the consumer group when Redis returns `NOGROUP` from: + +- `XREADGROUP` +- `XAUTOCLAIM` + +Implemented in: + +- [internal/queue/redis_audit_consumer.go](../../internal/queue/redis_audit_consumer.go) +- [internal/queue/redis_audit_consumer_test.go](../../internal/queue/redis_audit_consumer_test.go) + +## Correct custom-resource API group mapping + +Custom-resource audit events were previously misread from `objectRef.apiVersion` alone. + +For CRs, audit events commonly carry: + +- `objectRef.apiGroup` +- `objectRef.apiVersion` + +separately. + +That meant `shop.example.com/v1` resources could be interpreted as core `/v1/...` resources. + +The consumer now combines: + +- `ObjectRef.APIGroup` +- `ObjectRef.APIVersion` + +correctly. + +Implemented in: + +- [internal/queue/redis_audit_consumer.go](../../internal/queue/redis_audit_consumer.go) +- [internal/queue/redis_audit_queue.go](../../internal/queue/redis_audit_queue.go) +- [internal/queue/redis_audit_consumer_test.go](../../internal/queue/redis_audit_consumer_test.go) +- [internal/queue/redis_audit_queue_test.go](../../internal/queue/redis_audit_queue_test.go) + +## Snapshot reconcile now refreshes repo state too + +During rule-change reconciliation, the watch manager originally emitted only: + +- `RequestClusterState` + +That was not enough. `FolderReconciler` compares: + +- current cluster state +- current repo state + +If repo state is stale, deletes can be missed. + +The watch manager now emits both: + +- `RequestRepoState` +- `RequestClusterState` + +for affected GitTargets. + +Implemented in: + +- [internal/watch/manager.go](../../internal/watch/manager.go) + +This is what fixed the last CRD deletion failure. + +## Current Effective Pipeline + +The easiest way to understand the system now is by resource class. + +### Normal resources, audit Redis enabled + +Examples: + +- ConfigMaps +- CRDs +- custom resources like `IceCreamOrder` + +Effective live path: + +1. kube-apiserver emits audit event +2. audit webhook receives it +3. producer stores raw audit payload in Redis / Valkey +4. audit consumer reads stream entry +5. audit consumer matches WatchRule / ClusterWatchRule +6. audit consumer routes a `git.Event` +7. BranchWorker writes the file and commits with real user attribution + +Watch informers still exist, but they do not route live mutating events for these resources when audit mode is enabled. + +### Secrets, audit Redis enabled + +Effective live path: + +1. kube-apiserver emits Secret audit event +2. audit webhook receives it +3. producer stores raw Secret audit payload in Redis / Valkey +4. audit consumer reads the stream entry +5. audit consumer routes the `git.Event` +6. Git write path applies SOPS encryption if configured +7. encrypted Secret content is committed to Git + +There is no longer a Secret-specific live-path exception. + +### Snapshot / reconcile path + +Regardless of resource type, watch manager still performs cluster-state snapshotting for GitTargets during: + +- initial sync +- rule changes +- discovery changes +- retry loops for unavailable GVRs + +That snapshot flow is still watch-manager driven, not audit-consumer driven. + +## Current Secret Handling + +Secrets are no longer a special case. The watch-path exception for Secrets was removed: when +`AuditLiveEventsEnabled` is true, all live watch events are suppressed for all resources, including +Secrets. The audit policy also no longer excludes Secrets. + +The current flow for a live Secret mutation: + +1. kube-apiserver emits a Secret audit event (with `data`/`binaryData` in the payload) +2. audit webhook receives it +3. producer stores the raw event JSON in Valkey stream field `payload_json` +4. audit consumer reads it, extracts the object via `extractObject` +5. the object passes through `sanitize.Sanitize` (which currently preserves `data`/`binaryData`) +6. Git write path applies SOPS encryption if configured before committing + +**The accepted tradeoff:** raw Secret material is persisted in Valkey before SOPS encryption applies. +This is why the audit queue must be secured. An unauthenticated Valkey with Secrets in the stream +is a plaintext Secret exfiltration point. + +**Why this is still the correct choice over the old exception:** the exception created split-brain +provenance (audit for everything, watch for Secrets), which caused author-attribution races and +added hidden complexity. Uniformity is operationally safer even if it means a stricter Valkey +security requirement. + +**Future improvement:** producer-side redaction of sensitive fields before writing to `payload_json`. +This would shrink the security blast radius without reintroducing a separate ingestion path. + +## Where the Architecture Is Still Uneven + +The system is correct and all suites pass, but there are still rough edges worth tracking. + +## 1. Audit payload persistence is too raw + +The audit producer stores raw `payload_json`. + +That is convenient for fidelity and debugging, but it means the security boundary is weak: + +- if the audit policy admits sensitive content +- Redis becomes a persistence layer for that content + +This is the core reason the Secret exception exists. + +## 2. Snapshot/reconcile is still broad and somewhat blunt + +Rule changes can trigger snapshot reconciles across affected GitTargets, which is correct, but still noisy: + +- all affected targets are collected +- repo and cluster state are re-read +- large atomic batches are emitted + +This is robust, but not especially surgical. + +It is more “refresh the world for this target” than “precisely reconcile only the minimal impacted subset”. + +## 3. Unavailable-GVR handling still affects nearby flows + +When a CRD disappears, the watch manager continues tracking unavailable GVRs and retrying them. + +That behavior is useful, but it still creates side-channel complexity: + +- discovery filtering changes +- retry reconciliation fires later +- unrelated GitTargets can get swept into broader snapshot churn + +It is better than before, but still not elegant. + +## 4. Snapshot logic and live logic are still conceptually separate systems + +The system now has: + +- audit-based live mutation ingestion +- watch-based cluster-state reconciliation + +Those two systems are coordinated, but they are not the same model. + +That means bugs tend to appear at the seams: + +- deduplication +- stale state +- source precedence +- rule-change timing + +## Future Improvement: Reduce Payload Sensitivity in the Queue + +The cleanest long-term fix is to stop storing full raw audit payloads and instead write only the +fields the consumer actually needs for routing and Git writing. This would dramatically reduce the +security blast radius of a compromised Valkey instance. + +Near-term practical step: redact `data`, `stringData`, and `binaryData` in the producer before +writing `payload_json`, similar to how the sanitizer works for Git output. This keeps the current +architecture but removes the most sensitive content from the queue. + +## Current State Summary + +The system is in a good operational state. All validation suites pass. + +### Clean and solid + +- audit is the single authority for all live mutations (including Secrets) +- audit consumer self-heals `NOGROUP` +- custom-resource audit API group mapping is correct +- username attribution is preserved through the audit path +- CRD delete flow works +- snapshot/reconcile still works correctly via the watch path + +### Still uneven + +- audit persistence stores raw `payload_json`, including raw Secret material +- snapshot/reconcile is broad rather than surgical +- watch and audit remain two conceptually separate systems coordinated at the seams + +## Practical Bottom Line + +Functionally solid. The main remaining architectural debt is raw payload persistence in Valkey — and +the mitigation for that is requiring auth on the queue, which is now documented and planned in +`final-redis-required-plan.md`. + +The main remaining architectural debt is Secret handling. + +Today’s Secret exception is justified, but it is still a workaround for the fact that the audit pipeline persists raw webhook payloads before redaction. diff --git a/docs/idea-application-editting.md b/docs/idea-application-editting.md new file mode 100644 index 000000000..7f59a218c --- /dev/null +++ b/docs/idea-application-editting.md @@ -0,0 +1,131 @@ +# Idea: GitOps-backed ArgoCD Application editing + +## Context + +gitops-reverser is currently positioned as a tool for "intent clusters" — test environments where you +explore changes before promoting them. This document explores one concrete and powerful use case for +that positioning: making the ArgoCD GUI a first-class GitOps authoring tool. + +## The idea + +When running ArgoCD in a test environment, every change you make through the GUI could automatically +be captured in git. The flow would look like this: + +1. A new ArgoCD `Application` is created (through the GUI or otherwise) + → gitops-reverser creates a new git branch and annotates the `Application` with the branch name. +2. The `Application` spec changes + → gitops-reverser commits the new state to that branch. +3. The `Application` is deleted + → gitops-reverser deletes the branch. +4. The branch is merged to main + → gitops-reverser removes the annotation. From this point on, the configuration lives in main + and is active on both test and production. + +The result: anyone who knows how to use the ArgoCD GUI can author GitOps-managed infrastructure +without ever touching a YAML file or understanding branching strategy. The PR is the handoff point — +the moment where "I'm done experimenting" becomes "ship it." + +## Why this is compelling + +- **Low barrier to entry.** The target user understands GitOps and Kubernetes but doesn't want to + manually manage branches and YAML. This pattern does that for them. +- **Clean lifecycle.** The branch mirrors the Application's lifetime exactly. There is no ambiguity + about what is in progress vs. what is live. +- **Bootstrapping is not a concern here.** Setting up ArgoCD itself is an advanced task, done by + platform engineers or automation. This pattern targets the users who come *after* that — the people + who create Applications day-to-day and want their changes tracked without extra friction. + +## Open design questions + +### Environment-specific configuration +The most significant gap. A test Application might have `replicas: 1`, no resource limits, and point +to a test branch. The production version needs different values for all three. The recommended approach +is a Kustomize overlay per environment that injects the environment-specific values (branch name, +resource limits, replica count) on top of a shared base. gitops-reverser would commit the base; the +overlay handles the rest. + +A ConfigMap-merge approach is also possible but adds abstraction that may not pay for itself. + +### The "config branch as source of truth" problem +If a user sets the `Application`'s `spec.source.targetRevision` to the gitops-reverser-managed branch +(instead of main), they have made the experiment branch the source of truth. This is probably not what +they intended. Should gitops-reverser detect and warn about this? Should it rewrite the field on merge? +This needs a clear policy. + +### Merge trigger: PR or automatic? +Two models: +- **PR-gated**: gitops-reverser opens a pull request when a merge signal is detected. A human approves + and merges. Safe, auditable, fits most team workflows. +- **Automatic**: gitops-reverser merges the branch directly when some condition is met (e.g. CI passes, + or the Application is explicitly marked "ready"). Faster, but harder to recover from mistakes. + +Both are valid. The PR model is the safer default and maps well to the annotation-removal lifecycle +described above. + +### Deletion semantics +"Branch deleted when Application is deleted" is the clean version of the lifecycle, but it is +destructive. Deleting a test Application to clean up cluster resources is routine; silently deleting +the git branch along with it may surprise users. An alternative: deletion closes (but does not delete) +the branch, or opens a "discard this?" PR. The right answer depends on how recoverable the user +expects deletions to be. + +### Auto-removing the Application on PR close +If the branch is deleted (or the PR is closed without merging), should gitops-reverser delete the +corresponding `Application` from the cluster? This would make the lifecycle fully bidirectional — git +events drive cluster state just as cluster state drives git. It is a powerful property but also +dangerous: a mistakenly closed PR would take down a running workload. + +## Grouping: what belongs on the same branch? + +One branch per `Application` is too granular. A user clicking around in the ArgoCD GUI in one session +will touch multiple Applications and their dependent resources. Creating a separate branch per object +produces a fragmented, unmergeable mess. + +There are a few strategies for deciding what belongs together, and they are not mutually exclusive: + +**Time-based batching.** If changes from the same source arrive within a short window (say 30–60 +seconds of each other), group them into one branch. This is low-effort to implement and works well for +interactive sessions. The risk is that two unrelated users happen to make changes in the same window +and their work gets combined. + +**Author-based grouping.** All changes from the same user go to the same branch, at least within a +session. This is the most intuitive model. It requires knowing who made the change — which is exactly +what the audit webhook pipeline in gitops-reverser is being built to provide. The audit log gives you +username attribution per event; grouping by author then becomes a natural downstream use of that +signal. + +**CRD relationship mapping.** This is the most powerful approach and the most complex. Many Kubernetes +resources reference each other: an `Application` references a source repo and destination namespace; a +`Deployment` owns `ReplicaSets`; a `Service` selects `Pods` by label. By building a lightweight +reference map — walking `ownerReferences`, label selectors, and known cross-resource fields — you can +infer with high probability that a set of changes belongs together, even if they come from different +API calls. You could represent this as a configurable "affinity map" on the WatchRule CRD: something +like `groupWith: [apps/v1/Deployment, v1/Service]` to express that changes to these resource types +should land on the same branch as the Application that owns them. + +In practice, a combination of author + time is probably the right starting point. The reference map +is the long-term vision but requires more design work to get right. + +## Known limitation: branch divergence and file conflicts + +This is a real drawback and should be stated honestly. + +Branches created by gitops-reverser start from main at creation time. As time passes and main moves +forward, those branches diverge. The longer a branch lives, the harder it becomes to merge — +especially if the same file was modified on main in the meantime (e.g. a shared `Deployment` that +another team already updated and promoted). + +This problem is not unique to gitops-reverser, but the pattern amplifies it because branches are +created automatically and may linger without the user realising. A few partial mitigations: + +- **Encourage short-lived branches.** The design should push users toward merge-early behaviour. A + branch that is open for more than a few days should surface a warning. +- **Detect conflicts early.** Before committing to a branch, gitops-reverser could check whether the + target file already exists on main and flag a divergence risk in the annotation or in the PR + description. +- **Rebase rather than merge.** When the PR is created, rebasing onto the current tip of main reduces + the diff and makes conflicts more obvious earlier. + +None of these fully solve the problem. If the same file genuinely exists on both the branch and main +with different content, a human needs to resolve it. This is an inherent limitation of the multi-branch +approach and is worth documenting clearly for users so they are not surprised. diff --git a/docs/future/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md b/docs/past/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md similarity index 100% rename from docs/future/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md rename to docs/past/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md diff --git a/docs/past/audit-consumer-next-steps.md b/docs/past/audit-consumer-next-steps.md new file mode 100644 index 000000000..bd9d118ef --- /dev/null +++ b/docs/past/audit-consumer-next-steps.md @@ -0,0 +1,191 @@ +# Audit Webhook Consumer: Current State and Next Steps + +## Purpose + +This document is the practical starting point for implementing the audit stream consumer. +It maps all existing design documents to what they actually cover, describes the current +code state precisely, and defines the concrete missing work. + +--- + +## Existing documents and what they are good for + +| Document | Use it for | +|---|---| +| [audit-webhook-redis-queue-plan.md](audit-webhook-redis-queue-plan.md) | **Start here.** Most authoritative. Defines what is done, what is not, and the open design question on handoff boundary. | +| [valkey-adoption-guide.md](valkey-adoption-guide.md) | Stream consumer patterns: XREADGROUP, XACK, XAUTOCLAIM, lock keys. Good reference when writing consumer code. | +| [multi-cluster-audit-ingestion-implications.md](multi-cluster-audit-ingestion-implications.md) | Future scope only. Do not let it distract from the single-cluster consumer. | +| [best-practices-webhook-ingress.md](best-practices-webhook-ingress.md) | Operational notes on audit ingress (TLS, mTLS, backpressure). Useful for hardening, not for consumer implementation. | +| [../future/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md](../future/AUDIT_WEBHOOK_NATS_ARCHITECTURE_PROPOSAL.md) | Historical. NATS was considered and rejected in favour of Valkey. Read only for context on why the current direction was chosen. | +| [../past/audit-webhook-experimental-design.md](../past/audit-webhook-experimental-design.md) | Original metrics-only design. Fully superseded. | + +--- + +## Exact current state in code + +### What is done + +**Producer: [internal/queue/redis_audit_queue.go](../../internal/queue/redis_audit_queue.go)** +- Writes to Valkey stream `gitopsreverser.audit.events.v1` via `XADD`. +- Fields written: `event_id`, `audit_id`, `cluster_id`, `verb`, `api_version`, `resource`, + `namespace`, `name`, `user`, `stage_timestamp`, `payload_json` (full audit event JSON). +- `event_id` is a deterministic SHA-256 hash built in [`buildEventID`](../../internal/queue/redis_audit_queue.go#L138) — dedup key material is already there. +- Optional approximate trimming via `maxLen`. + +**Ingress handler: [internal/webhook/audit_handler.go](../../internal/webhook/audit_handler.go)** +- Receives POST at `/audit-webhook/{clusterID}`. +- Decodes the Kubernetes `audit.k8s.io/v1 EventList`. +- Enqueues to Redis if `Queue != nil`, returns 500 on enqueue failure (correct at-least-once semantics). +- Then: increments metrics, optionally dumps to file, **logs and does nothing else**. +- [Line 229](../../internal/webhook/audit_handler.go#L229): `// For now we hardly do a thing`. + +**Wiring: [cmd/main.go](../../cmd/main.go)** +- Redis queue is created during startup from the configured `--audit-redis-*` settings. +- Wired into `AuditHandler` as the `Queue` field. +- No consumer runnable is registered. The stream accumulates indefinitely. + +**E2E coverage** +- [cmd/main_audit_server_test.go](../../cmd/main_audit_server_test.go) verifies events land in the stream after Kubernetes object changes. +- The stream write path is tested. The consumer path has no test coverage yet. + +### What does not exist + +- No XREADGROUP consumer loop anywhere in the codebase. +- No consumer group creation (`XGROUP CREATE`). +- No XACK after processing. +- No XAUTOCLAIM for stuck message recovery. +- No runnable in `cmd/main.go` for the consumer lifecycle. +- No handoff from audit stream into the existing git processing pipeline. + +--- + +## The existing pipeline the consumer must feed into + +The git write pipeline already works and should be reused, not replaced. + +``` +EventRouter.RouteEvent(providerName, providerNamespace, branch, git.Event) + -> WorkerManager.GetWorkerForTarget(...) + -> BranchWorker.Enqueue(git.Event) + -> git write / commit / push +``` + +- [`EventRouter.RouteEvent`](../../internal/watch/event_router.go#L72) — entry point +- [`WorkerManager.GetWorkerForTarget`](../../internal/git/worker_manager.go) — worker lookup +- [`BranchWorker.Enqueue`](../../internal/git/branch_worker.go#L172) — puts event into the in-memory write channel + +A [`git.Event`](../../internal/git/types.go#L119) needs: + +```go +type Event struct { + Object *unstructured.Unstructured // sanitized Kubernetes object + Identifier types.ResourceIdentifier // GVK + namespace + name + Operation string // CREATE, UPDATE, DELETE + UserInfo UserInfo // Username, UID — THIS is what audit gives us natively + Path string // from GitTarget.spec.path + GitTargetName string + GitTargetNamespace string + BootstrapOptions pathBootstrapOptions +} +``` + +The `UserInfo.Username` is the primary value that the audit stream provides over the +current validating webhook + in-memory correlation approach. It is available directly +from `auditv1.Event.User.Username` (or `ImpersonatedUser.Username` when set), and is +already extracted into the stream entry's `user` field. + +--- + +## Open design question (from the plan doc) + +The plan doc identifies this as the central unresolved question: + +> We still need to decide the cleanest handoff boundary into the current `EventRouter` +> and branch-worker architecture so we do not create a second partially overlapping pipeline. + +**Recommendation:** Feed the consumer directly into `EventRouter.RouteEvent(...)`. + +Reasons: +- `EventRouter` already handles target lookup, worker dispatch, and stream buffering + during initial reconciliation ([`GitTargetEventStream`](../../internal/reconcile/git_target_event_stream.go)). +- The consumer only needs to build a `git.Event` from the audit payload and call + `RouteEvent`. No new routing layer needed. +- The existing validating webhook → correlation store → watch path can stay in place + during the transition and be removed in a follow-up once the audit consumer is proven. + +--- + +## Concrete missing pieces, in order + +### ~~Step 1: Build a `git.Event` from a stream entry~~ ✅ Done + +Implemented in [`internal/queue/redis_audit_consumer.go`](../../internal/queue/redis_audit_consumer.go). + +- `parseAuditEvent` reads `payload_json` from the stream entry and unmarshals the full `auditv1.Event`. +- `verbToOperation` maps HTTP verbs to `OperationType` (create → CREATE, update/patch → UPDATE, delete/deletecollection → DELETE). Read-only verbs are skipped. +- Rule matching calls `ruleStore.GetMatchingRules` and `ruleStore.GetMatchingClusterRules` directly, passing the parsed group/version/resource/operation. +- `splitAPIVersion` splits `"apps/v1"` into `("apps", "v1")` and `"v1"` into `("", "v1")`. +- The existing `RuleStore` API was sufficient — no changes to `rulestore/store.go` were needed. + +Routing uses `EventRouter.RouteToGitTargetEventStream` (same path as the watch manager informers), not `RouteEvent` directly. This goes through the `GitTargetEventStream` buffering layer. + +### ~~Step 2: Deserialize the object~~ ✅ Done + +Implemented in `extractObject` in [`internal/queue/redis_audit_consumer.go`](../../internal/queue/redis_audit_consumer.go). + +- For non-DELETE operations: uses `ResponseObject.Raw`. +- For DELETE: uses `RequestObject.Raw`. +- Falls back to a minimal stub (apiVersion + kind + namespace + name) when neither is present. +- Passes the result through [`sanitize.Sanitize`](../../internal/sanitize/sanitize.go#L33). + +### ~~Step 3: Consumer runnable in `cmd/main.go`~~ ✅ Done + +`AuditConsumer` implements `manager.Runnable` in [`internal/queue/redis_audit_consumer.go`](../../internal/queue/redis_audit_consumer.go). + +- `Start` calls `XGROUP CREATE … MKSTREAM` (idempotent via `BUSYGROUP` error check), then enters the read loop. +- `XREADGROUP GROUP … COUNT 50 BLOCK 2s STREAMS … >`. +- `XAUTOCLAIM` ticker fires every 30s, reclaiming entries idle longer than 60s. +- Consumer ID defaults to `$POD_NAME`, falling back to `"gitopsreverser-consumer-0"`. + +Wired in [`cmd/main.go`](../../cmd/main.go) during startup and registered via `mgr.Add`. `NeedLeaderElection() = true`. + +### ~~Step 4: XACK only after safe handoff~~ ✅ Done + +`XACK` is called in `ackMessage` after `RouteToGitTargetEventStream` returns. Since `BranchWorker.Enqueue` is synchronous (puts the event into the in-memory channel before returning), this gives at-least-once semantics at the stream boundary. + +Poison-pill messages (unparseable payload, missing ObjectRef) are also ACKed immediately to prevent the consumer from blocking on them indefinitely. + +### ~~Step 5: E2E test for the consumer path~~ ✅ Done + +Implemented as `Describe("Audit Redis Consumer", Label("audit-redis"), ...)` in [`test/e2e/audit_redis_e2e_test.go`](../../test/e2e/audit_redis_e2e_test.go). + +- BeforeAll creates a namespace, applies git secrets, calls `applySOPSAgeKeyToNamespace` (required for the GitTarget encryption gate), connects to Valkey, and sets up GitProvider + GitTarget + WatchRule targeting `e2e/audit-consumer-test` on `main`. +- The `It` block creates a ConfigMap (name includes `GinkgoRandomSeed` so it is unique per run), waits for the audit stream entry to appear via `XRange`, then does `git pull` in a loop until the file `e2e/audit-consumer-test/v1/configmaps/{ns}/{cm}.yaml` exists with non-zero size. +- The stream is **not** deleted before the test (deleting the stream key also destroys the consumer group; cmName uniqueness is sufficient for isolation). +- Labels: `audit-redis` — runs together with the producer test via `make test-e2e-audit-redis`. + +--- + +## What to leave alone for now + +- The validating webhook + correlation store path ([`internal/correlation/store.go`](../../internal/correlation/store.go)). + Keep it running in parallel. Remove it only after the audit consumer has been validated end-to-end in e2e. +- The `BranchWorker` in-memory queue. Replacing it with a Redis-backed queue is the HA + follow-on work, not part of this consumer slice. +- Multi-cluster routing. The `clusterID` is already in the stream entry. Wire it into + log labels and metrics now, but do not implement per-cluster rule filtering yet. +- Idempotency markers (dedupe TTL in Redis). Implement after the consumer works. The + `event_id` field in the stream entry is already the dedup key. + +--- + +## Files changed + +| File | Change | +|---|---| +| [internal/queue/redis_audit_consumer.go](../../internal/queue/redis_audit_consumer.go) | **New.** XREADGROUP loop, XAUTOCLAIM, rule matching, object extraction, routing. | +| [internal/queue/redis_audit_consumer_test.go](../../internal/queue/redis_audit_consumer_test.go) | **New.** 34 unit tests, 91% coverage. | +| [cmd/main.go](../../cmd/main.go) | Wires `AuditConsumer` into the manager during startup. | +| [test/e2e/audit_redis_e2e_test.go](../../test/e2e/audit_redis_e2e_test.go) | **New `Describe` block.** Consumer e2e: GitProvider + GitTarget + WatchRule setup, audit event → git commit assertion (Step 5). | + +All steps are now complete. diff --git a/docs/design/audit-webhook-redis-queue-plan.md b/docs/past/audit-webhook-redis-queue-plan.md similarity index 96% rename from docs/design/audit-webhook-redis-queue-plan.md rename to docs/past/audit-webhook-redis-queue-plan.md index 9c2689ca5..de110acae 100644 --- a/docs/design/audit-webhook-redis-queue-plan.md +++ b/docs/past/audit-webhook-redis-queue-plan.md @@ -34,7 +34,7 @@ Today the flow is: Kube API Audit Backend -> /audit-webhook/{clusterID} -> Audit Handler - -> Valkey/Redis stream enqueue (optional, feature-flagged) + -> Valkey/Redis stream enqueue (required) -> stream persistence only ``` @@ -88,14 +88,13 @@ The queue can be enabled at runtime with dedicated audit Redis flags. Implemented controls: -1. `--audit-redis-enabled` -2. `--audit-redis-addr` -3. `--audit-redis-username` -4. `--audit-redis-password` -5. `--audit-redis-db` -6. `--audit-redis-stream` -7. `--audit-redis-max-len` -8. `--audit-redis-tls` +1. `--audit-redis-addr` +2. `--audit-redis-username` +3. `--audit-redis-password` +4. `--audit-redis-db` +5. `--audit-redis-stream` +6. `--audit-redis-max-len` +7. `--audit-redis-tls` Deployment status: diff --git a/docs/design/valkey-adoption-guide.md b/docs/past/valkey-adoption-guide.md similarity index 98% rename from docs/design/valkey-adoption-guide.md rename to docs/past/valkey-adoption-guide.md index afdc5a037..0452bf509 100644 --- a/docs/design/valkey-adoption-guide.md +++ b/docs/past/valkey-adoption-guide.md @@ -7,7 +7,7 @@ Helm chart, and why Valkey Streams are a good fit for the event pipeline. 1. Use **Valkey Streams + Consumer Groups** as the durable event bus. 2. Use **Valkey lock keys** (`SET key value NX EX`) for single active writer per `{repoURL, branch}` partition. -3. Keep Valkey as an **optional external dependency by default**. +3. Treat Valkey as a **required external dependency** for supported GitOps Reverser deployments. 4. Add an **optional Helm subchart** for convenience (disabled by default), not as the only deployment mode. This aligns with the current project direction in `README.md` (planned HA via Valkey) and with the rewrite goals in diff --git a/docs/runbooks/node-notready-inotify.md b/docs/runbooks/node-notready-inotify.md new file mode 100644 index 000000000..1a7fc0d87 --- /dev/null +++ b/docs/runbooks/node-notready-inotify.md @@ -0,0 +1,148 @@ +# Runbook: k3d Agent Nodes Stuck in NotReady — inotify Exhaustion + +## Summary + +One or more k3d agent nodes become `NotReady` after a Docker daemon restart, and a +`docker restart` of the affected container does not fix the problem. The node stays +stuck with `NodeStatusUnknown` and the kubelet never starts. + +--- + +## Symptoms + +``` +kubectl get nodes +NAME STATUS ROLES AGE VERSION +k3d-gitops-reverser-test-e2e-agent-1 NotReady 14d v1.35.2+k3s1 +k3d-gitops-reverser-test-e2e-agent-2 NotReady 14d v1.35.2+k3s1 +``` + +`kubectl describe node ` shows: + +``` +Reason: NodeStatusUnknown +Message: Kubelet stopped posting node status. +``` + +`docker logs ` shows a tight loop of: + +``` +couldn't get current server API group list: + Get "http://localhost:8080/api?timeout=32s": dial tcp [::1]:8080: connect: connection refused +``` + +`docker exec ps aux` shows **no k3s process** — only the entrypoint shell and a +`sleep 3` from the uncordon retry loop. + +--- + +## Root Cause + +### 1. Immediate cause — inotify instance limit exhausted + +The kubelet inside the agent container runs **cAdvisor**, which calls `inotify_init()` +to set up filesystem watches for container metrics. When the host kernel's +`fs.inotify.max_user_instances` limit is exhausted, `inotify_init()` returns +`EMFILE ("too many open files")` and cAdvisor (and therefore the kubelet) fails to +start: + +``` +E kubelet.go:1728] "Failed to start cAdvisor" err="inotify_init: too many open files" +``` + +The default limit on many Linux hosts is **128 instances**, which is far too low +for this workload. + +### 2. Why the entrypoint gets stuck + +The k3d entrypoint (`/bin/k3d-entrypoint.sh`) follows this sequence: + +```sh +/bin/k3s "$@" & # start k3s/kubelet in background +k3s_pid=$! + +until kubectl uncordon "$HOSTNAME"; do sleep 3; done # ← hangs here +``` + +When k3s fails to start (due to the cAdvisor/inotify error), it exits immediately. +The `until kubectl uncordon` loop then runs indefinitely because: + +- There is no kubeconfig inside the agent container — `kubectl` falls back to + `localhost:8080`, which does not exist inside the agent. +- The loop condition never becomes true, so the container appears running but + produces nothing but the `localhost:8080` connection-refused flood. + +### 3. Contributing factors — why the limit was reached + +The host `fs.inotify.max_user_instances` (default **128**) is a **per-user kernel +limit shared across all containers** on the Docker host. The following concurrent +workloads each consume inotify instances: + +| Workload | inotify consumers | +|---|---| +| VSCode devcontainer #1 (`gitops-reverser`) | VS Code server, language servers (gopls, etc.), file watchers | +| VSCode devcontainer #2 (`talks`) | Same: VS Code server, language servers, file watchers | +| k3d server node | kubelet + cAdvisor | +| k3d agent-0 | kubelet + cAdvisor + all running pods | +| k3d agent-1 | kubelet + cAdvisor | +| k3d agent-2 | kubelet + cAdvisor | + +With two VSCode devcontainers open simultaneously (each running gopls and multiple +VS Code extensions), the 128-instance limit was exhausted before the agent kubelet +could allocate its own instance. + +### 4. Memory + +Memory pressure was **not** the direct cause. At the time of investigation: + +- Total: 15 GiB, Used: ~10 GiB, Available: ~4.6 GiB, Swap: none +- `agent-0` alone held ~6 GiB (running all cluster workloads) +- The two devcontainers held ~2.5 GiB combined + +While the system is running low on headroom (no swap configured), memory did not +trigger the failure. However, if agent-1 and agent-2 recover and start scheduling +pods again, available memory could become a secondary concern. + +--- + +## Fix + +### Immediate (ephemeral — lost on host reboot) + +Run a privileged container against the Docker host to raise the limits: + +```bash +docker run --rm --privileged alpine \ + sysctl -w fs.inotify.max_user_watches=524288 \ + fs.inotify.max_user_instances=512 +``` + +Then restart the affected agent(s): + +```bash +docker restart k3d-gitops-reverser-test-e2e-agent-1 \ + k3d-gitops-reverser-test-e2e-agent-2 +``` + +### Permanent (survives host reboot) + +Add to `/etc/sysctl.conf` (or `/etc/sysctl.d/99-inotify.conf`) **on the Docker host +machine**: + +``` +fs.inotify.max_user_watches=524288 +fs.inotify.max_user_instances=512 +``` + +Then apply: `sudo sysctl --system` + +--- + +## Prevention (inotify) + +- Keep the permanent sysctl settings above on the Docker host. +- Be aware that opening a second VSCode devcontainer against any large repo will + meaningfully increase inotify consumption — this is what pushed the system over + the edge here. +- If the Docker host is a shared machine or a CI runner, consider adding these + settings as part of the host bootstrap/provisioning script. diff --git a/hack/connect-remote-k3d.sh b/hack/connect-remote-k3d.sh new file mode 100644 index 000000000..20347dbdc --- /dev/null +++ b/hack/connect-remote-k3d.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# connect-remote-k3d.sh +# Fetches the k3d kubeconfig from a remote host and sets up a local SSH tunnel. +# Usage: ./connect-remote-k3d.sh [remote-host] [cluster-name] +set -euo pipefail + +REMOTE_HOST="${1:-remote-dev.z65}" +CLUSTER_NAME="${2:-}" +KUBECONFIG_PATH="${HOME}/.kube/remote-k3d.yaml" + +echo "==> Connecting to ${REMOTE_HOST}..." + +# k3d needs a proper login shell on the remote side to work correctly +if [[ -n "$CLUSTER_NAME" ]]; then + RAW_KUBECONFIG=$(ssh "${REMOTE_HOST}" bash -l -c "k3d kubeconfig get ${CLUSTER_NAME}") +else + RAW_KUBECONFIG=$(ssh "${REMOTE_HOST}" bash -l -c "k3d kubeconfig get --all") +fi + +if [[ -z "$RAW_KUBECONFIG" ]]; then + echo "ERROR: Failed to fetch kubeconfig from ${REMOTE_HOST}" >&2 + exit 1 +fi + +echo "$RAW_KUBECONFIG" > "${KUBECONFIG_PATH}" +echo "==> Kubeconfig saved to ${KUBECONFIG_PATH}" + +# Extract port from the server URL (e.g. https://0.0.0.0:43567) +API_PORT=$(grep 'server:' "${KUBECONFIG_PATH}" | grep -oP ':\K[0-9]+$' | head -1) + +if [[ -z "$API_PORT" ]]; then + echo "ERROR: Could not determine API server port from kubeconfig" >&2 + exit 1 +fi + +echo "==> API server port: ${API_PORT}" + +# Rewrite the server address to localhost +sed -i "s|https://[^:]*:${API_PORT}|https://127.0.0.1:${API_PORT}|g" "${KUBECONFIG_PATH}" + +# Set up SSH tunnel if not already active +if ss -tlnp 2>/dev/null | grep -q ":${API_PORT} " || \ + ss -tlnp 2>/dev/null | grep -q ":${API_PORT}$"; then + echo "==> Tunnel on port ${API_PORT} already active, skipping" +else + echo "==> Setting up SSH tunnel: localhost:${API_PORT} -> ${REMOTE_HOST}:${API_PORT}" + ssh -f -N -L "${API_PORT}:127.0.0.1:${API_PORT}" "${REMOTE_HOST}" + echo "==> Tunnel established" +fi + +echo "" +echo "Done! Use your cluster with:" +echo " export KUBECONFIG=${KUBECONFIG_PATH}" +echo " kubectl get nodes" diff --git a/hack/e2e/deploy-controller.sh b/hack/e2e/deploy-controller.sh new file mode 100755 index 000000000..d9958bf8a --- /dev/null +++ b/hack/e2e/deploy-controller.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Update the controller deployment image and wait for rollout. +# Called after the install stamp is ready and the image is loaded. +# +# Inputs (env): +# - CTX (required): kube context +# - NAMESPACE (required): namespace containing the deployment +# - PROJECT_IMAGE (required): container image reference to set +# - IMAGE_DELIVERY_MODE (optional): load|pull; defaults to "load" +# - IMAGE_PULL_SECRET_NAME (optional): imagePullSecret to attach when using pull mode +# - IMAGE_PULL_SECRET_REGISTRY (optional): registry hostname for docker-registry secret creation +# - IMAGE_PULL_SECRET_USERNAME (optional): registry username for docker-registry secret creation +# - IMAGE_PULL_SECRET_PASSWORD (optional): registry password/token for docker-registry secret creation +# - CONTROLLER_CONTAINER (required): container name within the deployment +# - CONTROLLER_DEPLOY_SELECTOR (required): label selector to find the deployment + +: "${CTX:?CTX is required}" +: "${NAMESPACE:?NAMESPACE is required}" +: "${PROJECT_IMAGE:?PROJECT_IMAGE is required}" +: "${CONTROLLER_CONTAINER:?CONTROLLER_CONTAINER is required}" +: "${CONTROLLER_DEPLOY_SELECTOR:?CONTROLLER_DEPLOY_SELECTOR is required}" + +IMAGE_DELIVERY_MODE="${IMAGE_DELIVERY_MODE:-load}" +IMAGE_PULL_SECRET_NAME="${IMAGE_PULL_SECRET_NAME:-}" +IMAGE_PULL_SECRET_REGISTRY="${IMAGE_PULL_SECRET_REGISTRY:-}" +IMAGE_PULL_SECRET_USERNAME="${IMAGE_PULL_SECRET_USERNAME:-}" +IMAGE_PULL_SECRET_PASSWORD="${IMAGE_PULL_SECRET_PASSWORD:-}" + +ensure_image_pull_secret() { + if [[ -z "${IMAGE_PULL_SECRET_NAME}" ]]; then + return 0 + fi + + if [[ -n "${IMAGE_PULL_SECRET_REGISTRY}" && -n "${IMAGE_PULL_SECRET_USERNAME}" && -n "${IMAGE_PULL_SECRET_PASSWORD}" ]]; then + kubectl --context "${CTX}" -n "${NAMESPACE}" \ + create secret docker-registry "${IMAGE_PULL_SECRET_NAME}" \ + --docker-server="${IMAGE_PULL_SECRET_REGISTRY}" \ + --docker-username="${IMAGE_PULL_SECRET_USERNAME}" \ + --docker-password="${IMAGE_PULL_SECRET_PASSWORD}" \ + --dry-run=client -o yaml \ + | kubectl --context "${CTX}" apply -f - + fi + + kubectl --context "${CTX}" -n "${NAMESPACE}" patch deployment "${deploy}" --type=merge -p "$(cat </dev/null \ + | wc -l | tr -d ' ')" + +if [[ "${count}" -ne 1 ]]; then + echo "ERROR: Expected exactly 1 Deployment matching '${CONTROLLER_DEPLOY_SELECTOR}'" \ + "in namespace '${NAMESPACE}', found ${count}" >&2 + kubectl --context "${CTX}" -n "${NAMESPACE}" \ + get deploy -l "${CONTROLLER_DEPLOY_SELECTOR}" -o wide || true + exit 1 +fi + +deploy="$(kubectl --context "${CTX}" -n "${NAMESPACE}" \ + get deploy -l "${CONTROLLER_DEPLOY_SELECTOR}" \ + -o jsonpath='{.items[0].metadata.name}')" + +current_image="$(kubectl --context "${CTX}" -n "${NAMESPACE}" \ + get deploy "${deploy}" \ + -o jsonpath="{.spec.template.spec.containers[?(@.name==\"${CONTROLLER_CONTAINER}\")].image}")" +desired_replicas="$(kubectl --context "${CTX}" -n "${NAMESPACE}" \ + get deploy "${deploy}" -o jsonpath='{.spec.replicas}')" +available_replicas="$(kubectl --context "${CTX}" -n "${NAMESPACE}" \ + get deploy "${deploy}" -o jsonpath='{.status.availableReplicas}')" + +if [[ "${IMAGE_DELIVERY_MODE}" == "pull" ]]; then + ensure_image_pull_secret +fi + +if [[ "${current_image}" == "${PROJECT_IMAGE}" ]] \ + && [[ -n "${available_replicas}" ]] \ + && [[ "${available_replicas}" == "${desired_replicas}" ]]; then + echo "deployment/${deploy} already running ${PROJECT_IMAGE} (${available_replicas}/${desired_replicas} available); skipping" + exit 0 +fi + +echo "Setting deployment/${deploy} container '${CONTROLLER_CONTAINER}' to image '${PROJECT_IMAGE}'" +kubectl --context "${CTX}" -n "${NAMESPACE}" \ + set image "deployment/${deploy}" "${CONTROLLER_CONTAINER}=${PROJECT_IMAGE}" +kubectl --context "${CTX}" -n "${NAMESPACE}" \ + rollout status "deployment/${deploy}" --timeout=180s diff --git a/hack/e2e/ensure-valkey-auth.sh b/hack/e2e/ensure-valkey-auth.sh new file mode 100755 index 000000000..e49d6ac6c --- /dev/null +++ b/hack/e2e/ensure-valkey-auth.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Ensure the gitops-reverser namespace and valkey-auth Secret exist before +# helm install runs. Both operations are idempotent (dry-run + apply). +# +# Inputs (env): +# - CTX (required): kube context +# - NAMESPACE (required): namespace to create and populate +# - E2E_VALKEY_PASSWORD (required): password to put in the Secret + +: "${CTX:?CTX is required}" +: "${NAMESPACE:?NAMESPACE is required}" +: "${E2E_VALKEY_PASSWORD:?E2E_VALKEY_PASSWORD is required}" + +kubectl --context "${CTX}" create namespace "${NAMESPACE}" \ + --dry-run=client -o yaml \ + | kubectl --context "${CTX}" apply -f - + +kubectl --context "${CTX}" -n "${NAMESPACE}" \ + create secret generic valkey-auth \ + --from-literal=password="${E2E_VALKEY_PASSWORD}" \ + --dry-run=client -o yaml \ + | kubectl --context "${CTX}" apply -f - diff --git a/hack/e2e/inject-webhook-tls.sh b/hack/e2e/inject-webhook-tls.sh new file mode 100755 index 000000000..5d9e5f1e2 --- /dev/null +++ b/hack/e2e/inject-webhook-tls.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Generates the final audit webhook kubeconfig from the audit root CA and +# kube-apiserver client certificate, then restarts the k3d server node so +# kube-apiserver picks it up. +# +# Must run after the operator is deployed and cert-manager has issued the audit TLS secrets. +# Updates the mounted audit webhook kubeconfig in place after certificate issuance. + +set -euo pipefail + +CTX="${CTX:-k3d-gitops-reverser-test-e2e}" +CLUSTER_NAME="${CLUSTER_NAME:-gitops-reverser-test-e2e}" +NAMESPACE="${NAMESPACE:-gitops-reverser}" +WEBHOOK_CONFIG="${WEBHOOK_CONFIG:-test/e2e/cluster/audit/webhook-config.yaml}" +SERVER_CONTAINER="k3d-${CLUSTER_NAME}-server-0" +TMP_WEBHOOK_CONFIG="$(mktemp "${TMPDIR:-/tmp}/audit-webhook-config.XXXXXX")" + +cleanup() { + rm -f "${TMP_WEBHOOK_CONFIG}" +} +trap cleanup EXIT + +echo "🔐 Writing webhook config with stable root CA trust and apiserver client cert..." +bash hack/generate-audit-webhook-kubeconfig.sh > "${TMP_WEBHOOK_CONFIG}" +mkdir -p "$(dirname "${WEBHOOK_CONFIG}")" +mv "${TMP_WEBHOOK_CONFIG}" "${WEBHOOK_CONFIG}" + +echo "🔄 Restarting k3d server node '${SERVER_CONTAINER}' to reload webhook config..." +docker restart "${SERVER_CONTAINER}" + +echo "⏳ Waiting for cluster API to come back..." +for i in $(seq 1 40); do + if kubectl --context "${CTX}" --request-timeout=5s get ns >/dev/null 2>&1; then + echo "✅ Cluster healthy — webhook TLS injection complete" + exit 0 + fi + sleep 3 +done + +echo "❌ Cluster did not recover after node restart" >&2 +exit 1 diff --git a/hack/e2e/install-config-dir.sh b/hack/e2e/install-config-dir.sh new file mode 100755 index 000000000..1728b750c --- /dev/null +++ b/hack/e2e/install-config-dir.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Render and apply the controller via kustomize config-dir mode. +# Writes the rendered manifest to STAMP_FILE on success. +# +# Inputs (env): +# - CTX (required): kube context +# - NAMESPACE (required): target namespace +# - PROJECT_IMAGE (required): controller image to inject +# - KUSTOMIZE (optional): kustomize binary; defaults to "kustomize" +# - KUBECTL (optional): kubectl binary; defaults to "kubectl" +# - STAMP_FILE (required): path to write the rendered install.yaml + +: "${CTX:?CTX is required}" +: "${NAMESPACE:?NAMESPACE is required}" +: "${PROJECT_IMAGE:?PROJECT_IMAGE is required}" +: "${STAMP_FILE:?STAMP_FILE is required}" +: "${E2E_VALKEY_PASSWORD:?E2E_VALKEY_PASSWORD is required}" + +KUSTOMIZE="${KUSTOMIZE:-kustomize}" +KUBECTL="${KUBECTL:-kubectl}" + +kubectl --context "${CTX}" create namespace "${NAMESPACE}" \ + --dry-run=client -o yaml \ + | kubectl --context "${CTX}" apply -f - + +kubectl --context "${CTX}" -n "${NAMESPACE}" \ + create secret generic valkey-auth \ + --from-literal=password="${E2E_VALKEY_PASSWORD}" \ + --dry-run=client -o yaml \ + | kubectl --context "${CTX}" apply -f - + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +cp -R config "${tmpdir}/config" +(cd "${tmpdir}/config" && "${KUSTOMIZE}" edit set namespace "${NAMESPACE}") +(cd "${tmpdir}/config" && "${KUSTOMIZE}" edit set image gitops-reverser="${PROJECT_IMAGE}") + +(cd "${tmpdir}/config" && "${KUSTOMIZE}" build .) \ + | tee "${tmpdir}/install.yaml" \ + | "${KUBECTL}" --context "${CTX}" apply -f - + +# Replace stamp only when content changed (keeps mtime stable on no-op runs). +if [[ -f "${STAMP_FILE}" ]] && cmp -s "${tmpdir}/install.yaml" "${STAMP_FILE}"; then + : # no change +else + mv "${tmpdir}/install.yaml" "${STAMP_FILE}" +fi diff --git a/hack/e2e/install-plain-manifests.sh b/hack/e2e/install-plain-manifests.sh new file mode 100755 index 000000000..58e7cc032 --- /dev/null +++ b/hack/e2e/install-plain-manifests.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Render and apply the controller via the pre-built dist/install.yaml (plain-manifests-file mode). +# Patches the Redis address and injects the e2e ClusterIP, then applies via kustomize namespace overlay. +# Writes the rendered manifest to STAMP_FILE on success. +# +# Inputs (env): +# - CTX (required): kube context +# - NAMESPACE (required): target namespace +# - DEFAULT_AUDIT_REDIS_ADDR (required): Redis address baked into dist/install.yaml +# - E2E_AUDIT_REDIS_ADDR (required): Redis address to substitute in for e2e +# - E2E_CONTROLLER_SERVICE_CLUSTER_IP (required): ClusterIP to inject for the controller Service +# - KUSTOMIZE (optional): kustomize binary; defaults to "kustomize" +# - KUBECTL (optional): kubectl binary; defaults to "kubectl" +# - STAMP_FILE (required): path to write the rendered install.yaml + +: "${CTX:?CTX is required}" +: "${NAMESPACE:?NAMESPACE is required}" +: "${DEFAULT_AUDIT_REDIS_ADDR:?DEFAULT_AUDIT_REDIS_ADDR is required}" +: "${E2E_AUDIT_REDIS_ADDR:?E2E_AUDIT_REDIS_ADDR is required}" +: "${E2E_CONTROLLER_SERVICE_CLUSTER_IP:?E2E_CONTROLLER_SERVICE_CLUSTER_IP is required}" +: "${STAMP_FILE:?STAMP_FILE is required}" + +KUSTOMIZE="${KUSTOMIZE:-kustomize}" +KUBECTL="${KUBECTL:-kubectl}" + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +cp dist/install.yaml "${tmpdir}/install.yaml" + +# Patch Redis address +sed -i \ + "s|--audit-redis-addr=${DEFAULT_AUDIT_REDIS_ADDR}|--audit-redis-addr=${E2E_AUDIT_REDIS_ADDR}|" \ + "${tmpdir}/install.yaml" + +# Inject fixed ClusterIP for the controller Service +perl -0pi -e \ + "s/type: ClusterIP\n/type: ClusterIP\n clusterIP: ${E2E_CONTROLLER_SERVICE_CLUSTER_IP}\n/" \ + "${tmpdir}/install.yaml" + +# Wrap in a kustomize overlay to set the target namespace +cat >"${tmpdir}/kustomization.yaml" </dev/null + "${KUSTOMIZE}" build . +) | tee "${tmpdir}/rendered-install.yaml" \ + | "${KUBECTL}" --context "${CTX}" apply -f - + +# Replace stamp only when content changed (keeps mtime stable on no-op runs). +if [[ -f "${STAMP_FILE}" ]] && cmp -s "${tmpdir}/rendered-install.yaml" "${STAMP_FILE}"; then + : # no change +else + mv "${tmpdir}/rendered-install.yaml" "${STAMP_FILE}" +fi diff --git a/hack/e2e/load-image.sh b/hack/e2e/load-image.sh new file mode 100755 index 000000000..b125d4439 --- /dev/null +++ b/hack/e2e/load-image.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Load a container image into the k3d cluster and pin it against containerd GC. +# If the image is missing locally and PROJECT_IMAGE_PROVIDED is set, fails fast. +# If the image is missing locally and it's the local build, triggers a make rebuild. +# +# Why pinning matters +# ------------------- +# After k3d image import, the image lives in containerd's k8s.io namespace inside +# each node container. Containerd's CRI plugin runs image GC on behalf of the kubelet: +# when disk pressure occurs, or after an image goes unreferenced (e.g. because the +# deployment was deleted during a clean reinstall), containerd will evict it. +# +# Once evicted, the stamp file still says the image is loaded — because the stamp +# only tracks whether WE loaded it, not whether containerd still has it. On the next +# run Make skips the import and the rollout fails with ImagePullBackOff. +# +# The fix: after import we set the io.cri-containerd.pinned=pinned label on the image +# in every node's containerd. The CRI plugin checks this label before evicting any +# image and skips pinned ones unconditionally. This makes the stamp reliable: if it +# says loaded, containerd still has it. +# +# Inputs (env): +# - CTX (required): kube context (k3d-) +# - CLUSTER_NAME (required): k3d cluster name (without k3d- prefix) +# - PROJECT_IMAGE (required): image reference to load +# - PROJECT_IMAGE_PROVIDED (optional): non-empty means image came from outside; empty = local build +# - IMAGE_DELIVERY_MODE (optional): load|pull; defaults to "load" +# - CONTROLLER_ID_STAMP (required): make stamp target to rebuild if the local image is missing +# - CONTAINER_TOOL (optional): container tool binary; defaults to "docker" +# - K3D (optional): k3d binary; defaults to "k3d" +# - STAMP_FILE (required): path to write the loaded image ID + +: "${CTX:?CTX is required}" +: "${CLUSTER_NAME:?CLUSTER_NAME is required}" +: "${PROJECT_IMAGE:?PROJECT_IMAGE is required}" +: "${CONTROLLER_ID_STAMP:?CONTROLLER_ID_STAMP is required}" +: "${STAMP_FILE:?STAMP_FILE is required}" + +CONTAINER_TOOL="${CONTAINER_TOOL:-docker}" +K3D="${K3D:-k3d}" +IMAGE_DELIVERY_MODE="${IMAGE_DELIVERY_MODE:-load}" +IMAGE_REPO="${PROJECT_IMAGE%:*}" +IMAGE_TAG="${PROJECT_IMAGE##*:}" + +if [[ "${IMAGE_REPO}" == "${IMAGE_TAG}" ]]; then + IMAGE_REPO="${PROJECT_IMAGE}" + IMAGE_TAG="latest" +fi + +# Normalize to the fully-qualified reference containerd uses internally. +# Rules mirror Docker's normalization: +# no slash → docker.io/library/IMAGE +# slash, no dot/colon in first component → docker.io/ORG/IMAGE +# dot or colon in first component → registry is explicit +containerd_ref() { + local repo="$1" tag="$2" + local first="${repo%%/*}" + if [[ "${repo}" != *"/"* ]]; then + echo "docker.io/library/${repo}:${tag}" + elif [[ "${first}" != *"."* && "${first}" != *":"* && "${first}" != "localhost" ]]; then + echo "docker.io/${repo}:${tag}" + else + echo "${repo}:${tag}" + fi +} + +cluster_node_names() { + "${CONTAINER_TOOL}" ps --format '{{.Names}}' \ + | grep -E "^k3d-${CLUSTER_NAME}-(server|agent)-[0-9]+$" \ + | sort +} + +node_image_refs() { + local node_name="$1" + "${CONTAINER_TOOL}" exec "${node_name}" ctr -n k8s.io images ls -q 2>/dev/null || true +} + +find_pin_refs() { + local node_name="$1" + local normalized_ref="$2" + local raw_ref="$3" + local repo="$4" + local tag="$5" + + node_image_refs "${node_name}" | awk \ + -v normalized_ref="${normalized_ref}" \ + -v raw_ref="${raw_ref}" \ + -v repo="${repo}" \ + -v tag="${tag}" ' + $0 == normalized_ref || $0 == raw_ref { print; next } + index($0, repo ":") == 1 && $0 ~ (":" tag "$") { print; next } + index($0, "/" repo ":") > 0 && $0 ~ (":" tag "$") { print; next } + ' | sort -u +} + +pin_imported_image() { + local node_name="$1" + local normalized_ref="$2" + local raw_ref="$3" + local repo="$4" + local tag="$5" + local attempt refs ref + + for attempt in $(seq 1 10); do + refs="$(find_pin_refs "${node_name}" "${normalized_ref}" "${raw_ref}" "${repo}" "${tag}")" + if [[ -n "${refs}" ]]; then + while IFS= read -r ref; do + [[ -n "${ref}" ]] || continue + "${CONTAINER_TOOL}" exec "${node_name}" \ + ctr -n k8s.io images label "${ref}" io.cri-containerd.pinned=pinned \ + >/dev/null + done <<<"${refs}" + return 0 + fi + sleep 1 + done + + echo "ERROR: imported image ref for ${raw_ref} not found in ${node_name} after import" >&2 + echo "Known refs in ${node_name}:" >&2 + node_image_refs "${node_name}" >&2 + return 1 +} + +# Import the image directly into each node's containerd by piping docker save output. +# This bypasses k3d's volume-based import mechanism, which silently fails in +# Docker-outside-of-Docker CI environments (the shared image volume tarball is not +# accessible inside the k3s node containers). +import_image_direct() { + local ref="$1" + local node_name + while IFS= read -r node_name; do + echo " → importing into ${node_name}" + "${CONTAINER_TOOL}" save "${PROJECT_IMAGE}" \ + | "${CONTAINER_TOOL}" exec -i "${node_name}" \ + ctr -n k8s.io images import - + pin_imported_image "${node_name}" "${ref}" "${PROJECT_IMAGE}" "${IMAGE_REPO}" "${IMAGE_TAG}" + done < <(cluster_node_names) +} + +if [[ "${IMAGE_DELIVERY_MODE}" == "pull" ]]; then + echo "Skipping k3d image import for ${PROJECT_IMAGE}; cluster will pull from registry" + echo "pull:${PROJECT_IMAGE}" >"${STAMP_FILE}" + exit 0 +fi + +if ! "${CONTAINER_TOOL}" image inspect "${PROJECT_IMAGE}" >/dev/null 2>&1; then + if [[ -z "${PROJECT_IMAGE_PROVIDED:-}" ]]; then + echo "Local image ${PROJECT_IMAGE} missing; rebuilding..." + make "${CONTROLLER_ID_STAMP}" + else + echo "ERROR: PROJECT_IMAGE=${PROJECT_IMAGE} not found locally" >&2 + exit 2 + fi +fi + +img_id="$("${CONTAINER_TOOL}" inspect --format='{{.Id}}' "${PROJECT_IMAGE}")" + +if [[ -f "${STAMP_FILE}" ]] && [[ "$(<"${STAMP_FILE}")" == "${img_id}" ]]; then + echo "${PROJECT_IMAGE} (${img_id}) is already loaded into ${CTX} (stamp matches)" + exit 0 +fi + +ref="$(containerd_ref "${IMAGE_REPO}" "${IMAGE_TAG}")" + +echo "Loading ${PROJECT_IMAGE} (${img_id}) into ${CTX}" +import_image_direct "${ref}" + +echo "${img_id}" >"${STAMP_FILE}" diff --git a/hack/e2e/wait-flux-services.sh b/hack/e2e/wait-flux-services.sh new file mode 100755 index 000000000..276e5e33a --- /dev/null +++ b/hack/e2e/wait-flux-services.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Wait for all Flux-managed HelmReleases and Kustomizations to reach Ready. +# Skips suspended Kustomizations and known demo-only namespaces. +# +# Inputs (env): +# - CTX (required): kube context +# - FLUX_SERVICES_WAIT_TIMEOUT (optional): per-resource wait timeout; default 600s + +: "${CTX:?CTX is required}" +FLUX_SERVICES_WAIT_TIMEOUT="${FLUX_SERVICES_WAIT_TIMEOUT:-600s}" + +flux_ready_count=0 +echo "⏳ Waiting for Flux-managed installations to become ready..." + +for kind in \ + helmreleases.helm.toolkit.fluxcd.io \ + kustomizations.kustomize.toolkit.fluxcd.io +do + if [[ "${kind}" == "kustomizations.kustomize.toolkit.fluxcd.io" ]]; then + resources="$(kubectl --context "${CTX}" get "${kind}" --all-namespaces \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend \ + --no-headers 2>/dev/null \ + | awk '$3 != "true" && $1 != "podinfos-preview" && $1 != "podinfos-production" && $1 != "podinfos-intent" {print $1 " " $2}')" + else + resources="$(kubectl --context "${CTX}" get "${kind}" --all-namespaces \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null)" + fi + + [[ -z "${resources}" ]] && continue + + resource_count="$(printf '%s\n' "${resources}" | sed '/^$/d' | wc -l | tr -d ' ')" + flux_ready_count="$((flux_ready_count + resource_count))" + + while IFS=' ' read -r namespace name; do + [[ -n "${namespace}" ]] || continue + kubectl --context "${CTX}" -n "${namespace}" \ + wait "${kind}/${name}" \ + --for=condition=Ready \ + --timeout="${FLUX_SERVICES_WAIT_TIMEOUT}" + done <<<"${resources}" +done + +if [[ "${flux_ready_count}" -le 0 ]]; then + echo "ERROR: no Flux-managed e2e ready-check resources found" >&2 + exit 1 +fi + +echo "✓ Flux-managed installations ready: ${flux_ready_count}" diff --git a/hack/generate-audit-webhook-kubeconfig.sh b/hack/generate-audit-webhook-kubeconfig.sh new file mode 100755 index 000000000..5082ce82c --- /dev/null +++ b/hack/generate-audit-webhook-kubeconfig.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Prints an audit webhook kubeconfig that trusts the stable audit root CA and +# authenticates kube-apiserver with a dedicated client certificate. + +set -euo pipefail + +CTX="${CTX:-k3d-gitops-reverser-test-e2e}" +NAMESPACE="${NAMESPACE:-gitops-reverser}" +AUDIT_CLUSTER_ID="${AUDIT_CLUSTER_ID:-kind-e2e}" +AUDIT_WEBHOOK_SERVER_URL="${AUDIT_WEBHOOK_SERVER_URL:-https://127.0.0.1:30444/audit-webhook/${AUDIT_CLUSTER_ID}}" +AUDIT_TLS_SERVER_NAME="${AUDIT_TLS_SERVER_NAME:-gitops-reverser-audit.${NAMESPACE}.svc}" +WAIT_RETRIES="${WAIT_RETRIES:-60}" +WAIT_SECONDS="${WAIT_SECONDS:-2}" + +if [[ -n "${AUDIT_ROOT_CA_SECRET:-}" ]]; then + AUDIT_ROOT_CA_SECRET_CANDIDATES=("${AUDIT_ROOT_CA_SECRET}") +else + AUDIT_ROOT_CA_SECRET_CANDIDATES=("gitops-reverser-audit-root-ca") +fi + +if [[ -n "${AUDIT_CLIENT_SECRET:-}" ]]; then + AUDIT_CLIENT_SECRET_CANDIDATES=("${AUDIT_CLIENT_SECRET}") +else + AUDIT_CLIENT_SECRET_CANDIDATES=("audit-client-cert" "gitops-reverser-audit-client-cert") +fi + +secret_data() { + local secret_name="$1" + local key="$2" + kubectl --context "${CTX}" get secret "${secret_name}" \ + --namespace "${NAMESPACE}" \ + -o "jsonpath={.data.${key}}" 2>/dev/null || true +} + +wait_for_any_secret_key() { + local key="$1" + shift + local secret_names=("$@") + local value="" secret_name + + for _ in $(seq 1 "${WAIT_RETRIES}"); do + for secret_name in "${secret_names[@]}"; do + value="$(secret_data "${secret_name}" "${key}")" + if [ -n "${value}" ]; then + printf '%s' "${value}" + return 0 + fi + done + sleep "${WAIT_SECONDS}" + done + + echo "timed out waiting for one of [${secret_names[*]}] ${key}" >&2 + return 1 +} + +ROOT_CA_B64="$(wait_for_any_secret_key 'tls\.crt' "${AUDIT_ROOT_CA_SECRET_CANDIDATES[@]}")" +CLIENT_CERT_B64="$(wait_for_any_secret_key 'tls\.crt' "${AUDIT_CLIENT_SECRET_CANDIDATES[@]}")" +CLIENT_KEY_B64="$(wait_for_any_secret_key 'tls\.key' "${AUDIT_CLIENT_SECRET_CANDIDATES[@]}")" + +cat < attributed to user B (first for that key) -// - Watch event 2 (true) -> attributed to user B -// - Watch event 3 (false) -> attributed to user B (same entry) -// -// This tests the store's single entry handling of rapid changes. -func TestStore_RapidChangesWithContentReuse(t *testing.T) { - store := NewStore(60*time.Second, 100) - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - - // Simulate sanitized YAML for different spec values - specFalse := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - cool: false -`) - - specTrue := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - cool: true -`) - - // Webhook events (rapid succession): - // 1. User B changes to false - key1 := GenerateKey(id, "UPDATE", specFalse) - store.Put(key1, "userB") - - // 2. User B changes to true - key2 := GenerateKey(id, "UPDATE", specTrue) - store.Put(key2, "userB") - - // 3. User A changes to false (same content as step 1, overwrites userB) - key3 := GenerateKey(id, "UPDATE", specFalse) - store.Put(key3, "userA") - - // Verify keys - assert.Equal(t, key1, key3, "Same content should produce same key") - assert.NotEqual(t, key1, key2, "Different content should produce different keys") - - // Watch events arrive (delayed, in order): - // Watch event 1: spec changed to false - entry1, found1 := store.Get(key1) - require.True(t, found1, "First watch event should find correlation entry") - assert.Equal(t, "userB", entry1.Username, - "Should get the first user for false") - - // Watch event 2: spec changed to true - entry2, found2 := store.Get(key2) - require.True(t, found2, "Second watch event should find correlation entry") - assert.Equal(t, "userB", entry2.Username, "userB should be attributed") - - // Watch event 3: spec changed to false (can be accessed multiple times) - entry3, found3 := store.Get(key3) - require.True(t, found3, "Third watch should still find correlation") - assert.Equal(t, "userB", entry3.Username, "Should still get userB") -} - -// TestStore_QueueSolution demonstrates a potential solution using multi-value queues. -// This is a design exploration for handling the rapid-change scenario. -func TestStore_QueueSolution(t *testing.T) { - t.Skip("Design exploration - not yet implemented") - - // Proposed solution: Store multiple entries per key in a FIFO queue - // - Put appends to queue (with max queue depth per key) - // - GetAndDelete removes from front (FIFO) - // - // This would preserve the order of changes to the same content: - // - Webhook: false by userB -> queue[key_false] = [userB] - // - Webhook: true by userB -> queue[key_true] = [userB] - // - Webhook: false by userA -> queue[key_false] = [userB, userA] - // - Watch: false -> dequeues userB (correct!) - // - Watch: true -> dequeues userB (correct!) - // - Watch: false -> dequeues userA (correct!) - // - // Trade-offs: - // - More complex implementation - // - Higher memory usage (multiple entries per key) - // - Requires bounded queue depth per key -} diff --git a/internal/correlation/status_changes_integration_test.go b/internal/correlation/status_changes_integration_test.go deleted file mode 100644 index 53c6c5e7e..000000000 --- a/internal/correlation/status_changes_integration_test.go +++ /dev/null @@ -1,324 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package correlation - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - - "github.com/ConfigButler/gitops-reverser/internal/sanitize" - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -// TestStatusOnlyChanges_CorrelationBehavior validates how correlation store handles -// multiple status-only updates that result in identical sanitized content. -func TestStatusOnlyChanges_CorrelationBehavior(t *testing.T) { - store := NewStore(60*time.Second, 1000) - - // Base object (spec that won't change) - baseObj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - "spec": map[string]interface{}{ - "containers": []interface{}{ - map[string]interface{}{ - "name": "app", - "image": "nginx:1.0", - }, - }, - }, - }, - } - - id := types.ResourceIdentifier{ - Group: "", - Version: "v1", - Resource: "pods", - Namespace: "default", - Name: "test-pod", - } - - // Simulate 5 status-only updates by different users - statusUpdates := []struct { - status string - username string - }{ - {"Pending", "user1@example.com"}, - {"Running", "user2@example.com"}, - {"Running", "user3@example.com"}, // Same status, different user - {"Terminating", "user4@example.com"}, - {"Terminated", "user5@example.com"}, - } - - var correlationKeys []string - var sanitizedYAMLs [][]byte - - t.Log("=== Phase 1: Webhook stores correlation entries ===") - for i, update := range statusUpdates { - // Create object with different status - obj := baseObj.DeepCopy() - unstructured.SetNestedField(obj.Object, update.status, "status", "phase") - - // Sanitize (removes status) - sanitized := sanitize.Sanitize(obj) - yaml, err := sanitize.MarshalToOrderedYAML(sanitized) - require.NoError(t, err) - sanitizedYAMLs = append(sanitizedYAMLs, yaml) - - // Generate correlation key - key := GenerateKey(id, "UPDATE", yaml) - correlationKeys = append(correlationKeys, key) - - // Store in correlation (like webhook does) - store.Put(key, update.username) - - keyDisplay := key - if len(key) > 50 { - keyDisplay = key[:50] + "..." - } - t.Logf(" Update %d: status=%s, username=%s, key=%s", - i+1, update.status, update.username, keyDisplay) - } - - t.Log("") - t.Log("=== Analysis: Correlation Key Uniqueness ===") - // Check if all sanitized YAMLs are identical (they should be - status removed) - allIdentical := true - for i := 1; i < len(sanitizedYAMLs); i++ { - if string(sanitizedYAMLs[i]) != string(sanitizedYAMLs[0]) { - allIdentical = false - t.Logf(" YAML %d differs from YAML 0", i) - } - } - - if allIdentical { - t.Log(" ✓ All sanitized YAMLs are IDENTICAL (status removed)") - t.Log(" ✓ This means all correlation keys are THE SAME") - - // Verify all keys are identical - for i := 1; i < len(correlationKeys); i++ { - assert.Equal(t, correlationKeys[0], correlationKeys[i], - "All status-only updates should produce same correlation key") - } - } else { - t.Log(" ✗ Sanitized YAMLs differ (unexpected)") - } - - t.Log("") - t.Log("=== Phase 2: Correlation Store State ===") - t.Logf(" Total correlation entries: %d", store.Size()) - t.Log(" Expected: 1 entry (first user wins)") - - // Since all keys are identical, store has: - // - 1 unique key - // - 1 entry (first put wins, subsequent ignored) - assert.Equal(t, 1, store.Size(), "only first status update stored (single entry)") - - t.Log("") - t.Log("=== Phase 3: Watch informer fires (simulating actual cluster changes) ===") - - // Simulate watch informer firing for each status change - var retrievedUsernames []string - for i := range 5 { - // Watch sees the change and tries to enrich - entry, found := store.Get(correlationKeys[0]) - - if found { - retrievedUsernames = append(retrievedUsernames, entry.Username) - t.Logf(" Watch event %d: enriched with username=%s", i+1, entry.Username) - } else { - t.Logf(" Watch event %d: NO correlation found (miss)", i+1) - retrievedUsernames = append(retrievedUsernames, "") - } - } - - t.Log("") - t.Log("=== Results ===") - t.Logf(" Retrieved usernames: %v", retrievedUsernames) - t.Log("") - - // With single entry, all watch events get the first user (can be accessed multiple times) - expectedUsernames := []string{ - "user1@example.com", // First user (subsequent puts ignored) - "user1@example.com", // Same entry accessible multiple times - "user1@example.com", // Same entry accessible multiple times - "user1@example.com", // Same entry accessible multiple times - "user1@example.com", // Same entry accessible multiple times - } - - assert.Equal(t, expectedUsernames, retrievedUsernames, - "All watch events get enriched with first user") - - // Store should still have the entry - assert.Equal(t, 1, store.Size(), "correlation entry remains accessible") - - t.Log("") - t.Log("=== Conclusion ===") - t.Log(" • Webhook ignores correlation entry for same key if already claimed") - t.Log(" • Only first user is stored (single entry)") - t.Log(" • Watch fires 5 times (one per actual status change)") - t.Log(" • All watch events get enriched with first user") - t.Log(" • Result: 5 events enriched with same user") - t.Log("") - t.Log(" SOLUTION: Status-only changes naturally deduplicated!") - t.Log(" • Same sanitized content (spec unchanged)") - t.Log(" • Single entry prevents multiple commits") - t.Log(" • Only first status change is recorded in Git") - t.Log("") - t.Log(" ✓ Simplified store solves the deduplication problem") -} - -// TestStatusOnlyChanges_WatchDeduplication tests potential deduplication logic. -func TestStatusOnlyChanges_WatchDeduplication(t *testing.T) { - // This test demonstrates what SHOULD happen with deduplication - - type watchEvent struct { - sanitizedHash string - username string - } - - // Simulate watch events with content tracking - var events []watchEvent - lastSeenHash := make(map[string]string) // resource ID → last content hash - - baseObj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - "spec": map[string]interface{}{ - "containers": []interface{}{ - map[string]interface{}{ - "name": "app", - "image": "nginx:1.0", - }, - }, - }, - }, - } - - resourceKey := "default/test-pod" - - t.Log("=== Simulating 5 status changes ===") - statuses := []string{"Pending", "Running", "Running", "Terminating", "Terminated"} - usernames := []string{"user1", "user2", "user3", "user4", "user5"} - - for i, status := range statuses { - obj := baseObj.DeepCopy() - unstructured.SetNestedField(obj.Object, status, "status", "phase") - - sanitized := sanitize.Sanitize(obj) - yaml, _ := sanitize.MarshalToOrderedYAML(sanitized) - hash := string(yaml) // In reality, would use xxhash - - t.Logf(" Update %d: status=%s, user=%s", i+1, status, usernames[i]) - - // Deduplication logic - if lastHash, seen := lastSeenHash[resourceKey]; seen && lastHash == hash { - t.Logf(" → SKIPPED: identical to previous sanitized content") - continue - } - - // New content - enqueue it - events = append(events, watchEvent{ - sanitizedHash: hash, - username: usernames[i], - }) - lastSeenHash[resourceKey] = hash - t.Logf(" → ENQUEUED: new sanitized content") - } - - t.Log("") - t.Log("=== Results with Deduplication ===") - t.Logf(" Total watch callbacks: 5") - t.Logf(" Events actually enqueued: %d", len(events)) - t.Logf(" Duplicates skipped: %d", 5-len(events)) - - // With status-only changes, ALL should be identical after sanitization - assert.Len(t, events, 1, "only first event should be enqueued") - assert.Equal(t, "user1", events[0].username, "first user's change is recorded") - - t.Log("") - t.Log(" ✓ Deduplication prevents redundant commits for status-only changes") -} - -// TestSpecChanges_NoDeduplication validates that spec changes are NOT deduplicated. -func TestSpecChanges_NoDeduplication(t *testing.T) { - lastSeenHash := make(map[string]string) - resourceKey := "default/test-pod" - - baseObj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - }, - } - - t.Log("=== Simulating 3 spec changes ===") - specs := []map[string]interface{}{ - {"containers": []interface{}{map[string]interface{}{"name": "app", "image": "nginx:1.0"}}}, - {"containers": []interface{}{map[string]interface{}{"name": "app", "image": "nginx:1.1"}}}, - {"containers": []interface{}{map[string]interface{}{"name": "app", "image": "nginx:1.2"}}}, - } - - enqueueCount := 0 - for i, spec := range specs { - obj := baseObj.DeepCopy() - obj.Object["spec"] = spec - unstructured.SetNestedField(obj.Object, "Running", "status", "phase") - - sanitized := sanitize.Sanitize(obj) - yaml, _ := sanitize.MarshalToOrderedYAML(sanitized) - hash := string(yaml) - - t.Logf(" Update %d: image=%s", i+1, spec["containers"].([]interface{})[0].(map[string]interface{})["image"]) - - // Deduplication check - if lastHash, seen := lastSeenHash[resourceKey]; seen && lastHash == hash { - t.Logf(" → SKIPPED: duplicate content") - continue - } - - enqueueCount++ - lastSeenHash[resourceKey] = hash - t.Logf(" → ENQUEUED: new content detected") - } - - t.Log("") - t.Logf(" Events enqueued: %d/3", enqueueCount) - assert.Equal(t, 3, enqueueCount, "all spec changes should be enqueued") - - t.Log(" ✓ Spec changes are NOT deduplicated (correct behavior)") -} diff --git a/internal/correlation/store.go b/internal/correlation/store.go deleted file mode 100644 index 7a3758431..000000000 --- a/internal/correlation/store.go +++ /dev/null @@ -1,298 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package correlation provides an in-memory key-value store for correlating -// webhook admission events with watch events via sanitized object identity. -package correlation - -import ( - "container/list" - "context" - "encoding/binary" - "fmt" - "log/slog" - "strings" - "sync" - "time" - - "github.com/cespare/xxhash/v2" - - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -// Entry represents a stored correlation entry with username and timestamp. -type Entry struct { - Username string - Timestamp time.Time -} - -// Store is an in-memory key-value store with TTL and LRU eviction. -// It correlates webhook admission events with watch events using sanitized object identity. -// Each key stores a single entry. -type Store struct { - mu sync.Mutex - - // entries maps correlation keys to entries - entries map[string]*Entry - - // lruList maintains insertion order for LRU eviction - lruList *list.List - - // lruMap maps keys to list elements for O(1) removal - lruMap map[string]*list.Element - - // ttl is the duration after which entries expire - ttl time.Duration - - // maxEntries is the maximum number of entries before LRU eviction - maxEntries int - - // logger for warnings - logger *slog.Logger - - // metrics callbacks (optional) - onEviction func() // called when an entry is evicted (LRU or TTL) -} - -// NewStore creates a new correlation store with the specified TTL and maximum entries. -// ttl: duration after which entries expire (typically 5 minutes). -// maxEntries: maximum number of entries before LRU eviction (typically 10,000). -func NewStore(ttl time.Duration, maxEntries int) *Store { - return &Store{ - entries: make(map[string]*Entry), - lruList: list.New(), - lruMap: make(map[string]*list.Element), - ttl: ttl, - maxEntries: maxEntries, - logger: slog.Default(), - } -} - -// SetEvictionCallback sets a callback function that is invoked when an entry is evicted. -// This is typically used to increment eviction metrics. -func (s *Store) SetEvictionCallback(callback func()) { - s.mu.Lock() - defer s.mu.Unlock() - s.onEviction = callback -} - -// GenerateKey creates a correlation key from a resource identifier, operation, and sanitized content hash. -// Key format: {group}/{version}/{resource}/{namespace}/{name}:{operation}:{specHash} -// where specHash is the 16-character hex representation of xxhash64(sanitizedYAML). -func GenerateKey(id types.ResourceIdentifier, operation string, sanitizedYAML []byte) string { - const ( - hashByteSize = 8 // xxhash64 produces 8 bytes - hexCharSize = 16 // 8 bytes = 16 hex characters - ) - - // Compute xxhash64 of sanitized YAML (much faster than SHA256) - h := xxhash.Sum64(sanitizedYAML) - // Convert to 16-char hex string - hashBytes := make([]byte, hashByteSize) - binary.BigEndian.PutUint64(hashBytes, h) - hashStr := make([]byte, hexCharSize) - const hexTable = "0123456789abcdef" - for i := range hashByteSize { - hashStr[i*2] = hexTable[hashBytes[i]>>4] - hashStr[i*2+1] = hexTable[hashBytes[i]&0x0f] - } - - // Build key based on resource structure - hashString := string(hashStr) - if id.Namespace == "" { - // Cluster-scoped resource - if id.Group == "" { - // Core resource without group - return fmt.Sprintf("%s/%s/%s:%s:%s", - id.Version, id.Resource, id.Name, operation, hashString) - } - return fmt.Sprintf("%s/%s/%s/%s:%s:%s", - id.Group, id.Version, id.Resource, id.Name, operation, hashString) - } - // Namespaced resource - if id.Group == "" { - // Core resource without group - return fmt.Sprintf("%s/%s/%s/%s:%s:%s", - id.Version, id.Resource, id.Namespace, id.Name, operation, hashString) - } - return fmt.Sprintf("%s/%s/%s/%s/%s:%s:%s", - id.Group, id.Version, id.Resource, id.Namespace, id.Name, operation, hashString) -} - -// Put stores a correlation entry for the given key. -// If the key already exists and is not expired: -// - If the username matches, updates the timestamp. -// - If the username differs and existing is not empty or system:node, logs an info message and ignores the update. -// -// If the key exists but is expired, overwrites the entry. -// If the key does not exist, creates a new entry. -func (s *Store) Put(key string, username string) { - s.mu.Lock() - defer s.mu.Unlock() - - now := time.Now() - entry := &Entry{ - Username: username, - Timestamp: now, - } - - // Check if key already exists - if existing, exists := s.entries[key]; exists { - if time.Since(existing.Timestamp) > s.ttl { - // Entry is expired, overwrite - s.entries[key] = entry - s.lruList.MoveToFront(s.lruMap[key]) - return - } - // Entry is not expired - if existing.Username != username && existing.Username != "" && - !strings.HasPrefix(existing.Username, "system:node") { - // Different user and existing is not empty or system:node, ignore and log info - s.logger.InfoContext( - context.Background(), - "ignoring user attribution since it was already claimed", - "key", - key, - "existing_user", - existing.Username, - "attempted_user", - username, - ) - // Still move to front for LRU - s.lruList.MoveToFront(s.lruMap[key]) - return - } - // Same user or existing is empty or system:node, update/overwrite - s.entries[key] = entry - s.lruList.MoveToFront(s.lruMap[key]) - return - } - - // Check capacity and evict if necessary - if len(s.entries) >= s.maxEntries { - s.evictLRU() - } - - // Create new entry for this key - s.entries[key] = entry - - // Add to LRU list - elem := s.lruList.PushFront(key) - s.lruMap[key] = elem -} - -// Get retrieves the correlation entry for the given key. -// Returns the entry and true if found and not expired, or nil and false otherwise. -// Expired entries are automatically removed during lookup. -func (s *Store) Get(key string) (*Entry, bool) { - s.mu.Lock() - defer s.mu.Unlock() - - entry, exists := s.entries[key] - if !exists { - return nil, false - } - - // Check TTL - if time.Since(entry.Timestamp) > s.ttl { - // Entry expired, remove it - s.removeEntry(key) - if s.onEviction != nil { - s.onEviction() - } - return nil, false - } - - // Return entry without removing it - return entry, true -} - -// EvictExpired removes all expired entries from the store. -// This is typically called periodically to prevent unbounded growth. -func (s *Store) EvictExpired() int { - s.mu.Lock() - defer s.mu.Unlock() - - now := time.Now() - evicted := 0 - - // Iterate through all entries and remove expired ones - for key, entry := range s.entries { - if now.Sub(entry.Timestamp) > s.ttl { - s.removeEntry(key) - if s.onEviction != nil { - s.onEviction() - } - evicted++ - } - } - - return evicted -} - -// Size returns the current number of entries in the store. -func (s *Store) Size() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.entries) -} - -// Clear removes all entries from the store. -func (s *Store) Clear() { - s.mu.Lock() - defer s.mu.Unlock() - - s.entries = make(map[string]*Entry) - s.lruList = list.New() - s.lruMap = make(map[string]*list.Element) -} - -// evictLRU removes the least recently used entry. -// Must be called with lock held. -func (s *Store) evictLRU() { - if s.lruList.Len() == 0 { - return - } - - // Get the oldest element (back of list) - oldest := s.lruList.Back() - if oldest == nil { - return - } - - key, ok := oldest.Value.(string) - if !ok { - return - } - s.removeEntry(key) - - if s.onEviction != nil { - s.onEviction() - } -} - -// removeEntry removes an entry from the store and LRU tracking. -// Must be called with lock held. -func (s *Store) removeEntry(key string) { - delete(s.entries, key) - - if elem, exists := s.lruMap[key]; exists { - s.lruList.Remove(elem) - delete(s.lruMap, key) - } -} diff --git a/internal/correlation/store_test.go b/internal/correlation/store_test.go deleted file mode 100644 index 3bc27551a..000000000 --- a/internal/correlation/store_test.go +++ /dev/null @@ -1,549 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package correlation - -import ( - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -// TestGenerateKey_Determinism verifies that the same inputs always produce the same key. -func TestGenerateKey_Determinism(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - sanitizedYAML := []byte("apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app") - - key1 := GenerateKey(id, operation, sanitizedYAML) - key2 := GenerateKey(id, operation, sanitizedYAML) - - assert.Equal(t, key1, key2, "Same inputs should produce identical keys") -} - -// TestGenerateKey_DifferentOperations verifies different operations produce different keys. -func TestGenerateKey_DifferentOperations(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - sanitizedYAML := []byte("apiVersion: apps/v1\nkind: Deployment") - - keyCreate := GenerateKey(id, "CREATE", sanitizedYAML) - keyUpdate := GenerateKey(id, "UPDATE", sanitizedYAML) - keyDelete := GenerateKey(id, "DELETE", sanitizedYAML) - - assert.NotEqual(t, keyCreate, keyUpdate, "CREATE and UPDATE should produce different keys") - assert.NotEqual(t, keyCreate, keyDelete, "CREATE and DELETE should produce different keys") - assert.NotEqual(t, keyUpdate, keyDelete, "UPDATE and DELETE should produce different keys") -} - -// TestGenerateKey_DifferentContent verifies different sanitized content produces different keys. -func TestGenerateKey_DifferentContent(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - - yaml1 := []byte("apiVersion: apps/v1\nkind: Deployment\nspec:\n replicas: 1") - yaml2 := []byte("apiVersion: apps/v1\nkind: Deployment\nspec:\n replicas: 2") - - key1 := GenerateKey(id, operation, yaml1) - key2 := GenerateKey(id, operation, yaml2) - - assert.NotEqual(t, key1, key2, "Different sanitized content should produce different keys") -} - -// TestGenerateKey_NamespacedVsClusterScoped verifies correct key format for both types. -func TestGenerateKey_NamespacedVsClusterScoped(t *testing.T) { - tests := []struct { - name string - id types.ResourceIdentifier - expected string // partial match for key structure - }{ - { - name: "namespaced with group", - id: types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app"), - expected: "apps/v1/deployments/default/my-app:", - }, - { - name: "cluster-scoped with group", - id: types.NewResourceIdentifier("rbac.authorization.k8s.io", "v1", "clusterroles", "", "admin"), - expected: "rbac.authorization.k8s.io/v1/clusterroles/admin:", - }, - { - name: "namespaced core resource", - id: types.NewResourceIdentifier("", "v1", "configmaps", "default", "my-config"), - expected: "v1/configmaps/default/my-config:", - }, - { - name: "cluster-scoped core resource", - id: types.NewResourceIdentifier("", "v1", "namespaces", "", "kube-system"), - expected: "v1/namespaces/kube-system:", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - key := GenerateKey(tt.id, "UPDATE", []byte("test")) - assert.Contains(t, key, tt.expected, "Key should contain expected structure") - assert.Contains(t, key, "UPDATE:", "Key should contain operation") - }) - } -} - -// TestStore_BasicPutAndGet verifies basic store operations. -func TestStore_BasicPutAndGet(t *testing.T) { - store := NewStore(60*time.Second, 100) - key := "test/key" - username := "alice" - - // Put entry - store.Put(key, username) - assert.Equal(t, 1, store.Size(), "Store should have one entry") - - // Get entry - entry, found := store.Get(key) - require.True(t, found, "Entry should be found") - assert.Equal(t, username, entry.Username, "Username should match") - assert.Equal(t, 1, store.Size(), "Store should still have the entry after Get") - - // Get again should succeed - entry2, found2 := store.Get(key) - require.True(t, found2, "Entry should still be found") - assert.Equal(t, username, entry2.Username, "Username should still match") -} - -// TestStore_UpdateExisting verifies that existing entries are not overwritten with different users. -func TestStore_UpdateExisting(t *testing.T) { - store := NewStore(60*time.Second, 100) - key := "test/key" - - store.Put(key, "alice") - store.Put(key, "bob") // Does not overwrite, logs info - - // Get returns the original entry - entry, found := store.Get(key) - require.True(t, found, "Entry should be found") - assert.Equal(t, "alice", entry.Username, "Should return the original username") - assert.Equal(t, 1, store.Size(), "Store should still have the entry after Get") -} - -// TestStore_OverwriteEmptyUsername verifies that empty username attributions can be overwritten. -func TestStore_OverwriteEmptyUsername(t *testing.T) { - store := NewStore(60*time.Second, 100) - key := "test/key" - - // Put empty username first - store.Put(key, "") - entry1, found1 := store.Get(key) - require.True(t, found1, "Entry should be found") - assert.Empty(t, entry1.Username, "Should have empty username") - - // Put real username - should overwrite - store.Put(key, "alice") - entry2, found2 := store.Get(key) - require.True(t, found2, "Entry should still be found") - assert.Equal(t, "alice", entry2.Username, "Should be overwritten with real username") - - // Now that it's claimed by alice, bob should not overwrite - store.Put(key, "bob") - entry3, found3 := store.Get(key) - require.True(t, found3, "Entry should still be found") - assert.Equal(t, "alice", entry3.Username, "Should not be overwritten by different user") -} - -// TestStore_OverwriteSystemNodeUsername verifies that system:node username attributions can be overwritten. -func TestStore_OverwriteSystemNodeUsername(t *testing.T) { - store := NewStore(60*time.Second, 100) - key := "test/key" - - // Put system:node username first - store.Put(key, "system:node:worker-1") - entry1, found1 := store.Get(key) - require.True(t, found1, "Entry should be found") - assert.Equal(t, "system:node:worker-1", entry1.Username, "Should have system:node username") - - // Put real username - should overwrite - store.Put(key, "alice") - entry2, found2 := store.Get(key) - require.True(t, found2, "Entry should still be found") - assert.Equal(t, "alice", entry2.Username, "Should be overwritten with real username") - - // Now that it's claimed by alice, bob should not overwrite - store.Put(key, "bob") - entry3, found3 := store.Get(key) - require.True(t, found3, "Entry should still be found") - assert.Equal(t, "alice", entry3.Username, "Should not be overwritten by different user") -} - -// TestStore_TTLExpiry verifies entries expire after TTL. -func TestStore_TTLExpiry(t *testing.T) { - ttl := 100 * time.Millisecond - store := NewStore(ttl, 100) - key := "test/key" - - store.Put(key, "alice") - - // Immediate get should succeed - entry, found := store.Get(key) - require.True(t, found, "Entry should be found immediately") - assert.Equal(t, "alice", entry.Username) - - // Re-add and wait for expiry - store.Put(key, "alice") - time.Sleep(ttl + 50*time.Millisecond) // Wait longer than TTL - - // Should be expired - _, found = store.Get(key) - assert.False(t, found, "Entry should be expired after TTL") - assert.Equal(t, 0, store.Size(), "Expired entry should be removed") -} - -// TestStore_LRUEviction verifies LRU eviction when capacity is reached. -func TestStore_LRUEviction(t *testing.T) { - maxEntries := 3 - store := NewStore(60*time.Second, maxEntries) - - evictionCount := 0 - store.SetEvictionCallback(func() { - evictionCount++ - }) - - // Fill to capacity - store.Put("key1", "user1") - store.Put("key2", "user2") - store.Put("key3", "user3") - assert.Equal(t, 3, store.Size(), "Store should be at capacity") - assert.Equal(t, 0, evictionCount, "No evictions yet") - - // Add one more - should evict oldest (key1) - store.Put("key4", "user4") - assert.Equal(t, 3, store.Size(), "Store should remain at capacity") - assert.Equal(t, 1, evictionCount, "One eviction should have occurred") - - // key1 should be evicted - _, found := store.Get("key1") - assert.False(t, found, "Oldest entry should have been evicted") - - // Others should still exist - _, found = store.Get("key2") - assert.True(t, found, "Second entry should still exist") - _, found = store.Get("key3") - assert.True(t, found, "Third entry should still exist") - _, found = store.Get("key4") - assert.True(t, found, "New entry should exist") -} - -// TestStore_LRUUpdateRefreshes verifies that updating a key refreshes its LRU position. -func TestStore_LRUUpdateRefreshes(t *testing.T) { - maxEntries := 2 - store := NewStore(60*time.Second, maxEntries) - - store.Put("key1", "user1") - store.Put("key2", "user2") - - // Update key1 with same user - should move it to front - store.Put("key1", "user1") - - // Add key3 - should evict key2 (oldest), not key1 - store.Put("key3", "user3") - - _, found := store.Get("key1") - assert.True(t, found, "Updated entry should still exist") - - _, found = store.Get("key2") - assert.False(t, found, "Oldest entry should have been evicted") - - _, found = store.Get("key3") - assert.True(t, found, "New entry should exist") -} - -// TestStore_EvictExpired verifies manual expiry of old entries. -func TestStore_EvictExpired(t *testing.T) { - ttl := 50 * time.Millisecond - store := NewStore(ttl, 100) - - evictionCount := 0 - store.SetEvictionCallback(func() { - evictionCount++ - }) - - // Add entries - store.Put("key1", "user1") - time.Sleep(30 * time.Millisecond) - store.Put("key2", "user2") - time.Sleep(30 * time.Millisecond) // key1 should be expired, key2 not yet - - // Manually evict expired - evicted := store.EvictExpired() - assert.Equal(t, 1, evicted, "One entry should be evicted") - assert.Equal(t, 1, evictionCount, "Eviction callback should be called") - assert.Equal(t, 1, store.Size(), "One entry should remain") - - _, found := store.Get("key1") - assert.False(t, found, "Expired entry should be removed") - - _, found = store.Get("key2") - assert.True(t, found, "Non-expired entry should remain") -} - -// TestStore_Clear verifies clearing all entries. -func TestStore_Clear(t *testing.T) { - store := NewStore(60*time.Second, 100) - - store.Put("key1", "user1") - store.Put("key2", "user2") - store.Put("key3", "user3") - assert.Equal(t, 3, store.Size()) - - store.Clear() - assert.Equal(t, 0, store.Size(), "Store should be empty after clear") - - _, found := store.Get("key1") - assert.False(t, found, "No entries should exist after clear") -} - -// TestStore_ConcurrentAccess verifies thread-safety under concurrent load. -func TestStore_ConcurrentAccess(_ *testing.T) { - store := NewStore(5*time.Second, 1000) - concurrency := 10 - operationsPerGoroutine := 100 - - var wg sync.WaitGroup - wg.Add(concurrency) - - for i := range concurrency { - go func(_ int) { - defer wg.Done() - for range operationsPerGoroutine { - key := GenerateKey( - types.NewResourceIdentifier("apps", "v1", "deployments", "default", "app"), - "UPDATE", - []byte("test content"), - ) - store.Put(key, "user") - store.Get(key) // Can be called multiple times - } - }(i) - } - - wg.Wait() - // If we get here without deadlock or panic, thread-safety is working -} - -// TestStore_EvictionCallback verifies the callback is invoked for all eviction types. -func TestStore_EvictionCallback(t *testing.T) { - ttl := 50 * time.Millisecond - store := NewStore(ttl, 2) - - evictionCount := 0 - store.SetEvictionCallback(func() { - evictionCount++ - }) - - // LRU eviction - store.Put("key1", "user1") - store.Put("key2", "user2") - store.Put("key3", "user3") // Evicts key1 - assert.Equal(t, 1, evictionCount, "LRU eviction should trigger callback") - - // TTL eviction via Get (expired entry accessed) - time.Sleep(ttl + 20*time.Millisecond) - _, found := store.Get("key2") // Should be expired and trigger eviction - assert.False(t, found) - assert.Equal(t, 2, evictionCount, "TTL eviction should trigger callback") - - // TTL eviction via EvictExpired - both key3 and key4 will be expired - store.Put("key4", "user4") - time.Sleep(ttl + 20*time.Millisecond) - evicted := store.EvictExpired() - assert.Equal(t, 2, evicted, "Should evict 2 expired entries (key3 and key4)") - assert.Equal(t, 4, evictionCount, "Manual eviction should trigger callback for each evicted entry") -} - -// TestStore_SanitizeEquivalence verifies that sanitized objects produce identical keys. -func TestStore_SanitizeEquivalence(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - - // Simulate webhook sanitized YAML - webhookYAML := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - replicas: 3 -`) - - // Simulate watch sanitized YAML (should be identical after sanitization) - watchYAML := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - replicas: 3 -`) - - webhookKey := GenerateKey(id, operation, webhookYAML) - watchKey := GenerateKey(id, operation, watchYAML) - - assert.Equal(t, webhookKey, watchKey, "Sanitized webhook and watch objects should produce the same key") -} - -// TestStore_NoResourceVersionDependence verifies keys are independent of resourceVersion. -func TestStore_NoResourceVersionDependence(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - - // YAML with different resourceVersions (which should be sanitized away) - yaml1 := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - resourceVersion: "12345" -spec: - replicas: 3 -`) - - yaml2 := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - resourceVersion: "67890" -spec: - replicas: 3 -`) - - // If sanitization is working correctly upstream, these should be identical after sanitization - // For this test, we're verifying the key generation is deterministic - key1 := GenerateKey(id, operation, yaml1) - key2 := GenerateKey(id, operation, yaml2) - - // Keys will differ because we're passing different YAML - // But the point is that the correlation system expects sanitized YAML as input - assert.NotEqual(t, key1, key2, "Different YAML content produces different keys") - - // The real test is that when the same sanitized YAML is provided, keys match - sanitizedYAML := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app -spec: - replicas: 3 -`) - - key3 := GenerateKey(id, operation, sanitizedYAML) - key4 := GenerateKey(id, operation, sanitizedYAML) - assert.Equal(t, key3, key4, "Same sanitized YAML produces identical keys") -} - -// TestGenerateKey_SanitizationRobustness verifies that different YAML formatting -// produces the same key after sanitization (key order, whitespace, etc.). -// This test demonstrates that the sanitization pipeline normalizes YAML before hashing. -func TestGenerateKey_SanitizationRobustness(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - - // Note: In production, raw YAML goes through sanitize.Sanitize() + MarshalToOrderedYAML() - // BEFORE being passed to GenerateKey. This test simulates that by passing - // already-sanitized YAML that should hash identically. - - // Canonical sanitized YAML (what sanitize.MarshalToOrderedYAML produces) - canonicalYAML := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - replicas: 3 - selector: - matchLabels: - app: my-app -`) - - // When the same logical object is sanitized, it should produce identical YAML - // regardless of input formatting, which then produces identical keys - key1 := GenerateKey(id, operation, canonicalYAML) - key2 := GenerateKey(id, operation, canonicalYAML) - - assert.Equal(t, key1, key2, "Identical sanitized YAML should produce identical keys") - - // Different content (different replica count) should produce different key - differentYAML := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app - namespace: default -spec: - replicas: 5 - selector: - matchLabels: - app: my-app -`) - - key3 := GenerateKey(id, operation, differentYAML) - assert.NotEqual(t, key1, key3, "Different content should produce different keys") - - // Verify the sanitization contract: same keys produce same hash - assert.Equal(t, key1, key2, "Deterministic hashing of same content") -} - -// TestGenerateKey_ContentSensitivity verifies that actual content changes -// produce different keys even with same structure. -func TestGenerateKey_ContentSensitivity(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "my-app") - operation := "UPDATE" - - yaml1 := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app -spec: - replicas: 3 -`) - - yaml2 := []byte(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-app -spec: - replicas: 5 -`) - - key1 := GenerateKey(id, operation, yaml1) - key2 := GenerateKey(id, operation, yaml2) - - assert.NotEqual(t, key1, key2, "Different replicas value should produce different keys") -} - -// TestGenerateKey_SpecificHashFormat verifies xxhash produces expected format. -func TestGenerateKey_SpecificHashFormat(t *testing.T) { - id := types.NewResourceIdentifier("apps", "v1", "deployments", "default", "test") - operation := "UPDATE" - yaml := []byte("test content") - - key := GenerateKey(id, operation, yaml) - - // Key should contain a 16-character hex hash - assert.Contains(t, key, "apps/v1/deployments/default/test:UPDATE:") - assert.Len(t, key, len("apps/v1/deployments/default/test:UPDATE:")+16, - "Key should end with 16-character hash") -} diff --git a/internal/git/git_operations_test.go b/internal/git/git_operations_test.go index ab67bd30b..e71f13a17 100644 --- a/internal/git/git_operations_test.go +++ b/internal/git/git_operations_test.go @@ -20,9 +20,11 @@ package git import ( "context" + "errors" "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -39,6 +41,26 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/types" ) +func skipIfPublicNetworkUnavailable(t *testing.T, err error) { + t.Helper() + if err == nil { + return + } + + var timeoutErr interface{ Timeout() bool } + if errors.As(err, &timeoutErr) && timeoutErr.Timeout() { + t.Skipf("skipping public connectivity test because network timed out: %v", err) + } + + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "no such host") || + strings.Contains(msg, "i/o timeout") || + strings.Contains(msg, "temporary failure in name resolution") || + strings.Contains(msg, "context deadline exceeded") { + t.Skipf("skipping public connectivity test because network is unavailable: %v", err) + } +} + // TestMain initializes the controller-runtime logger before running tests. // This prevents "log.SetLogger(...) was never called" warnings when code uses log.FromContext(). func TestMain(m *testing.M) { @@ -126,6 +148,7 @@ func TestCheckRepo_PublicConnectivity(t *testing.T) { ctx := context.Background() remoteURL := "https://github.com/octocat/Hello-World.git" repoInfo, err := CheckRepo(ctx, remoteURL, nil) + skipIfPublicNetworkUnavailable(t, err) tempDir := t.TempDir() require.NoError(t, err) @@ -170,6 +193,7 @@ func TestCheckRepo_PublicConnectivityEmpty(t *testing.T) { remoteURL := "https://github.com/ConfigButler/empty.git" repoInfo, err := CheckRepo(ctx, remoteURL, nil) + skipIfPublicNetworkUnavailable(t, err) require.NoError(t, err) assert.Empty(t, repoInfo.DefaultBranch) diff --git a/internal/queue/redis_audit_consumer.go b/internal/queue/redis_audit_consumer.go new file mode 100644 index 000000000..fda080f9b --- /dev/null +++ b/internal/queue/redis_audit_consumer.go @@ -0,0 +1,570 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/redis/go-redis/v9" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" + "github.com/ConfigButler/gitops-reverser/internal/sanitize" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +const ( + // defaultConsumerGroup is the consumer group name used for the audit stream. + defaultConsumerGroup = "gitopsreverser-consumer" + + // consumerReadCount is the maximum number of messages fetched per XREADGROUP call. + consumerReadCount = 50 + + // consumerBlockDuration is how long XREADGROUP blocks waiting for new messages. + consumerBlockDuration = 2 * time.Second + + // autoClaimMinIdle is the minimum idle time before a pending entry is reclaimed. + autoClaimMinIdle = 60 * time.Second + + // autoClaimInterval controls how often XAUTOCLAIM runs. + autoClaimInterval = 30 * time.Second +) + +// AuditEventRouter is the subset of watch.EventRouter used by the consumer. +// watch.EventRouter satisfies this interface without modification. +type AuditEventRouter interface { + RouteToGitTargetEventStream(event git.Event, gitDest itypes.ResourceReference) error +} + +// AuditConsumerConfig configures the Redis stream consumer. +type AuditConsumerConfig struct { + Addr string + Username string + AuthValue string + DB int + Stream string + TLSEnabled bool + + // Group is the Redis consumer group name. Defaults to defaultConsumerGroup. + Group string + + // ConsumerID uniquely identifies this consumer replica (e.g. pod name). + ConsumerID string +} + +// AuditConsumer reads audit events from a Redis stream and routes them into +// the existing git write pipeline via EventRouter. +type AuditConsumer struct { + client *redis.Client + stream string + group string + consumerID string + ruleStore *rulestore.RuleStore + eventRouter AuditEventRouter + log logr.Logger +} + +// NewAuditConsumer creates a new AuditConsumer. It does not start consuming; +// call Start to begin the consume loop. +func NewAuditConsumer( + cfg AuditConsumerConfig, + ruleStore *rulestore.RuleStore, + eventRouter AuditEventRouter, + log logr.Logger, +) (*AuditConsumer, error) { + if strings.TrimSpace(cfg.Addr) == "" { + return nil, errors.New("redis address is required") + } + + stream := strings.TrimSpace(cfg.Stream) + if stream == "" { + stream = DefaultRedisAuditStream + } + + group := strings.TrimSpace(cfg.Group) + if group == "" { + group = defaultConsumerGroup + } + + consumerID := strings.TrimSpace(cfg.ConsumerID) + if consumerID == "" { + consumerID = "gitopsreverser-consumer-0" + } + + options := &redis.Options{ + Addr: cfg.Addr, + Username: cfg.Username, + Password: cfg.AuthValue, + DB: cfg.DB, + } + if cfg.TLSEnabled { + options.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + + return &AuditConsumer{ + client: redis.NewClient(options), + stream: stream, + group: group, + consumerID: consumerID, + ruleStore: ruleStore, + eventRouter: eventRouter, + log: log.WithName("audit-consumer"), + }, nil +} + +// NeedLeaderElection returns true so only the elected leader runs the consumer. +func (c *AuditConsumer) NeedLeaderElection() bool { + return true +} + +// Start implements manager.Runnable. It bootstraps the consumer group and +// enters the read loop until ctx is cancelled. +func (c *AuditConsumer) Start(ctx context.Context) error { + c.log.Info("Starting audit stream consumer", + "stream", c.stream, + "group", c.group, + "consumer", c.consumerID) + + if err := c.ensureConsumerGroup(ctx); err != nil { + return fmt.Errorf("failed to ensure consumer group: %w", err) + } + + autoClaimTicker := time.NewTicker(autoClaimInterval) + defer autoClaimTicker.Stop() + + for { + select { + case <-ctx.Done(): + c.log.Info("Audit stream consumer stopping") + return nil + case <-autoClaimTicker.C: + c.runAutoClaimCycle(ctx) + default: + if err := c.readAndProcessBatch(ctx); err != nil { + // Log but do not exit — transient Redis errors should not crash the process. + c.log.Error(err, "Error reading from audit stream, retrying") + select { + case <-ctx.Done(): + return nil + case <-time.After(time.Second): + } + } + } + } +} + +// ensureConsumerGroup creates the consumer group if it does not already exist. +// MKSTREAM also creates the stream itself when missing. +func (c *AuditConsumer) ensureConsumerGroup(ctx context.Context) error { + err := c.client.XGroupCreateMkStream(ctx, c.stream, c.group, "$").Err() + if err != nil && !isAlreadyExistsErr(err) { + return err + } + c.log.Info("Consumer group ready", "stream", c.stream, "group", c.group) + return nil +} + +// readAndProcessBatch fetches up to consumerReadCount messages from the stream +// and processes each one, ACKing only after successful routing. +func (c *AuditConsumer) readAndProcessBatch(ctx context.Context) error { + streams, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: c.group, + Consumer: c.consumerID, + Streams: []string{c.stream, ">"}, + Count: consumerReadCount, + Block: consumerBlockDuration, + }).Result() + + if errors.Is(err, redis.Nil) { + // No new messages within the block window; this is normal. + return nil + } + if err != nil { + if isNoGroupErr(err) { + if ensureErr := c.ensureConsumerGroup(ctx); ensureErr != nil { + return fmt.Errorf("XREADGROUP failed with NOGROUP and group recreation failed: %w", ensureErr) + } + return nil + } + return fmt.Errorf("XREADGROUP failed: %w", err) + } + + for _, stream := range streams { + for _, msg := range stream.Messages { + c.processMessage(ctx, msg) + } + } + return nil +} + +// runAutoClaimCycle reclaims pending entries that have been idle longer than +// autoClaimMinIdle, then processes them. +func (c *AuditConsumer) runAutoClaimCycle(ctx context.Context) { + start := "0-0" + for { + messages, nextStart, err := c.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: c.stream, + Group: c.group, + Consumer: c.consumerID, + MinIdle: autoClaimMinIdle, + Start: start, + Count: consumerReadCount, + }).Result() + + if err != nil { + if isNoGroupErr(err) { + if ensureErr := c.ensureConsumerGroup(ctx); ensureErr != nil { + c.log.Error(ensureErr, "XAUTOCLAIM failed with NOGROUP and group recreation failed") + return + } + return + } + c.log.Error(err, "XAUTOCLAIM failed") + return + } + + for _, msg := range messages { + c.processMessage(ctx, msg) + } + + // "0-0" is returned when there are no more pending entries to scan. + if nextStart == "0-0" || len(messages) == 0 { + return + } + start = nextStart + } +} + +// processMessage handles a single stream entry: parses the audit event, +// matches rules, builds git.Event(s), routes them, and ACKs. +func (c *AuditConsumer) processMessage(ctx context.Context, msg redis.XMessage) { + log := c.log.WithValues("msgID", msg.ID) + + auditEvent, err := parseAuditEvent(msg.Values) + if err != nil { + log.Error(err, "Failed to parse audit event; skipping and ACKing to avoid poison-pill") + c.ackMessage(ctx, msg.ID) + return + } + + // Only process ResponseComplete entries to avoid duplicates across stages. + if auditEvent.Stage != auditv1.StageResponseComplete { + c.ackMessage(ctx, msg.ID) + return + } + + // Only process mutating verbs. + op, ok := verbToOperation(auditEvent.Verb) + if !ok { + c.ackMessage(ctx, msg.ID) + return + } + + if auditEvent.ObjectRef == nil { + c.ackMessage(ctx, msg.ID) + return + } + + clusterID := stringField(msg.Values, "cluster_id") + if err := c.routeAuditEvent(ctx, log, auditEvent, op, clusterID); err != nil { + log.Error(err, "Failed to route audit event; ACKing to avoid poison-pill") + } + c.ackMessage(ctx, msg.ID) +} + +// routeAuditEvent performs rule matching, object extraction, and routing for one audit event. +func (c *AuditConsumer) routeAuditEvent( + _ context.Context, + log logr.Logger, + auditEvent auditv1.Event, + op configv1alpha1.OperationType, + clusterID string, +) error { + ref := auditEvent.ObjectRef + apiGroup, apiVersion := objectRefGroupVersion(ref) + resourcePlural := ref.Resource + namespace := ref.Namespace + name := ref.Name + isClusterScoped := namespace == "" + op = effectiveAuditOperation(auditEvent, op) + + wrRules := c.ruleStore.GetMatchingRules(nil, resourcePlural, op, apiGroup, apiVersion, isClusterScoped) + cwrRules := c.ruleStore.GetMatchingClusterRules(resourcePlural, op, apiGroup, apiVersion, isClusterScoped, nil) + + if len(wrRules) == 0 && len(cwrRules) == 0 { + return nil + } + + fullAPIVersion := apiVersion + if apiGroup != "" { + fullAPIVersion = apiGroup + "/" + apiVersion + } + + sanitized, err := extractObject(auditEvent, op, fullAPIVersion, ref.Resource, namespace, name) + if err != nil { + return fmt.Errorf("extracting object for %s/%s: %w", namespace, name, err) + } + + id := itypes.NewResourceIdentifier(apiGroup, apiVersion, resourcePlural, namespace, name) + userInfo := resolveUserInfo(auditEvent) + + routed := c.routeToMatchedRules(log, sanitized, id, op, userInfo, clusterID, wrRules, cwrRules) + + log.V(1).Info("Processed audit stream entry", + "resource", resourcePlural, "namespace", namespace, "name", name, + "operation", op, "user", userInfo.Username, "clusterID", clusterID, + "routed", routed) + return nil +} + +// routeToMatchedRules dispatches git.Events to all matched WatchRule and ClusterWatchRule targets. +// It returns the number of successfully routed events. +func (c *AuditConsumer) routeToMatchedRules( + log logr.Logger, + sanitized *unstructured.Unstructured, + id itypes.ResourceIdentifier, + op configv1alpha1.OperationType, + userInfo git.UserInfo, + clusterID string, + wrRules []rulestore.CompiledRule, + cwrRules []rulestore.CompiledClusterRule, +) int { + routed := 0 + for _, rule := range wrRules { + ev := buildGitEvent(sanitized, id, op, userInfo, rule.Path, rule.GitTargetRef, rule.GitTargetNamespace) + gitDest := itypes.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace) + if err := c.eventRouter.RouteToGitTargetEventStream(ev, gitDest); err != nil { + log.V(1).Info("Failed to route audit event via WatchRule", "error", err, + "gitTarget", gitDest.String(), "clusterID", clusterID) + continue + } + routed++ + } + for _, rule := range cwrRules { + ev := buildGitEvent(sanitized, id, op, userInfo, rule.Path, rule.GitTargetRef, rule.GitTargetNamespace) + gitDest := itypes.NewResourceReference(rule.GitTargetRef, rule.GitTargetNamespace) + if err := c.eventRouter.RouteToGitTargetEventStream(ev, gitDest); err != nil { + log.V(1).Info("Failed to route audit event via ClusterWatchRule", "error", err, + "gitTarget", gitDest.String(), "clusterID", clusterID) + continue + } + routed++ + } + return routed +} + +// buildGitEvent constructs a git.Event for a given rule match. +func buildGitEvent( + sanitized *unstructured.Unstructured, + id itypes.ResourceIdentifier, + op configv1alpha1.OperationType, + userInfo git.UserInfo, + path, gitTargetRef, gitTargetNamespace string, +) git.Event { + return git.Event{ + Object: sanitized.DeepCopy(), + Identifier: id, + Operation: string(op), + UserInfo: userInfo, + Path: path, + GitTargetName: gitTargetRef, + GitTargetNamespace: gitTargetNamespace, + } +} + +// resolveUserInfo extracts the effective username from an audit event, preferring +// the impersonated user when present. +func resolveUserInfo(event auditv1.Event) git.UserInfo { + username := event.User.Username + if event.ImpersonatedUser != nil && event.ImpersonatedUser.Username != "" { + username = event.ImpersonatedUser.Username + } + return git.UserInfo{Username: username} +} + +// ackMessage ACKs a single message, logging any error. +func (c *AuditConsumer) ackMessage(ctx context.Context, msgID string) { + if err := c.client.XAck(ctx, c.stream, c.group, msgID).Err(); err != nil { + c.log.Error(err, "Failed to ACK stream message", "msgID", msgID) + } +} + +// parseAuditEvent unmarshals the audit event from the stream entry's payload_json field. +func parseAuditEvent(values map[string]interface{}) (auditv1.Event, error) { + raw, ok := values["payload_json"] + if !ok { + return auditv1.Event{}, errors.New("stream entry missing payload_json field") + } + + var payload string + switch v := raw.(type) { + case string: + payload = v + case []byte: + payload = string(v) + default: + return auditv1.Event{}, fmt.Errorf("unexpected payload_json type %T", raw) + } + + var event auditv1.Event + if err := json.Unmarshal([]byte(payload), &event); err != nil { + return auditv1.Event{}, fmt.Errorf("failed to unmarshal audit event: %w", err) + } + return event, nil +} + +// extractObject obtains the Kubernetes object from the audit event and sanitizes it. +// For DELETE operations the RequestObject is used; otherwise the ResponseObject. +func extractObject( + event auditv1.Event, + op configv1alpha1.OperationType, + apiVersion, resource, namespace, name string, +) (*unstructured.Unstructured, error) { + var raw []byte + + if op == configv1alpha1.OperationDelete { + if event.RequestObject != nil && len(event.RequestObject.Raw) > 0 { + raw = event.RequestObject.Raw + } + } else { + if event.ResponseObject != nil && len(event.ResponseObject.Raw) > 0 { + raw = event.ResponseObject.Raw + } + } + + if len(raw) == 0 { + // Fall back to a minimal stub so downstream pipeline always has an object. + u := &unstructured.Unstructured{} + u.SetAPIVersion(apiVersion) + u.SetKind(resource) + u.SetNamespace(namespace) + u.SetName(name) + return u, nil + } + + obj := &unstructured.Unstructured{} + if err := obj.UnmarshalJSON(raw); err != nil { + return nil, fmt.Errorf("failed to unmarshal object JSON: %w", err) + } + + return sanitize.Sanitize(obj), nil +} + +func effectiveAuditOperation(event auditv1.Event, op configv1alpha1.OperationType) configv1alpha1.OperationType { + if op == configv1alpha1.OperationDelete { + return op + } + if auditEventObjectMarkedForDeletion(event) { + return configv1alpha1.OperationDelete + } + return op +} + +func auditEventObjectMarkedForDeletion(event auditv1.Event) bool { + return auditObjectMarkedForDeletion(event.ResponseObject) || auditObjectMarkedForDeletion(event.RequestObject) +} + +func auditObjectMarkedForDeletion(rawObj *runtime.Unknown) bool { + if rawObj == nil || len(rawObj.Raw) == 0 { + return false + } + + obj := &unstructured.Unstructured{} + if err := obj.UnmarshalJSON(rawObj.Raw); err != nil { + return false + } + + return !obj.GetDeletionTimestamp().IsZero() +} + +// verbToOperation maps a Kubernetes audit verb to a configv1alpha1.OperationType. +// Returns (op, true) for mutating verbs, ("", false) for read-only or unknown verbs. +func verbToOperation(verb string) (configv1alpha1.OperationType, bool) { + switch strings.ToLower(verb) { + case "create": + return configv1alpha1.OperationCreate, true + case "update", "patch": + return configv1alpha1.OperationUpdate, true + case "delete", "deletecollection": + return configv1alpha1.OperationDelete, true + default: + return "", false + } +} + +// splitAPIVersion splits a Kubernetes apiVersion string (e.g. "apps/v1" or "v1") +// into (group, version). Core resources have an empty group. +func splitAPIVersion(apiVersion string) (string, string) { + group, version, found := strings.Cut(apiVersion, "/") + if !found { + return "", apiVersion + } + return group, version +} + +func objectRefGroupVersion(ref *auditv1.ObjectReference) (string, string) { + if ref == nil { + return "", "" + } + + group, version := splitAPIVersion(ref.APIVersion) + if group != "" { + return group, version + } + + return ref.APIGroup, version +} + +// stringField safely reads a string value from a stream entry's Values map. +func stringField(values map[string]interface{}, key string) string { + v, ok := values[key] + if !ok { + return "" + } + switch s := v.(type) { + case string: + return s + case []byte: + return string(s) + default: + return fmt.Sprintf("%v", v) + } +} + +// isAlreadyExistsErr returns true when the Redis error indicates that the consumer +// group already exists (BUSYGROUP). +func isAlreadyExistsErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "BUSYGROUP") +} + +func isNoGroupErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "NOGROUP") +} diff --git a/internal/queue/redis_audit_consumer_test.go b/internal/queue/redis_audit_consumer_test.go new file mode 100644 index 000000000..820573fff --- /dev/null +++ b/internal/queue/redis_audit_consumer_test.go @@ -0,0 +1,1003 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/go-logr/logr" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" + + configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// fakeEventRouter records calls to RouteToGitTargetEventStream. +type fakeEventRouter struct { + calls []routeCall + errFor map[string]error // keyed by gitDest.Key() +} + +type routeCall struct { + Event git.Event + GitDest itypes.ResourceReference +} + +func (f *fakeEventRouter) RouteToGitTargetEventStream(event git.Event, gitDest itypes.ResourceReference) error { + f.calls = append(f.calls, routeCall{Event: event, GitDest: gitDest}) + if f.errFor != nil { + if err, ok := f.errFor[gitDest.Key()]; ok { + return err + } + } + return nil +} + +// --- splitAPIVersion --- + +func TestSplitAPIVersion(t *testing.T) { + tests := []struct { + name string + apiVersion string + wantGroup string + wantVer string + }{ + {"core v1", "v1", "", "v1"}, + {"apps/v1", "apps/v1", "apps", "v1"}, + {"networking", "networking.k8s.io/v1", "networking.k8s.io", "v1"}, + {"beta", "extensions/v1beta1", "extensions", "v1beta1"}, + {"empty", "", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g, v := splitAPIVersion(tt.apiVersion) + assert.Equal(t, tt.wantGroup, g) + assert.Equal(t, tt.wantVer, v) + }) + } +} + +// --- verbToOperation --- + +func TestVerbToOperation(t *testing.T) { + tests := []struct { + verb string + wantOp configv1alpha1.OperationType + wantFound bool + }{ + {"create", configv1alpha1.OperationCreate, true}, + {"CREATE", configv1alpha1.OperationCreate, true}, + {"update", configv1alpha1.OperationUpdate, true}, + {"patch", configv1alpha1.OperationUpdate, true}, + {"PATCH", configv1alpha1.OperationUpdate, true}, + {"delete", configv1alpha1.OperationDelete, true}, + {"deletecollection", configv1alpha1.OperationDelete, true}, + {"get", "", false}, + {"list", "", false}, + {"watch", "", false}, + {"", "", false}, + } + for _, tt := range tests { + t.Run(tt.verb, func(t *testing.T) { + op, ok := verbToOperation(tt.verb) + assert.Equal(t, tt.wantFound, ok) + assert.Equal(t, tt.wantOp, op) + }) + } +} + +// --- parseAuditEvent --- + +func TestParseAuditEvent_Success(t *testing.T) { + ev := auditv1.Event{ + Verb: "create", + Stage: auditv1.StageResponseComplete, + } + ev.User.Username = "alice" + payload, err := json.Marshal(ev) + require.NoError(t, err) + + got, err := parseAuditEvent(map[string]interface{}{"payload_json": string(payload)}) + require.NoError(t, err) + assert.Equal(t, "alice", got.User.Username) + assert.Equal(t, auditv1.StageResponseComplete, got.Stage) +} + +func TestParseAuditEvent_BytesPayload(t *testing.T) { + ev := auditv1.Event{Verb: "delete"} + payload, _ := json.Marshal(ev) + + got, err := parseAuditEvent(map[string]interface{}{"payload_json": payload}) + require.NoError(t, err) + assert.Equal(t, "delete", got.Verb) +} + +func TestParseAuditEvent_MissingField(t *testing.T) { + _, err := parseAuditEvent(map[string]interface{}{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing payload_json") +} + +func TestParseAuditEvent_InvalidJSON(t *testing.T) { + _, err := parseAuditEvent(map[string]interface{}{"payload_json": "not-json"}) + require.Error(t, err) +} + +func TestParseAuditEvent_UnexpectedType(t *testing.T) { + _, err := parseAuditEvent(map[string]interface{}{"payload_json": 42}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unexpected payload_json type") +} + +// --- resolveUserInfo --- + +func TestResolveUserInfo(t *testing.T) { + t.Run("no impersonation", func(t *testing.T) { + ev := auditv1.Event{} + ev.User.Username = "alice" + ui := resolveUserInfo(ev) + assert.Equal(t, "alice", ui.Username) + }) + + t.Run("with impersonation", func(t *testing.T) { + ev := auditv1.Event{ + ImpersonatedUser: &authv1.UserInfo{Username: "bob"}, + } + ev.User.Username = "alice" + ui := resolveUserInfo(ev) + assert.Equal(t, "bob", ui.Username) + }) + + t.Run("impersonated user empty username falls back", func(t *testing.T) { + ev := auditv1.Event{ + ImpersonatedUser: &authv1.UserInfo{Username: ""}, + } + ev.User.Username = "alice" + ui := resolveUserInfo(ev) + assert.Equal(t, "alice", ui.Username) + }) +} + +// --- extractObject --- + +func TestExtractObject_ResponseObject(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetNamespace("default") + obj.SetName("cm") + raw, _ := obj.MarshalJSON() + + ev := auditv1.Event{ + ResponseObject: &runtime.Unknown{Raw: raw}, + } + + got, err := extractObject(ev, configv1alpha1.OperationCreate, "v1", "ConfigMap", "default", "cm") + require.NoError(t, err) + assert.Equal(t, "cm", got.GetName()) +} + +func TestExtractObject_DeleteUsesRequestObject(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetNamespace("default") + obj.SetName("cm-deleted") + raw, _ := obj.MarshalJSON() + + ev := auditv1.Event{ + RequestObject: &runtime.Unknown{Raw: raw}, + ResponseObject: nil, + } + + got, err := extractObject(ev, configv1alpha1.OperationDelete, "v1", "ConfigMap", "default", "cm-deleted") + require.NoError(t, err) + assert.Equal(t, "cm-deleted", got.GetName()) +} + +func TestExtractObject_FallbackStubWhenNoRaw(t *testing.T) { + ev := auditv1.Event{} // no RequestObject / ResponseObject + + got, err := extractObject(ev, configv1alpha1.OperationCreate, "v1", "ConfigMap", "default", "cm") + require.NoError(t, err) + assert.Equal(t, "cm", got.GetName()) + assert.Equal(t, "default", got.GetNamespace()) +} + +func TestExtractObject_InvalidJSON(t *testing.T) { + ev := auditv1.Event{ + ResponseObject: &runtime.Unknown{Raw: []byte("not-json")}, + } + _, err := extractObject(ev, configv1alpha1.OperationUpdate, "v1", "ConfigMap", "default", "cm") + require.Error(t, err) +} + +func TestEffectiveAuditOperation_TerminatingUpdateBecomesDelete(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("apiextensions.k8s.io/v1") + obj.SetKind("CustomResourceDefinition") + obj.SetName("icecreamorders.shop.example.com") + now := metav1.Now() + obj.SetDeletionTimestamp(&now) + raw, err := obj.MarshalJSON() + require.NoError(t, err) + + ev := auditv1.Event{ + ResponseObject: &runtime.Unknown{Raw: raw}, + } + + got := effectiveAuditOperation(ev, configv1alpha1.OperationUpdate) + assert.Equal(t, configv1alpha1.OperationDelete, got) +} + +func TestEffectiveAuditOperation_TerminatingRequestObjectBecomesDelete(t *testing.T) { + requestObject := []byte(`{ + "kind": "CustomResourceDefinition", + "apiVersion": "apiextensions.k8s.io/v1", + "metadata": { + "name": "icecreamorders.shop.example.com", + "uid": "c812c2be-b7a1-4062-9d4b-1238c53ed81b", + "resourceVersion": "2775", + "generation": 1, + "creationTimestamp": "2026-04-08T19:04:29Z", + "deletionTimestamp": "2026-04-08T19:04:57Z", + "finalizers": [ + "customresourcecleanup.apiextensions.k8s.io" + ] + }, + "status": { + "conditions": [ + { + "type": "NamesAccepted", + "status": "True", + "lastTransitionTime": "2026-04-08T19:04:29Z", + "reason": "NoConflicts", + "message": "no conflicts found" + }, + { + "type": "Established", + "status": "True", + "lastTransitionTime": "2026-04-08T19:04:29Z", + "reason": "InitialNamesAccepted", + "message": "the initial names have been accepted" + }, + { + "type": "Terminating", + "status": "True", + "lastTransitionTime": "2026-04-08T19:04:57Z", + "reason": "InstanceDeletionInProgress", + "message": "CustomResource deletion is in progress" + } + ], + "acceptedNames": { + "plural": "icecreamorders", + "singular": "icecreamorder", + "shortNames": [ + "ico" + ], + "kind": "IceCreamOrder", + "listKind": "IceCreamOrderList" + }, + "storedVersions": [ + "v1" + ] + } +}`) + responseObject := []byte(`{ + "kind": "Status", + "apiVersion": "v1", + "metadata": {}, + "status": "Failure", + "message": "customresourcedefinitions.apiextensions.k8s.io \"icecreamorders.shop.example.com\" not found", + "reason": "NotFound", + "details": { + "name": "icecreamorders.shop.example.com", + "group": "apiextensions.k8s.io", + "kind": "customresourcedefinitions" + }, + "code": 404 +}`) + + ev := auditv1.Event{ + RequestObject: &runtime.Unknown{Raw: requestObject}, + ResponseObject: &runtime.Unknown{Raw: responseObject}, + } + + got := effectiveAuditOperation(ev, configv1alpha1.OperationUpdate) + assert.Equal(t, configv1alpha1.OperationDelete, got) +} + +func TestExtractObject_TerminatingRequestObjectUsedAfterDeleteCoercion(t *testing.T) { + requestObject := []byte(`{ + "kind": "CustomResourceDefinition", + "apiVersion": "apiextensions.k8s.io/v1", + "metadata": { + "name": "icecreamorders.shop.example.com", + "uid": "c812c2be-b7a1-4062-9d4b-1238c53ed81b", + "resourceVersion": "2775", + "generation": 1, + "creationTimestamp": "2026-04-08T19:04:29Z", + "deletionTimestamp": "2026-04-08T19:04:57Z", + "finalizers": [ + "customresourcecleanup.apiextensions.k8s.io" + ] + }, + "status": { + "conditions": [ + { + "type": "Terminating", + "status": "True", + "lastTransitionTime": "2026-04-08T19:04:57Z", + "reason": "InstanceDeletionInProgress", + "message": "CustomResource deletion is in progress" + } + ] + } +}`) + responseObject := []byte(`{ + "kind": "Status", + "apiVersion": "v1", + "metadata": {}, + "status": "Failure", + "message": "customresourcedefinitions.apiextensions.k8s.io \"icecreamorders.shop.example.com\" not found", + "reason": "NotFound", + "details": { + "name": "icecreamorders.shop.example.com", + "group": "apiextensions.k8s.io", + "kind": "customresourcedefinitions" + }, + "code": 404 +}`) + + ev := auditv1.Event{ + RequestObject: &runtime.Unknown{Raw: requestObject}, + ResponseObject: &runtime.Unknown{Raw: responseObject}, + } + + op := effectiveAuditOperation(ev, configv1alpha1.OperationUpdate) + require.Equal(t, configv1alpha1.OperationDelete, op) + + got, err := extractObject( + ev, + op, + "apiextensions.k8s.io/v1", + "customresourcedefinitions", + "", + "icecreamorders.shop.example.com", + ) + require.NoError(t, err) + assert.Equal(t, "icecreamorders.shop.example.com", got.GetName()) + assert.Equal(t, "apiextensions.k8s.io/v1", got.GetAPIVersion()) + assert.Equal(t, "CustomResourceDefinition", got.GetKind()) +} + +func TestEffectiveAuditOperation_NonTerminatingUpdateStaysUpdate(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetNamespace("default") + obj.SetName("cm") + raw, err := obj.MarshalJSON() + require.NoError(t, err) + + ev := auditv1.Event{ + ResponseObject: &runtime.Unknown{Raw: raw}, + } + + got := effectiveAuditOperation(ev, configv1alpha1.OperationUpdate) + assert.Equal(t, configv1alpha1.OperationUpdate, got) +} + +// --- isAlreadyExistsErr --- + +func TestIsAlreadyExistsErr(t *testing.T) { + assert.True(t, isAlreadyExistsErr(errors.New("BUSYGROUP Consumer Group name already exists"))) + assert.False(t, isAlreadyExistsErr(errors.New("some other error"))) + assert.False(t, isAlreadyExistsErr(nil)) +} + +// --- stringField --- + +func TestStringField(t *testing.T) { + values := map[string]interface{}{ + "str": "hello", + "bytes": []byte("world"), + "int": 42, + } + assert.Equal(t, "hello", stringField(values, "str")) + assert.Equal(t, "world", stringField(values, "bytes")) + assert.Equal(t, "42", stringField(values, "int")) + assert.Empty(t, stringField(values, "missing")) +} + +// --- NewAuditConsumer --- + +func TestNewAuditConsumer_RequiresAddress(t *testing.T) { + _, err := NewAuditConsumer( + AuditConsumerConfig{}, + rulestore.NewStore(), + &fakeEventRouter{}, + logr.Discard(), + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "redis address") +} + +func TestNewAuditConsumer_DefaultsApplied(t *testing.T) { + mr := miniredis.RunT(t) + c, err := NewAuditConsumer( + AuditConsumerConfig{Addr: mr.Addr()}, + rulestore.NewStore(), + &fakeEventRouter{}, + logr.Discard(), + ) + require.NoError(t, err) + assert.Equal(t, DefaultRedisAuditStream, c.stream) + assert.Equal(t, defaultConsumerGroup, c.group) + assert.Equal(t, "gitopsreverser-consumer-0", c.consumerID) +} + +// --- processMessage integration using miniredis --- + +func newTestConsumer( + t *testing.T, + mr *miniredis.Miniredis, + rs *rulestore.RuleStore, + er AuditEventRouter, +) *AuditConsumer { + t.Helper() + c, err := NewAuditConsumer( + AuditConsumerConfig{ + Addr: mr.Addr(), + Stream: "test-stream", + Group: "test-group", + ConsumerID: "test-consumer", + }, + rs, + er, + logr.Discard(), + ) + require.NoError(t, err) + return c +} + +func pushAuditMessage(t *testing.T, mr *miniredis.Miniredis, ev auditv1.Event) { + t.Helper() + payload, err := json.Marshal(ev) + require.NoError(t, err) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + err = client.XAdd(context.Background(), &redis.XAddArgs{ + Stream: "test-stream", + ID: "*", + Values: map[string]interface{}{ + "payload_json": string(payload), + "cluster_id": "test-cluster", + }, + }).Err() + require.NoError(t, err) +} + +func TestProcessMessage_NonResponseCompleteStageIsACKed(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("create", auditv1.StageRequestReceived, "configmaps", "default", "cm") + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + // No routing happened. + assert.Empty(t, er.calls) + // Message is ACKed (pending count = 0). + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_ReadOnlyVerbIsACKed(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("get", auditv1.StageResponseComplete, "configmaps", "default", "cm") + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + assert.Empty(t, er.calls) + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_NoObjectRefIsACKed(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := auditv1.Event{ + Verb: "create", + Stage: auditv1.StageResponseComplete, + // ObjectRef intentionally nil + } + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + assert.Empty(t, er.calls) + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_NoMatchingRulesIsACKed(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) // empty rule store + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("create", auditv1.StageResponseComplete, "configmaps", "default", "cm") + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + assert.Empty(t, er.calls) + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_MatchingRuleRoutesAndACKs(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + + rs := rulestore.NewStore() + rs.AddOrUpdateWatchRule( + makeWatchRule("my-rule", []string{"configmaps"}, []string{"v1"}, []string{""}), + "my-target", "default", + "my-provider", "default", + "main", "state/", + ) + + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetNamespace("default") + obj.SetName("cm-test") + raw, _ := obj.MarshalJSON() + + ev := makeAuditEvent("create", auditv1.StageResponseComplete, "configmaps", "default", "cm-test") + ev.ResponseObject = &runtime.Unknown{Raw: raw} + + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + require.Len(t, er.calls, 1) + assert.Equal(t, "cm-test", er.calls[0].Event.Identifier.Name) + assert.Equal(t, string(configv1alpha1.OperationCreate), er.calls[0].Event.Operation) + assert.Equal(t, "state/", er.calls[0].Event.Path) + assert.Equal(t, "my-target", er.calls[0].GitDest.Name) + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_ImpersonatedUserPropagated(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + + rs := rulestore.NewStore() + rs.AddOrUpdateWatchRule( + makeWatchRule("my-rule", []string{"configmaps"}, []string{"v1"}, []string{""}), + "my-target", "default", + "my-provider", "default", + "main", "", + ) + + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("update", auditv1.StageResponseComplete, "configmaps", "default", "cm") + ev.User.Username = "system:serviceaccount:kube-system:replicaset-controller" + ev.ImpersonatedUser = &authv1.UserInfo{Username: "real-user"} + + pushAuditMessage(t, mr, ev) + require.NoError(t, c.readAndProcessBatch(context.Background())) + + require.Len(t, er.calls, 1) + assert.Equal(t, "real-user", er.calls[0].Event.UserInfo.Username) +} + +func TestProcessMessage_PoisonPillIsACKed(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + // Push an entry with invalid payload_json. + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + err := client.XAdd(context.Background(), &redis.XAddArgs{ + Stream: "test-stream", + ID: "*", + Values: map[string]interface{}{"payload_json": "not-valid-json"}, + }).Err() + require.NoError(t, err) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + assert.Empty(t, er.calls) + assertNoPendingMessages(t, mr) +} + +func TestEnsureConsumerGroup_IdempotentOnExistingGroup(t *testing.T) { + mr := miniredis.RunT(t) + c := newTestConsumer(t, mr, rulestore.NewStore(), &fakeEventRouter{}) + + require.NoError(t, c.ensureConsumerGroup(context.Background())) + // Second call must not return an error. + require.NoError(t, c.ensureConsumerGroup(context.Background())) +} + +func TestNeedLeaderElection(t *testing.T) { + c := &AuditConsumer{} + assert.True(t, c.NeedLeaderElection()) +} + +func TestStartCancellation(t *testing.T) { + mr := miniredis.RunT(t) + c := newTestConsumer(t, mr, rulestore.NewStore(), &fakeEventRouter{}) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + err := c.Start(ctx) + assert.NoError(t, err) +} + +// --- helpers --- + +func makeAuditEvent(verb string, stage auditv1.Stage, resource, namespace, name string) auditv1.Event { + ev := auditv1.Event{ + Verb: verb, + Stage: stage, + ObjectRef: &auditv1.ObjectReference{ + APIVersion: "v1", + Resource: resource, + Namespace: namespace, + Name: name, + }, + } + ev.User.Username = "test-user" + return ev +} + +func makeWatchRule( + name string, + resources, versions, groups []string, +) configv1alpha1.WatchRule { + ops := []configv1alpha1.OperationType{configv1alpha1.OperationAll} + return configv1alpha1.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: configv1alpha1.WatchRuleSpec{ + Rules: []configv1alpha1.ResourceRule{ + { + Operations: ops, + APIGroups: groups, + APIVersions: versions, + Resources: resources, + }, + }, + }, + } +} + +func TestProcessMessage_ClusterWatchRuleRoutesAndACKs(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + + rs := rulestore.NewStore() + rs.AddOrUpdateClusterWatchRule( + makeClusterWatchRule( + "cwr", + []string{"nodes"}, + []string{"v1"}, + []string{""}, + configv1alpha1.ResourceScopeCluster, + ), + "my-target", + "default", + "my-provider", + "default", + "main", + "cluster/", + ) + + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + // Cluster-scoped resource: no namespace. + ev := makeAuditEvent("create", auditv1.StageResponseComplete, "nodes", "", "node-1") + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + require.Len(t, er.calls, 1) + assert.Equal(t, "node-1", er.calls[0].Event.Identifier.Name) + assert.Equal(t, "cluster/", er.calls[0].Event.Path) + assertNoPendingMessages(t, mr) +} + +func TestProcessMessage_CustomResourceUsesObjectRefAPIGroup(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + + rs := rulestore.NewStore() + rs.AddOrUpdateWatchRule( + makeWatchRule("rule-cr", []string{"icecreamorders"}, []string{"v1"}, []string{"shop.example.com"}), + "my-target", "default", + "my-provider", "default", + "main", "live/", + ) + + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("create", auditv1.StageResponseComplete, "icecreamorders", "default", "order-1") + ev.ObjectRef.APIGroup = "shop.example.com" + ev.ObjectRef.APIVersion = "v1" + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + require.Len(t, er.calls, 1) + assert.Equal(t, "shop.example.com", er.calls[0].Event.Identifier.Group) + assert.Equal(t, "v1", er.calls[0].Event.Identifier.Version) + assert.Equal(t, "icecreamorders", er.calls[0].Event.Identifier.Resource) + assert.Equal(t, "default", er.calls[0].Event.Identifier.Namespace) + assert.Equal(t, "order-1", er.calls[0].Event.Identifier.Name) + assert.Equal(t, "live/", er.calls[0].Event.Path) + assertNoPendingMessages(t, mr) +} + +func TestRouteTMatchedRules_PartialRouteFailureStillACKs(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{ + errFor: map[string]error{ + itypes.NewResourceReference("failing-target", "default").Key(): errors.New("router unavailable"), + }, + } + + rs := rulestore.NewStore() + rs.AddOrUpdateWatchRule( + makeWatchRule("rule-ok", []string{"configmaps"}, []string{"v1"}, []string{""}), + "ok-target", "default", + "my-provider", "default", + "main", "ok/", + ) + rs.AddOrUpdateWatchRule( + makeWatchRule("rule-fail", []string{"configmaps"}, []string{"v1"}, []string{""}), + "failing-target", "default", + "my-provider", "default", + "main", "fail/", + ) + + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("update", auditv1.StageResponseComplete, "configmaps", "default", "cm") + pushAuditMessage(t, mr, ev) + + require.NoError(t, c.readAndProcessBatch(context.Background())) + + // One rule failed, one succeeded — message is still ACKed. + assert.Len(t, er.calls, 2) + assertNoPendingMessages(t, mr) +} + +func TestRunAutoClaimCycle_ReclaimsIdleMessages(t *testing.T) { + mr := miniredis.RunT(t) + + rs := rulestore.NewStore() + rs.AddOrUpdateWatchRule( + makeWatchRule("my-rule", []string{"configmaps"}, []string{"v1"}, []string{""}), + "my-target", "default", + "my-provider", "default", + "main", "", + ) + + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rs, er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + ev := makeAuditEvent("create", auditv1.StageResponseComplete, "configmaps", "default", "cm-1") + pushAuditMessage(t, mr, ev) + + // Read but do NOT ACK — simulates a crashed consumer leaving a pending entry. + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + _, err := client.XReadGroup(context.Background(), &redis.XReadGroupArgs{ + Group: "test-group", + Consumer: "other-consumer", + Streams: []string{"test-stream", ">"}, + Count: 10, + Block: -1, + }).Result() + require.NoError(t, err) + + // Fast-forward miniredis time so the message appears idle. + mr.FastForward(autoClaimMinIdle + time.Second) + + // Override the consumer's minIdle to 0 by using XAUTOCLAIM directly with no min-idle. + // This verifies the cycle loop processes and ACKs the reclaimed message. + messages, _, err := client.XAutoClaim(context.Background(), &redis.XAutoClaimArgs{ + Stream: "test-stream", + Group: "test-group", + Consumer: "test-consumer", + MinIdle: 0, // claim regardless of idle time + Start: "0-0", + Count: 100, + }).Result() + require.NoError(t, err) + require.NotEmpty(t, messages, "expected at least one pending message to reclaim") + + // Now exercise processMessage directly on the reclaimed entry. + for _, msg := range messages { + c.processMessage(context.Background(), msg) + } + + require.Len(t, er.calls, 1) + assertNoPendingMessages(t, mr) +} + +func TestRunAutoClaimCycle_NoopWhenNoPendingMessages(t *testing.T) { + mr := miniredis.RunT(t) + c := newTestConsumer(t, mr, rulestore.NewStore(), &fakeEventRouter{}) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + // Should not panic or error with an empty stream. + c.runAutoClaimCycle(context.Background()) +} + +func TestStart_ErrorOnEnsureGroupReturnsError(t *testing.T) { + // Point consumer at a non-existent Redis server to trigger an error. + c, err := NewAuditConsumer( + AuditConsumerConfig{ + Addr: "127.0.0.1:19999", // nothing listening here + Stream: "test-stream", + Group: "test-group", + ConsumerID: "test-consumer", + }, + rulestore.NewStore(), + &fakeEventRouter{}, + logr.Discard(), + ) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + err = c.Start(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to ensure consumer group") +} + +func TestReadAndProcessBatch_XReadGroupErrorReturnsError(t *testing.T) { + // Point consumer at a dead server so XREADGROUP fails. + c, err := NewAuditConsumer( + AuditConsumerConfig{ + Addr: "127.0.0.1:19999", + Stream: "test-stream", + Group: "test-group", + ConsumerID: "test-consumer", + }, + rulestore.NewStore(), + &fakeEventRouter{}, + logr.Discard(), + ) + require.NoError(t, err) + + err = c.readAndProcessBatch(context.Background()) + require.Error(t, err) +} + +func TestReadAndProcessBatch_RecreatesGroupOnNoGroup(t *testing.T) { + mr := miniredis.RunT(t) + er := &fakeEventRouter{} + c := newTestConsumer(t, mr, rulestore.NewStore(), er) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + + err := client.Del(context.Background(), "test-stream").Err() + require.NoError(t, err) + + err = c.readAndProcessBatch(context.Background()) + require.NoError(t, err) + + groups, err := client.XInfoGroups(context.Background(), "test-stream").Result() + require.NoError(t, err) + require.Len(t, groups, 1) + assert.Equal(t, "test-group", groups[0].Name) +} + +func TestRunAutoClaimCycle_RecreatesGroupOnNoGroup(t *testing.T) { + mr := miniredis.RunT(t) + c := newTestConsumer(t, mr, rulestore.NewStore(), &fakeEventRouter{}) + require.NoError(t, c.ensureConsumerGroup(context.Background())) + + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + + err := client.Del(context.Background(), "test-stream").Err() + require.NoError(t, err) + + c.runAutoClaimCycle(context.Background()) + + groups, err := client.XInfoGroups(context.Background(), "test-stream").Result() + require.NoError(t, err) + require.Len(t, groups, 1) + assert.Equal(t, "test-group", groups[0].Name) +} + +func assertNoPendingMessages(t *testing.T, mr *miniredis.Miniredis) { + t.Helper() + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + defer client.Close() + + info, err := client.XPending(context.Background(), "test-stream", "test-group").Result() + require.NoError(t, err) + assert.Zero(t, info.Count, "consumer should have ACKed all stream entries") +} + +func makeClusterWatchRule( + name string, + resources, versions, groups []string, + scope configv1alpha1.ResourceScope, +) configv1alpha1.ClusterWatchRule { + ops := []configv1alpha1.OperationType{configv1alpha1.OperationAll} + return configv1alpha1.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: configv1alpha1.ClusterWatchRuleSpec{ + Rules: []configv1alpha1.ClusterResourceRule{ + { + Operations: ops, + APIGroups: groups, + APIVersions: versions, + Resources: resources, + Scope: scope, + }, + }, + }, + } +} diff --git a/internal/queue/redis_audit_queue.go b/internal/queue/redis_audit_queue.go index 6e0c05f24..cce48710b 100644 --- a/internal/queue/redis_audit_queue.go +++ b/internal/queue/redis_audit_queue.go @@ -94,11 +94,13 @@ func (q *RedisAuditQueue) Enqueue(ctx context.Context, clusterID string, event a objectRef := event.ObjectRef apiVersion := "" + apiGroup := "" resource := "" namespace := "" name := "" if objectRef != nil { apiVersion = objectRef.APIVersion + apiGroup = objectRef.APIGroup resource = objectRef.Resource namespace = objectRef.Namespace name = objectRef.Name @@ -109,6 +111,7 @@ func (q *RedisAuditQueue) Enqueue(ctx context.Context, clusterID string, event a "audit_id": string(event.AuditID), "cluster_id": clusterID, "verb": event.Verb, + "api_group": apiGroup, "api_version": apiVersion, "resource": resource, "namespace": namespace, diff --git a/internal/queue/redis_audit_queue_test.go b/internal/queue/redis_audit_queue_test.go index c8302160f..f0d75be6f 100644 --- a/internal/queue/redis_audit_queue_test.go +++ b/internal/queue/redis_audit_queue_test.go @@ -75,6 +75,7 @@ func TestRedisAuditQueue_Enqueue(t *testing.T) { assert.Equal(t, "audit-123", entry["audit_id"]) assert.Equal(t, "cluster-a", entry["cluster_id"]) assert.Equal(t, "create", entry["verb"]) + assert.Empty(t, entry["api_group"]) assert.Equal(t, "v1", entry["api_version"]) assert.Equal(t, "configmaps", entry["resource"]) assert.Equal(t, "default", entry["namespace"]) @@ -83,3 +84,47 @@ func TestRedisAuditQueue_Enqueue(t *testing.T) { assert.NotEmpty(t, entry["event_id"]) assert.NotEmpty(t, entry["payload_json"]) } + +func TestRedisAuditQueue_EnqueueCustomResourceStoresAPIGroup(t *testing.T) { + mr := miniredis.RunT(t) + + queue, err := NewRedisAuditQueue(RedisAuditQueueConfig{ + Addr: mr.Addr(), + Stream: "audit.events.test", + MaxLen: 100, + }) + require.NoError(t, err) + + event := auditv1.Event{ + AuditID: "audit-cr-123", + Verb: "create", + RequestURI: "/apis/shop.example.com/v1/namespaces/default/icecreamorders/order-1", + Stage: "ResponseComplete", + StageTimestamp: metav1.MicroTime{ + Time: time.Date(2026, 3, 5, 10, 0, 0, 0, time.UTC), + }, + ObjectRef: &auditv1.ObjectReference{ + APIGroup: "shop.example.com", + APIVersion: "v1", + Resource: "icecreamorders", + Namespace: "default", + Name: "order-1", + }, + } + event.User.Username = "test-user" + + err = queue.Enqueue(context.Background(), "cluster-a", event) + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + entries, err := client.XRange(context.Background(), "audit.events.test", "-", "+").Result() + require.NoError(t, err) + require.Len(t, entries, 1) + + entry := entries[0].Values + assert.Equal(t, "shop.example.com", entry["api_group"]) + assert.Equal(t, "v1", entry["api_version"]) + assert.Equal(t, "icecreamorders", entry["resource"]) + assert.Equal(t, "default", entry["namespace"]) + assert.Equal(t, "order-1", entry["name"]) +} diff --git a/internal/reconcile/folder_reconciler.go b/internal/reconcile/folder_reconciler.go index 830390a14..3e739aa5c 100644 --- a/internal/reconcile/folder_reconciler.go +++ b/internal/reconcile/folder_reconciler.go @@ -84,6 +84,8 @@ func NewFolderReconciler( // StartReconciliation initiates the reconciliation process by requesting state. func (r *FolderReconciler) StartReconciliation(_ context.Context) error { + r.ResetState() + r.logger.Info("Starting reconciliation") // Emit control events to request both cluster and repo state @@ -104,6 +106,16 @@ func (r *FolderReconciler) StartReconciliation(_ context.Context) error { return nil } +// ResetState clears any previously observed repo/cluster snapshots so the next +// reconciliation cycle only runs on a fresh pair of state events. +func (r *FolderReconciler) ResetState() { + r.clusterResources = nil + r.gitResources = nil + r.clusterObjects = nil + r.clusterStateSeen = false + r.gitStateSeen = false +} + // OnClusterState handles cluster state events and triggers reconciliation. func (r *FolderReconciler) OnClusterState(event events.ClusterStateEvent) { if !event.GitDest.Equal(r.gitDest) { diff --git a/internal/reconcile/folder_reconciler_test.go b/internal/reconcile/folder_reconciler_test.go index 02824e1dc..c0a170b5b 100644 --- a/internal/reconcile/folder_reconciler_test.go +++ b/internal/reconcile/folder_reconciler_test.go @@ -354,6 +354,41 @@ func TestFolderReconciler_EmitsSingleBatch(t *testing.T) { assert.Contains(t, batch.CommitMessage, "reconcile: sync 3 resources") } +func TestFolderReconciler_ResetStateRequiresFreshRepoAndClusterSnapshots(t *testing.T) { + mockEmitter := &MockReconcileEmitter{} + mockControlEmitter := &MockControlEventEmitter{} + gitDest := types.NewResourceReference("test-gitdest", "default") + reconciler := NewFolderReconciler(gitDest, mockEmitter, mockControlEmitter, log.Log) + + initialResource := types.ResourceIdentifier{Group: "", Version: "v1", Resource: "configmaps", Name: "old"} + updatedResource := types.ResourceIdentifier{Group: "", Version: "v1", Resource: "configmaps", Name: "new"} + + reconciler.OnClusterState(events.ClusterStateEvent{ + GitDest: gitDest, + Resources: []types.ResourceIdentifier{initialResource}, + }) + reconciler.OnRepoState(events.RepoStateEvent{ + GitDest: gitDest, + Resources: []types.ResourceIdentifier{initialResource}, + }) + assert.Len(t, mockEmitter.Batches, 1, "Initial snapshot should emit one batch") + + reconciler.ResetState() + assert.False(t, reconciler.HasBothStates(), "ResetState should clear observed snapshot flags") + + reconciler.OnRepoState(events.RepoStateEvent{ + GitDest: gitDest, + Resources: []types.ResourceIdentifier{updatedResource}, + }) + assert.Len(t, mockEmitter.Batches, 1, "Fresh repo state alone should not reconcile against stale cluster state") + + reconciler.OnClusterState(events.ClusterStateEvent{ + GitDest: gitDest, + Resources: []types.ResourceIdentifier{updatedResource}, + }) + assert.Len(t, mockEmitter.Batches, 2, "Fresh cluster+repo snapshots should trigger the next batch") +} + // MockReconcileEmitter is a mock implementation of ReconcileEmitter for testing. type MockReconcileEmitter struct { Batches []git.ReconcileBatch diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index 564d17d90..c7eb3cfd4 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -65,14 +65,6 @@ var ( // RepoBranchQueueDepth is a gauge for per-repo-branch queue depth. RepoBranchQueueDepth metric.Int64UpDownCounter - // WebhookCorrelationsTotal counts correlation entries created by webhook. - WebhookCorrelationsTotal metric.Int64Counter - // EnrichHitsTotal counts successful webhook→watch correlation enrichments. - EnrichHitsTotal metric.Int64Counter - // EnrichMissesTotal counts failed webhook→watch correlations (no match found). - EnrichMissesTotal metric.Int64Counter - // KVEvictionsTotal counts correlation store evictions (TTL or LRU). - KVEvictionsTotal metric.Int64Counter // WatchDuplicatesSkippedTotal counts watch events skipped due to duplicate sanitized content. WatchDuplicatesSkippedTotal metric.Int64Counter // AuditEventsReceivedTotal counts audit events received from Kubernetes API server. @@ -136,10 +128,6 @@ func InitOTLPExporter(_ context.Context) (func(context.Context) error, error) { {"gitopsreverser_ownership_conflicts_total", &OwnershipConflictsTotal}, {"gitopsreverser_lease_acquire_failures_total", &LeaseAcquireFailuresTotal}, {"gitopsreverser_marker_conflicts_total", &MarkerConflictsTotal}, - {"gitopsreverser_webhook_correlations_total", &WebhookCorrelationsTotal}, - {"gitopsreverser_enrich_hits_total", &EnrichHitsTotal}, - {"gitopsreverser_enrich_misses_total", &EnrichMissesTotal}, - {"gitopsreverser_kv_evictions_total", &KVEvictionsTotal}, {"gitopsreverser_watch_duplicates_skipped_total", &WatchDuplicatesSkippedTotal}, {"gitopsreverser_audit_events_received_total", &AuditEventsReceivedTotal}, {"gitopsreverser_secret_encryption_attempts_total", &SecretEncryptionAttemptsTotal}, diff --git a/internal/watch/informers.go b/internal/watch/informers.go index 2e33a8aec..4427004cc 100644 --- a/internal/watch/informers.go +++ b/internal/watch/informers.go @@ -58,7 +58,7 @@ func (m *Manager) addHandlers(inf cache.SharedIndexInformer, g GVR) { } // handleEvent converts an informer object to Unstructured, matches rules, sanitizes, -// enriches with correlation username, and enqueues commit events. +// and enqueues commit events unless audit is authoritative for live mutations. // Implements deduplication to skip status-only changes. func (m *Manager) handleEvent(obj interface{}, g GVR, op configv1alpha1.OperationType) { u := toUnstructuredFromInformer(obj) @@ -99,8 +99,7 @@ func (m *Manager) handleEvent(obj interface{}, g GVR, op configv1alpha1.Operatio return } - // Attempt correlation enrichment - userInfo := m.tryEnrichFromCorrelation(ctx, sanitized, id, string(op)) + userInfo := git.UserInfo{} // username not available from watch path // Emit basic metrics for watcher path (mirrors webhook semantics). // Count each watched object processed by the informer path. @@ -111,6 +110,17 @@ func (m *Manager) handleEvent(obj interface{}, g GVR, op configv1alpha1.Operatio telemetry.GitCommitQueueSize.Add(ctx, enqueueCount) } + if m.AuditLiveEventsEnabled { + m.Log.V(1).Info("Skipping live watch event routing because audit stream is authoritative", + "identifier", id.String(), + "operation", op, + "scope", g.Scope, + "resource", g.Resource, + "watchRuleMatches", len(wrRules), + "clusterWatchRuleMatches", len(cwrRules)) + return + } + m.routeWatchRules(wrRules, sanitized, id, string(op), userInfo) m.routeClusterWatchRules(cwrRules, sanitized, id, string(op), userInfo) } diff --git a/internal/watch/informers_test.go b/internal/watch/informers_test.go new file mode 100644 index 000000000..aff52483c --- /dev/null +++ b/internal/watch/informers_test.go @@ -0,0 +1,267 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +import ( + "context" + "sync" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/reconcile" + "github.com/ConfigButler/gitops-reverser/internal/rulestore" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +var initWatchMetricsOnce sync.Once + +type recordingEnqueuer struct { + events []git.Event +} + +func (r *recordingEnqueuer) Enqueue(event git.Event) { + r.events = append(r.events, event) +} + +func (r *recordingEnqueuer) EnqueueRequest(request *git.WriteRequest) { + if request == nil { + return + } + r.events = append(r.events, request.Events...) +} + +func (r *recordingEnqueuer) EnqueueBatch(_ *git.ReconcileBatch) {} + +func TestHandleEvent_SkipsLiveRoutingWhenAuditLiveEventsEnabled(t *testing.T) { + initWatchMetricsOnce.Do(func() { + _, _ = telemetry.InitOTLPExporter(context.Background()) + }) + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add core scheme: %v", err) + } + if err := configv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add api scheme: %v", err) + } + + store := rulestore.NewStore() + store.AddOrUpdateWatchRule( + configv1alpha1.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "wr", Namespace: "default"}, + Spec: configv1alpha1.WatchRuleSpec{ + TargetRef: configv1alpha1.LocalTargetReference{Name: "target"}, + Rules: []configv1alpha1.ResourceRule{{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"configmaps"}, + Operations: []configv1alpha1.OperationType{configv1alpha1.OperationAll}, + }}, + }, + }, + "target", "default", "provider", "default", "main", "state", + ) + + router := &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: make(map[string]*reconcile.GitTargetEventStream), + } + enqueuer := &recordingEnqueuer{} + stream := reconcile.NewGitTargetEventStream("target", "default", enqueuer, logr.Discard()) + stream.OnReconciliationComplete() + gitDest := itypes.NewResourceReference("target", "default") + router.RegisterGitTargetEventStream(gitDest, stream) + + fakeK8s := fakeclient.NewClientBuilder(). + WithScheme(scheme). + WithObjects(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + }). + Build() + + manager := &Manager{ + Client: fakeK8s, + Log: logr.Discard(), + RuleStore: store, + EventRouter: router, + AuditLiveEventsEnabled: true, + } + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-test", + Namespace: "default", + }, + Data: map[string]string{"key": "value"}, + } + + manager.handleEvent(obj, GVR{Group: "", Version: "v1", Resource: "configmaps"}, configv1alpha1.OperationCreate) + + if len(enqueuer.events) != 0 { + t.Fatalf( + "expected no live events to be enqueued when audit live events are enabled, got %d", + len(enqueuer.events), + ) + } +} + +func TestHandleEvent_SkipsClusterScopedLiveRoutingWhenAuditLiveEventsEnabled(t *testing.T) { + initWatchMetricsOnce.Do(func() { + _, _ = telemetry.InitOTLPExporter(context.Background()) + }) + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add core scheme: %v", err) + } + if err := configv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add api scheme: %v", err) + } + + store := rulestore.NewStore() + store.AddOrUpdateClusterWatchRule( + configv1alpha1.ClusterWatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "cwr"}, + Spec: configv1alpha1.ClusterWatchRuleSpec{ + TargetRef: configv1alpha1.NamespacedTargetReference{Name: "target", Namespace: "default"}, + Rules: []configv1alpha1.ClusterResourceRule{{ + Scope: configv1alpha1.ResourceScopeCluster, + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"namespaces"}, + Operations: []configv1alpha1.OperationType{configv1alpha1.OperationAll}, + }}, + }, + }, + "target", "default", "provider", "default", "main", "state", + ) + + router := &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: make(map[string]*reconcile.GitTargetEventStream), + } + enqueuer := &recordingEnqueuer{} + stream := reconcile.NewGitTargetEventStream("target", "default", enqueuer, logr.Discard()) + stream.OnReconciliationComplete() + gitDest := itypes.NewResourceReference("target", "default") + router.RegisterGitTargetEventStream(gitDest, stream) + + fakeK8s := fakeclient.NewClientBuilder().WithScheme(scheme).Build() + + manager := &Manager{ + Client: fakeK8s, + Log: logr.Discard(), + RuleStore: store, + EventRouter: router, + AuditLiveEventsEnabled: true, + } + + obj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-scope-test"}, + } + + manager.handleEvent( + obj, + GVR{Group: "", Version: "v1", Resource: "namespaces", Scope: configv1alpha1.ResourceScopeCluster}, + configv1alpha1.OperationCreate, + ) + + if len(enqueuer.events) != 0 { + t.Fatalf("expected cluster-scoped live event to be skipped, got %d", len(enqueuer.events)) + } +} + +func TestHandleEvent_SkipsSecretLiveRoutingWhenAuditLiveEventsEnabled(t *testing.T) { + initWatchMetricsOnce.Do(func() { + _, _ = telemetry.InitOTLPExporter(context.Background()) + }) + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add core scheme: %v", err) + } + if err := configv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add api scheme: %v", err) + } + + store := rulestore.NewStore() + store.AddOrUpdateWatchRule( + configv1alpha1.WatchRule{ + ObjectMeta: metav1.ObjectMeta{Name: "wr", Namespace: "default"}, + Spec: configv1alpha1.WatchRuleSpec{ + TargetRef: configv1alpha1.LocalTargetReference{Name: "target"}, + Rules: []configv1alpha1.ResourceRule{{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"secrets"}, + Operations: []configv1alpha1.OperationType{configv1alpha1.OperationAll}, + }}, + }, + }, + "target", "default", "provider", "default", "main", "state", + ) + + router := &EventRouter{ + Log: logr.Discard(), + gitTargetStreams: make(map[string]*reconcile.GitTargetEventStream), + } + enqueuer := &recordingEnqueuer{} + stream := reconcile.NewGitTargetEventStream("target", "default", enqueuer, logr.Discard()) + stream.OnReconciliationComplete() + gitDest := itypes.NewResourceReference("target", "default") + router.RegisterGitTargetEventStream(gitDest, stream) + + fakeK8s := fakeclient.NewClientBuilder(). + WithScheme(scheme). + WithObjects(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + }). + Build() + + manager := &Manager{ + Client: fakeK8s, + Log: logr.Discard(), + RuleStore: store, + EventRouter: router, + AuditLiveEventsEnabled: true, + } + + obj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-test", + Namespace: "default", + }, + StringData: map[string]string{"token": "value"}, + } + + manager.handleEvent(obj, GVR{Group: "", Version: "v1", Resource: "secrets"}, configv1alpha1.OperationCreate) + + if len(enqueuer.events) != 0 { + t.Fatalf("expected secret live event to be skipped, got %d", len(enqueuer.events)) + } +} diff --git a/internal/watch/manager.go b/internal/watch/manager.go index a3d2d8e03..3e56108cf 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -41,12 +41,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" - "github.com/ConfigButler/gitops-reverser/internal/correlation" "github.com/ConfigButler/gitops-reverser/internal/events" - "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/rulestore" "github.com/ConfigButler/gitops-reverser/internal/sanitize" - "github.com/ConfigButler/gitops-reverser/internal/telemetry" "github.com/ConfigButler/gitops-reverser/internal/types" ) @@ -65,9 +62,9 @@ type Manager struct { RuleStore *rulestore.RuleStore // EventRouter dispatches events to branch workers (replaces EventQueue). EventRouter *EventRouter - // CorrelationStore enables webhook→watch username enrichment. - CorrelationStore *correlation.Store - + // AuditLiveEventsEnabled makes the audit pipeline authoritative for live mutating events. + // Watchers still support discovery and snapshot/reconcile flows. + AuditLiveEventsEnabled bool // Deduplication: tracks last seen content hash per resource to skip status-only changes lastSeenMu sync.RWMutex lastSeenHash map[string]uint64 // resourceKey → content hash (key uses types.ResourceIdentifier.Key) @@ -259,43 +256,6 @@ func (m *Manager) isDuplicateContent( return false } -// tryEnrichFromCorrelation attempts to enrich an event with username from the correlation store. -func (m *Manager) tryEnrichFromCorrelation( - ctx context.Context, - sanitized *unstructured.Unstructured, - id types.ResourceIdentifier, - operation string, -) git.UserInfo { - userInfo := git.UserInfo{} // default: no username - - if m.CorrelationStore == nil { - return userInfo - } - - log := m.Log.WithName("correlation") - sanitizedYAML, err := sanitize.MarshalToOrderedYAML(sanitized) - if err != nil { - log.Error(err, "Failed to marshal for correlation", "identifier", id.String()) - return userInfo - } - - key := correlation.GenerateKey(id, operation, sanitizedYAML) - entry, found := m.CorrelationStore.Get(key) - if !found { - telemetry.EnrichMissesTotal.Add(ctx, 1) - log.V(1).Info("No correlation match", "identifier", id.String(), "key", key) - return userInfo - } - - userInfo.Username = entry.Username - telemetry.EnrichHitsTotal.Add(ctx, 1) - log.V(1).Info("Enriched with username", - "identifier", id.String(), - "username", entry.Username, - "key", key) - return userInfo -} - // dynamicClientFromConfig builds a dynamic client from the controller's REST config. // If m.dynamicClient is set (e.g. in tests) it is returned directly. func (m *Manager) dynamicClientFromConfig(log logr.Logger) dynamic.Interface { @@ -1296,17 +1256,28 @@ func (m *Manager) beginReconciliationForAffectedTargets(log logr.Logger) { } } -// emitSnapshotForRuleChange emits RequestClusterState for every affected GitTarget so -// that a single "reconcile: sync N resources" commit is produced instead of N individual -// [CREATE] commits from informer ADDED events. +// emitSnapshotForRuleChange emits fresh repo and cluster state requests for every affected +// GitTarget so FolderReconciler diffs against current repository contents rather than a +// stale cached repo snapshot from an earlier reconcile. func (m *Manager) emitSnapshotForRuleChange(ctx context.Context, log logr.Logger) { if m.EventRouter == nil { - log.Info("EventRouter not set, skipping RequestClusterState emission") + log.Info("EventRouter not set, skipping snapshot emission") return } targets := m.collectAffectedGitTargets() - log.Info("Emitting RequestClusterState for affected GitTargets after rule change", "count", len(targets)) + log.Info("Emitting fresh repo and cluster state for affected GitTargets after rule change", "count", len(targets)) for _, gitDest := range targets { + if m.EventRouter != nil && m.EventRouter.ReconcilerManager != nil { + if reconciler, exists := m.EventRouter.ReconcilerManager.GetReconciler(gitDest); exists { + reconciler.ResetState() + } + } + if err := m.EventRouter.ProcessControlEvent(ctx, events.ControlEvent{ + Type: events.RequestRepoState, + GitDest: gitDest, + }); err != nil { + log.Error(err, "failed to emit RequestRepoState for rule change", "gitDest", gitDest) + } if err := m.EventRouter.ProcessControlEvent(ctx, events.ControlEvent{ Type: events.RequestClusterState, GitDest: gitDest, diff --git a/internal/webhook/event_handler.go b/internal/webhook/event_handler.go deleted file mode 100644 index 436be02fd..000000000 --- a/internal/webhook/event_handler.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package webhook handles admission webhook requests and experimental audit webhook for the GitOps Reverser controller. -// The admission webhook serves as a correlation store only - it captures user attribution but does NOT -// enqueue events. The watch path (informers) is the sole source of events to the queue. -// The audit webhook collects experimental metrics from Kubernetes audit events. -package webhook - -import ( - "context" - "errors" - "fmt" - "net/http" - "os" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - apitypes "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" - - "github.com/ConfigButler/gitops-reverser/internal/correlation" - "github.com/ConfigButler/gitops-reverser/internal/sanitize" - "github.com/ConfigButler/gitops-reverser/internal/telemetry" - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -//nolint:lll // Kubebuilder webhook annotation -// +kubebuilder:webhook:path=/process-validating-webhook,mutating=false,failurePolicy=ignore,sideEffects=None,groups="*",resources="*",verbs=create;update;delete,versions="*",name=gitops-reverser.configbutler.ai,admissionReviewVersions=v1 - -// EventHandler handles all incoming admission requests. -// It stores correlation entries (username → sanitized content) for ALL resources. -// The watch path will filter based on rules - webhook just captures user attribution. -type EventHandler struct { - Client client.Client - Decoder *admission.Decoder - CorrelationStore *correlation.Store -} - -// Handle implements admission.Handler. -// - -func (h *EventHandler) Handle(ctx context.Context, req admission.Request) admission.Response { - log := logf.FromContext(ctx) - - // Metrics: attribute role based on pod labels - roleAttr := h.getPodRoleAttribute(ctx) - telemetry.EventsReceivedTotal.Add(ctx, 1, metric.WithAttributes(roleAttr)) - - // Safety: require decoder - if h.Decoder == nil { - err := errors.New("decoder is not initialized") - log.Error(err, "Webhook handler received request but decoder is nil") - return admission.Errored(http.StatusInternalServerError, err) - } - - // Decode incoming object (handles create/update/delete and missing payloads) - obj, err := h.decodeObject(ctx, req) - if err != nil { - log.Error(err, "Failed to decode admission request", "operation", req.Operation, "kind", req.Kind.Kind) - return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to decode request: %w", err)) - } - - log.V(1).Info("Successfully decoded resource", //nolint:lll // Structured log - "kind", obj.GetKind(), "name", obj.GetName(), "namespace", obj.GetNamespace(), "operation", req.Operation) - - // Store correlation for ALL resources (watch path will filter based on rules) - identifier := types.FromAdmissionRequest(req) - h.storeCorrelation(ctx, obj, identifier, req) - - return admission.Allowed("request is allowed") -} - -// decodeObject decodes the AdmissionRequest into an Unstructured object, with sensible fallbacks. -func (h *EventHandler) decodeObject(ctx context.Context, req admission.Request) (*unstructured.Unstructured, error) { - log := logf.FromContext(ctx) - obj := &unstructured.Unstructured{} - - switch { - case req.Operation == "DELETE" && req.OldObject.Size() > 0: - if err := (*h.Decoder).DecodeRaw(req.OldObject, obj); err != nil { - return nil, err - } - case req.Object.Size() > 0: - if err := (*h.Decoder).Decode(req, obj); err != nil { - return nil, err - } - default: - // If no object data is available, create a minimal object from admission request metadata - log.V(1).Info("No object data available, creating minimal object from request metadata") - obj.SetAPIVersion(req.Kind.Group + "/" + req.Kind.Version) - obj.SetKind(req.Kind.Kind) - obj.SetName(req.Name) - obj.SetNamespace(req.Namespace) - } - - return obj, nil -} - -// storeCorrelation stores a correlation entry for ALL resources. -// Watch path will filter based on rules - webhook just captures user attribution. -func (h *EventHandler) storeCorrelation( - ctx context.Context, - obj *unstructured.Unstructured, - identifier types.ResourceIdentifier, - req admission.Request, -) { - log := logf.FromContext(ctx) - - if h.CorrelationStore == nil { - return - } - - sanitizedObj := sanitize.Sanitize(obj) - sanitizedYAML, err := sanitize.MarshalToOrderedYAML(sanitizedObj) - if err != nil { - log.Error(err, "Failed to marshal sanitized object for correlation", "identifier", identifier.String()) - return - } - - key := correlation.GenerateKey(identifier, string(req.Operation), sanitizedYAML) - h.CorrelationStore.Put(key, req.UserInfo.Username) - - // Increment webhook-specific correlation metric - roleAttr := h.getPodRoleAttribute(ctx) - telemetry.WebhookCorrelationsTotal.Add(ctx, 1, metric.WithAttributes(roleAttr)) - - log.V(1).Info("Stored correlation entry (unfiltered)", - "kind", obj.GetKind(), - "name", obj.GetName(), - "namespace", obj.GetNamespace(), - "username", req.UserInfo.Username, - "operation", req.Operation) -} - -// getPodRoleAttribute returns the role attribute for metrics based on pod labels. -func (h *EventHandler) getPodRoleAttribute(ctx context.Context) attribute.KeyValue { - podName := os.Getenv("POD_NAME") - podNamespace := os.Getenv("POD_NAMESPACE") - - if podName == "" || podNamespace == "" { - return attribute.String("role", "unknown") - } - - pod := &corev1.Pod{} - err := h.Client.Get(ctx, apitypes.NamespacedName{Name: podName, Namespace: podNamespace}, pod) - if err != nil { - return attribute.String("role", "unknown") - } - - // Check if pod has the leader label - if role, ok := pod.Labels["role"]; ok && role == "leader" { - return attribute.String("role", "leader") - } - - return attribute.String("role", "follower") -} - -// InjectDecoder injects the decoder. -func (h *EventHandler) InjectDecoder(d *admission.Decoder) error { - h.Decoder = d - log := logf.Log.WithName("webhook") - log.Info("Decoder successfully injected into EventHandler - ready to decode all Kubernetes resource types") - return nil -} diff --git a/internal/webhook/event_handler_test.go b/internal/webhook/event_handler_test.go deleted file mode 100644 index ceae3b481..000000000 --- a/internal/webhook/event_handler_test.go +++ /dev/null @@ -1,818 +0,0 @@ -//go:build legacy_crd -// +build legacy_crd - -/* -SPDX-License-Identifier: Apache-2.0 - -Copyright 2025 ConfigButler - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - admissionv1 "k8s.io/api/admission/v1" - authenticationv1 "k8s.io/api/authentication/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - configv1alpha1 "github.com/ConfigButler/gitops-reverser/api/v1alpha1" - "github.com/ConfigButler/gitops-reverser/internal/correlation" - "github.com/ConfigButler/gitops-reverser/internal/rulestore" - "github.com/ConfigButler/gitops-reverser/internal/telemetry" -) - -func TestEventHandler_Handle_MatchingRule(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule that matches Pods - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods"}, - }, - }, - }, - } - ruleStore.AddOrUpdate(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Pod - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - "spec": map[string]interface{}{ - "containers": []interface{}{ - map[string]interface{}{ - "name": "test-container", - "image": "nginx:latest", - }, - }, - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook allows the request - assert.True(t, response.Allowed) - assert.Equal(t, "request is allowed", response.Result.Message) - - // Verify correlation entry was stored (webhook's sole responsibility) - assert.Equal(t, 1, correlationStore.Size(), "correlation entry should be stored") - - // NOTE: Webhook does NOT enqueue events - that is the watch path's responsibility -} - -func TestEventHandler_Handle_NoMatchingRule(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule that matches Services (not Pods) - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"services"}, - }, - }, - }, - } - ruleStore.AddOrUpdate(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Pod (which doesn't match the Service rule) - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify - assert.True(t, response.Allowed) - assert.Equal(t, "request is allowed", response.Result.Message) - assert.Equal(t, 0, correlationStore.Size(), "no correlation entry for non-matching rule") -} - -func TestEventHandler_Handle_MultipleMatchingRules(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add multiple rules that match Pods - rule1 := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule-1", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "repo-config-1"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods"}, - }, - }, - }, - } - - rule2 := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule-2", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "repo-config-2"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods", "services"}, - }, - }, - }, - } - - ruleStore.AddOrUpdate(rule1) - ruleStore.AddOrUpdate(rule2) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Pod - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify - assert.True(t, response.Allowed) - - // Webhook stores ONE correlation entry per resource change - // (not one per matching rule - that's the watch path's job to create multiple events) - assert.Equal(t, 1, correlationStore.Size(), "single correlation entry for resource") -} - -func TestEventHandler_Handle_ExcludedByLabels(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule that excludes resources with ignore label using ObjectSelector - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - ObjectSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "configbutler.ai/ignore", - Operator: metav1.LabelSelectorOpDoesNotExist, - }, - }, - }, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods"}, - }, - }, - }, - } - ruleStore.AddOrUpdate(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Pod with ignore label - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "ignored-pod", - "namespace": "default", - "labels": map[string]interface{}{ - "configbutler.ai/ignore": "true", - }, - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook allows request (webhook doesn't filter) - assert.True(t, response.Allowed) - // Webhook stores correlation for ALL resources (watch path filters based on rules) - assert.Equal(t, 1, correlationStore.Size(), "correlation entry should be stored even if rule excludes it") -} - -func TestEventHandler_Handle_InvalidJSON(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request with invalid JSON - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Object: runtime.RawExtension{ - Raw: []byte(`{"invalid": json}`), // Invalid JSON - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify - assert.False(t, response.Allowed) - assert.Equal(t, int32(400), response.Result.Code) // Bad Request - assert.Equal(t, 0, correlationStore.Size()) // No correlation for invalid request -} - -func TestEventHandler_Handle_NamespacedIngressResource(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule matching Ingress resources (namespace-scoped) - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ingress-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"ingresses"}, - }, - }, - }, - } - ruleStore.AddOrUpdateWatchRule(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for an Ingress (namespace-scoped resource) - ingress := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "networking.k8s.io/v1", - "kind": "Ingress", - "metadata": map[string]interface{}{ - "name": "test-ingress", - "namespace": "default", - }, - "spec": map[string]interface{}{ - "rules": []interface{}{}, - }, - }, - } - ingress.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "networking.k8s.io", - Version: "v1", - Kind: "Ingress", - }) - - ingressBytes, err := ingress.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "networking.k8s.io", - Version: "v1", - Resource: "ingresses", - }, - Object: runtime.RawExtension{ - Raw: ingressBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook stored correlation - assert.True(t, response.Allowed) - assert.Equal(t, 1, correlationStore.Size(), "correlation entry should be stored") -} - -func TestEventHandler_Handle_DifferentOperations(t *testing.T) { - operations := []admissionv1.Operation{ - admissionv1.Create, - admissionv1.Update, - admissionv1.Delete, - } - - for _, operation := range operations { - t.Run(string(operation), func(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule that matches Pods - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods"}, - }, - }, - }, - } - ruleStore.AddOrUpdate(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: operation, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook stored correlation - assert.True(t, response.Allowed) - assert.Equal(t, 1, correlationStore.Size(), "correlation entry should be stored") - }) - } -} - -func TestEventHandler_Handle_ClusterScopedResource(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a WatchRule for namespaces (namespace-scoped rule watching cluster-scoped resource) - // Note: Webhook stores correlation for ALL resources regardless of rules - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "namespace-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "cluster-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"namespaces"}, - }, - }, - }, - } - ruleStore.AddOrUpdateWatchRule(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Namespace (cluster-scoped resource) - namespace := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": map[string]interface{}{ - "name": "test-namespace", - }, - }, - } - - namespaceBytes, err := namespace.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "namespaces", - }, - Object: runtime.RawExtension{ - Raw: namespaceBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "cluster-admin", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook stored correlation (webhook stores for ALL resources) - assert.True(t, response.Allowed) - assert.Equal(t, 1, correlationStore.Size(), "webhook stores correlation for all resources") -} - -func TestEventHandler_InjectDecoder(t *testing.T) { - handler := &EventHandler{} - - scheme := runtime.NewScheme() - decoder := admission.NewDecoder(scheme) - - err := handler.InjectDecoder(&decoder) - require.NoError(t, err) - assert.NotNil(t, handler.Decoder) - assert.Equal(t, &decoder, handler.Decoder) -} - -func TestEventHandler_Handle_SanitizationApplied(t *testing.T) { - // Setup - ctx := context.Background() - _, err := telemetry.InitOTLPExporter(ctx) - require.NoError(t, err) - - scheme := runtime.NewScheme() - client := fake.NewClientBuilder().WithScheme(scheme).Build() - - ruleStore := rulestore.NewStore() - correlationStore := correlation.NewStore(60*time.Second, 1000) - - // Add a rule that matches Pods - rule := configv1alpha1.WatchRule{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-rule", - Namespace: "default", - }, - Spec: configv1alpha1.WatchRuleSpec{ - GitRepoConfigRef: configv1alpha1.NamespacedName{Name: "test-repo-config"}, - Rules: []configv1alpha1.ResourceRule{ - { - Resources: []string{"pods"}, - }, - }, - }, - } - ruleStore.AddOrUpdate(rule) - - handler := &EventHandler{ - Client: client, - RuleStore: ruleStore, - CorrelationStore: correlationStore, - } - - // Create decoder - decoder := admission.NewDecoder(scheme) - handler.Decoder = &decoder - - // Create admission request for a Pod with status (should be sanitized) - pod := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]interface{}{ - "name": "test-pod", - "namespace": "default", - "creationTimestamp": "2025-01-01T00:00:00Z", // Should be removed - "uid": "12345", // Should be removed - }, - "spec": map[string]interface{}{ - "containers": []interface{}{ - map[string]interface{}{ - "name": "test-container", - "image": "nginx:latest", - }, - }, - }, - "status": map[string]interface{}{ // Should be removed - "phase": "Running", - }, - }, - } - - podBytes, err := pod.MarshalJSON() - require.NoError(t, err) - - req := admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - UID: "test-uid", - Operation: admissionv1.Create, - Resource: metav1.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - Object: runtime.RawExtension{ - Raw: podBytes, - }, - UserInfo: authenticationv1.UserInfo{ - Username: "test-user", - }, - }, - } - - // Execute - response := handler.Handle(ctx, req) - - // Verify webhook stored correlation (actual responsibility) - assert.True(t, response.Allowed) - assert.Equal(t, 1, correlationStore.Size(), "correlation entry should be stored") -} diff --git a/test/e2e/audit_redis_e2e_test.go b/test/e2e/audit_redis_e2e_test.go index 13aec6beb..f6ef42ef9 100644 --- a/test/e2e/audit_redis_e2e_test.go +++ b/test/e2e/audit_redis_e2e_test.go @@ -22,6 +22,8 @@ import ( "context" "fmt" "os" + "os/exec" + "path/filepath" "strings" "time" @@ -31,7 +33,8 @@ import ( ) const ( - defaultAuditRedisStream = "gitopsreverser.audit.events.v1" + defaultAuditRedisStream = "gitopsreverser.audit.events.v1" + defaultE2EValkeyPassword = "e2e-valkey-password" ) var _ = Describe("Audit Redis Queue", Label("audit-redis"), Ordered, func() { @@ -57,18 +60,15 @@ var _ = Describe("Audit Redis Queue", Label("audit-redis"), Ordered, func() { By("connecting to Valkey through the e2e port-forward") client := redis.NewClient(&redis.Options{ - Addr: valkeyPortForwardAddr(), + Addr: valkeyPortForwardAddr(), + Password: valkeyPortForwardPassword(), }) Eventually(func(g Gomega) { g.Expect(client.Ping(context.Background()).Err()).NotTo(HaveOccurred()) }, 30*time.Second, 2*time.Second).Should(Succeed()) - By("clearing the dedicated stream for deterministic assertions") - _, err := client.Del(context.Background(), streamName).Result() - Expect(err).NotTo(HaveOccurred()) - By("triggering kube-apiserver audit events with a ConfigMap create/update") - _, err = kubectlRunInNamespace( + _, err := kubectlRunInNamespace( testNs, "delete", "configmap", @@ -128,6 +128,146 @@ var _ = Describe("Audit Redis Queue", Label("audit-redis"), Ordered, func() { }) }) +// auditConsumerDescribe holds the consumer-path e2e test. +// Label: audit-redis (runs together with the producer test via make test-e2e-audit-redis). +var _ = Describe("Audit Redis Consumer", Label("audit-redis"), Ordered, func() { + var ( + testNs string + valkeyClient *redis.Client + gitProvName string + gitTargetName string + watchRuleName string + cmName string + gitCheckout string // local git checkout path; read from E2E_CHECKOUT_DIR env + ) + + BeforeAll(func() { + By("resolving git checkout path from E2E_CHECKOUT_DIR") + gitCheckout = strings.TrimSpace(os.Getenv("E2E_CHECKOUT_DIR")) + Expect(gitCheckout).NotTo(BeEmpty(), + "E2E_CHECKOUT_DIR must be set (run via make test-e2e or make test-e2e-audit-redis)") + + By("creating test namespace and applying git secrets") + testNs = testNamespaceFor("audit-consumer") + _, _ = kubectlRun("create", "namespace", testNs) + secretsYaml := strings.TrimSpace(os.Getenv("E2E_SECRETS_YAML")) + Expect(secretsYaml).NotTo(BeEmpty(), "E2E_SECRETS_YAML must be set by BeforeSuite") + _, err := kubectlRunInNamespace(testNs, "apply", "-f", secretsYaml) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + By("connecting to Valkey through the e2e port-forward") + valkeyClient = redis.NewClient(&redis.Options{ + Addr: valkeyPortForwardAddr(), + Password: valkeyPortForwardPassword(), + }) + Eventually(func(g Gomega) { + g.Expect(valkeyClient.Ping(context.Background()).Err()).NotTo(HaveOccurred()) + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("setting up GitProvider, GitTarget and WatchRule for consumer test") + seed := GinkgoRandomSeed() + gitProvName = fmt.Sprintf("audit-consumer-gitprovider-%d", seed) + gitTargetName = fmt.Sprintf("audit-consumer-gittarget-%d", seed) + watchRuleName = fmt.Sprintf("audit-consumer-watchrule-%d", seed) + cmName = fmt.Sprintf("audit-consumer-cm-%d", seed) + + repoURL := fmt.Sprintf(giteaRepoURLTemplate, strings.TrimSpace(os.Getenv("E2E_REPO_NAME"))) + Expect(repoURL).NotTo(ContainSubstring("%!(EXTRA"), "E2E_REPO_NAME must be set") + + gitSecretName := strings.TrimSpace(os.Getenv("E2E_GIT_SECRET_HTTP")) + if gitSecretName == "" { + gitSecretName = "git-creds" + } + + createGitProviderWithURLInNamespace(gitProvName, testNs, gitSecretName, repoURL) + verifyResourceStatus("gitprovider", gitProvName, testNs, "True", "Ready", "") + + createGitTarget(gitTargetName, testNs, gitProvName, "e2e/audit-consumer-test", "main") + verifyResourceStatus("gittarget", gitTargetName, testNs, "True", "Ready", "") + + data := struct { + Name string + Namespace string + DestinationName string + }{ + Name: watchRuleName, + Namespace: testNs, + DestinationName: gitTargetName, + } + err = applyFromTemplate("test/e2e/templates/manager/watchrule-configmap.tmpl", data, testNs) + Expect(err).NotTo(HaveOccurred(), "failed to apply WatchRule") + verifyResourceStatus("watchrule", watchRuleName, testNs, "True", "Ready", "") + }) + + AfterAll(func() { + cleanupWatchRule(watchRuleName, testNs) + cleanupGitTarget(gitTargetName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", gitProvName, "--ignore-not-found=true") + cleanupNamespace(testNs) + if valkeyClient != nil { + _ = valkeyClient.Close() + } + }) + + It("should produce a Git commit from an audit stream event consumed by the consumer", func() { + streamName := defaultAuditRedisStream + + By("creating a ConfigMap to produce an audit event") + cmData := struct { + Name string + Namespace string + }{Name: cmName, Namespace: testNs} + err := applyFromTemplate("test/e2e/templates/manager/configmap.tmpl", cmData, testNs) + Expect(err).NotTo(HaveOccurred(), "failed to create test ConfigMap") + + By("waiting for the audit stream entry to appear (producer path)") + // cmName includes GinkgoRandomSeed so it uniquely identifies this test run's entry. + Eventually(func(g Gomega) { + entries, readErr := valkeyClient.XRange(context.Background(), streamName, "-", "+").Result() + g.Expect(readErr).NotTo(HaveOccurred()) + found := false + for _, entry := range entries { + resource, _ := entry.Values["resource"].(string) + name, _ := entry.Values["name"].(string) + if resource == "configmaps" && name == cmName { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "audit stream entry for ConfigMap should appear") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("waiting for the Git commit to appear (consumer path)") + expectedFile := filepath.Join(gitCheckout, + "e2e/audit-consumer-test", + fmt.Sprintf("v1/configmaps/%s/%s.yaml", testNs, cmName)) + + Eventually(func(g Gomega) { + pullCmd := exec.Command("git", "pull") + pullCmd.Dir = gitCheckout + pullOut, pullErr := pullCmd.CombinedOutput() + g.Expect(pullErr).NotTo(HaveOccurred(), + fmt.Sprintf("git pull failed: %s", string(pullOut))) + + info, statErr := os.Stat(expectedFile) + g.Expect(statErr).NotTo(HaveOccurred(), + fmt.Sprintf("ConfigMap file should exist at %s", expectedFile)) + g.Expect(info.Size()).To(BeNumerically(">", 0)) + + authorCmd := exec.Command("git", "log", "-1", "--pretty=%an") + authorCmd.Dir = gitCheckout + authorOut, authorErr := authorCmd.CombinedOutput() + g.Expect(authorErr).NotTo(HaveOccurred(), + fmt.Sprintf("git log author failed: %s", string(authorOut))) + g.Expect(string(authorOut)).To(ContainSubstring("system:admin")) + }, 3*time.Minute, 3*time.Second).Should(Succeed()) + + By("cleaning up test ConfigMap") + _, _ = kubectlRunInNamespace(testNs, "delete", "configmap", cmName, "--ignore-not-found=true") + }) +}) + func valkeyPortForwardAddr() string { port := strings.TrimSpace(os.Getenv("VALKEY_PORT")) if port == "" { @@ -135,3 +275,11 @@ func valkeyPortForwardAddr() string { } return "127.0.0.1:" + port } + +func valkeyPortForwardPassword() string { + password := strings.TrimSpace(os.Getenv("E2E_VALKEY_PASSWORD")) + if password == "" { + return defaultE2EValkeyPassword + } + return password +} diff --git a/test/e2e/cluster/README.md b/test/e2e/cluster/README.md index 575465f4e..84c96e2ae 100644 --- a/test/e2e/cluster/README.md +++ b/test/e2e/cluster/README.md @@ -8,11 +8,12 @@ By default, the cluster bootstrap also disables packaged k3s Traefik and k3s Ser - [`start-cluster.sh`](start-cluster.sh): creates/reuses a k3d cluster and configures kubeconfig for devcontainer usage - [`audit/policy.yaml`](audit/policy.yaml): Kubernetes audit policy used by kube-apiserver -- [`audit/webhook-config.yaml`](audit/webhook-config.yaml): webhook target used by kube-apiserver audit backend +- [`audit/webhook-config.yaml`](audit/webhook-config.yaml): tracked bootstrap source for the mounted audit webhook kubeconfig ## How It Works -`start-cluster.sh` mounts the local audit directory into the k3d server node and sets: +The Makefile first copies the tracked audit assets into `.stamps/cluster//audit/`. +`start-cluster.sh` mounts that generated directory into the k3d server node and sets: - `audit-policy-file=/etc/kubernetes/audit/policy.yaml` - `audit-webhook-config-file=/etc/kubernetes/audit/webhook-config.yaml` @@ -26,13 +27,14 @@ It also disables these packaged k3s components by default: - `traefik` - `servicelb` -The webhook URL in [`audit/webhook-config.yaml`](audit/webhook-config.yaml) targets: +The bootstrap source in [`audit/webhook-config.yaml`](audit/webhook-config.yaml) targets: `https://10.43.200.200:9444/audit-webhook/kind-e2e` Notes: - `insecure-skip-tls-verify: true` is intentional for local e2e - `kind-e2e` in the path is the cluster ID label used by tests/metrics +- after install, `hack/e2e/inject-webhook-tls.sh` replaces the mounted `.stamps` copy with the final CA-trusting mTLS kubeconfig ## Usage diff --git a/test/e2e/cluster/audit/policy.yaml b/test/e2e/cluster/audit/policy.yaml index 07fb14e4b..53ae62946 100644 --- a/test/e2e/cluster/audit/policy.yaml +++ b/test/e2e/cluster/audit/policy.yaml @@ -10,7 +10,7 @@ rules: resources: # CORE: Drop runtime noise - group: "" - resources: ["events", "endpoints", "nodes", "pods", "secrets", "bindings", "componentstatuses", "*/status"] + resources: ["events", "endpoints", "nodes", "pods", "bindings", "componentstatuses", "*/status"] # AUTH: Drop ephemeral checks - group: "authentication.k8s.io" diff --git a/test/e2e/cluster/audit/webhook-config.yaml b/test/e2e/cluster/audit/webhook-config.yaml index ac9f361d0..f429ccae6 100644 --- a/test/e2e/cluster/audit/webhook-config.yaml +++ b/test/e2e/cluster/audit/webhook-config.yaml @@ -1,16 +1,15 @@ -# Kubernetes audit webhook configuration for k3d cluster -# This configures the kube-apiserver to send audit events to the gitops-reverser webhook +# Kubernetes audit webhook bootstrap configuration for k3d e2e. +# This tracked source file is copied into .stamps/cluster//audit/ before +# cluster creation. The mounted .stamps copy is what kube-apiserver reads, and +# hack/e2e/inject-webhook-tls.sh overwrites that generated copy after +# cert-manager issues the final audit TLS materials. apiVersion: v1 kind: Config preferences: {} clusters: - name: audit-webhook cluster: - # IMPORTANT: Use fixed ClusterIP instead of DNS name - # kube-apiserver starts before CoreDNS, so DNS resolution fails at startup - # The ClusterIP is set in config/service.yaml - server: https://10.43.200.200:9444/audit-webhook/kind-e2e - # Skip TLS verification for testing (webhook uses self-signed cert from cert-manager) + server: https://127.0.0.1:30444/audit-webhook/kind-e2e insecure-skip-tls-verify: true contexts: - name: webhook diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index 4e75cd6cb..f7f72c002 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -10,7 +10,7 @@ DISABLE_K3S_TRAEFIK="${DISABLE_K3S_TRAEFIK:-true}" DISABLE_K3S_SERVICELB="${DISABLE_K3S_SERVICELB:-true}" KUBE_APISERVER_MAX_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_REQUESTS_INFLIGHT:-800}" KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT="${KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT:-400}" -AUDIT_DIR_REL="test/e2e/cluster/audit" +AUDIT_DIR_REL="${AUDIT_DIR_REL:-test/e2e/cluster/audit}" K3D_CREATE_LOG_FILE="${TMPDIR:-/tmp}/k3d-create-${CLUSTER_NAME}.log" REPO_PWD="$(pwd -P)" @@ -61,6 +61,20 @@ cluster_exists() { k3d cluster list -o json 2>/dev/null | grep -q "\"name\":\"${CLUSTER_NAME}\"" } +cluster_context_name() { + echo "k3d-${CLUSTER_NAME}" +} + +cluster_is_healthy() { + local context_name + context_name="$(cluster_context_name)" + + if ! kubectl config get-contexts "${context_name}" >/dev/null 2>&1; then + return 1 + fi + + kubectl --context "${context_name}" --request-timeout=10s get ns >/dev/null 2>&1 +} ensure_k3d_stat_compat_path() { local host_project_path="$1" local can_sudo=false @@ -216,12 +230,13 @@ main() { # k3d writes/merges kubeconfig into the default location; ensure it exists so the context is usable. mkdir -p "${HOME}/.kube" + export HOST_PROJECT_PATH + HOST_PROJECT_PATH="$(resolve_host_project_path)" + ensure_k3d_stat_compat_path "${HOST_PROJECT_PATH}" + if cluster_exists; then - echo "♻️ Reusing existing k3d cluster '${CLUSTER_NAME}' (no delete/recreate)" + echo "♻️ Reusing existing k3d cluster '${CLUSTER_NAME}'" else - export HOST_PROJECT_PATH - HOST_PROJECT_PATH="$(resolve_host_project_path)" - ensure_k3d_stat_compat_path "${HOST_PROJECT_PATH}" create_cluster "${HOST_PROJECT_PATH}" fi @@ -229,6 +244,20 @@ main() { merge_kubeconfig rewrite_kubeconfig_for_devcontainer + if ! cluster_is_healthy; then + echo "❌ Existing k3d cluster '${CLUSTER_NAME}' is unhealthy." >&2 + echo "Refusing to auto-recreate it because cluster-scoped stamps may now be inconsistent." >&2 + echo "" >&2 + echo "Cleanup and retry:" >&2 + echo " make clean-cluster" >&2 + echo "or:" >&2 + echo " k3d cluster delete ${CLUSTER_NAME}" >&2 + echo " rm -rf .stamps/cluster/$(cluster_context_name)" >&2 + exit 1 + fi + + kubectl --context "$(cluster_context_name)" --request-timeout=30s get ns >/dev/null + echo "✅ Cluster setup complete!" echo "Try:" echo " kubectl get nodes" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ffa2776aa..6968b2244 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -163,11 +163,14 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyControllerUp).Should(Succeed()) }) - It("should route webhook traffic to the running controller pod", func() { - By("verifying controller service selects the running controller pod") - verifyWebhookService := func(g Gomega) { - // Get controller service endpoints - output, err := kubectlRunInNamespace( + It("should expose the admission and metrics service", func() { + By("verifying controller service exists") + _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) + Expect(err).NotTo(HaveOccurred(), "Controller service should exist") + + By("verifying controller service routes to the controller pod") + Eventually(func(g Gomega) { + output, endpointsErr := kubectlRunInNamespace( namespace, "get", "endpoints", @@ -175,13 +178,11 @@ var _ = Describe("Manager", Ordered, func() { "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}", ) - g.Expect(err).NotTo(HaveOccurred(), "Failed to get controller service endpoints") + g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get controller service endpoints") - // Filter out kubectl deprecation warnings from output lines := utils.GetNonEmptyLines(output) podSet := map[string]struct{}{} for _, line := range lines { - // Skip warning lines if !strings.HasPrefix(line, "Warning:") && !strings.Contains(line, "deprecated") && strings.Contains(line, "gitops-reverser") { @@ -193,64 +194,42 @@ var _ = Describe("Manager", Ordered, func() { podNames = append(podNames, podName) } - // Should only have one endpoint in single-pod mode. g.Expect(podNames).To(HaveLen(1), "controller service should route to exactly 1 pod") g.Expect(podNames[0]).To(Equal(controllerPodName), "controller service should route to controller pod") - - By(fmt.Sprintf("✅ Controller service correctly routes to pod: %s", controllerPodName)) - } - Eventually(verifyWebhookService, 30*time.Second).Should(Succeed()) + }, 30*time.Second).Should(Succeed()) }) - It("should expose admission and audit ports on one controller service", func() { - By("verifying controller service exists") - _, err := kubectlRunInNamespace(namespace, "get", "svc", controllerServiceName) - Expect(err).NotTo(HaveOccurred(), "Controller service should exist") + It("should expose the audit service separately", func() { + By("verifying audit service exists") + _, err := kubectlRunInNamespace(namespace, "get", "svc", auditServiceName) + Expect(err).NotTo(HaveOccurred(), "Audit service should exist") - By("verifying controller service routes to the controller pod") + By("verifying audit service routes to the controller pod") Eventually(func(g Gomega) { output, endpointsErr := kubectlRunInNamespace( namespace, "get", "endpoints", - controllerServiceName, + auditServiceName, "-o", "jsonpath={.subsets[*].addresses[*].targetRef.name}", ) - g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get controller service endpoints") + g.Expect(endpointsErr).NotTo(HaveOccurred(), "Failed to get audit service endpoints") lines := utils.GetNonEmptyLines(output) - podSet := map[string]struct{}{} + var podNames []string for _, line := range lines { - if !strings.HasPrefix(line, "Warning:") && - !strings.Contains(line, "deprecated") && - strings.Contains(line, "gitops-reverser") { - podSet[line] = struct{}{} + if strings.HasPrefix(line, "Warning:") || strings.Contains(line, "deprecated") { + continue } - } - var podNames []string - for podName := range podSet { - podNames = append(podNames, podName) + podNames = append(podNames, line) } - g.Expect(podNames).To(HaveLen(1), "controller service should route to exactly 1 pod") - g.Expect(podNames[0]).To(Equal(controllerPodName), "controller service should route to controller pod") + g.Expect(podNames).To(HaveLen(1), "audit service should route to exactly 1 pod") + g.Expect(podNames[0]).To(Equal(controllerPodName), "audit service should route to controller pod") }, 30*time.Second).Should(Succeed()) }) - It("should have webhook registration configured", func() { - By("verifying webhook registration for event handler") - verifyWebhook := func(g Gomega) { - jsonPath := "jsonpath={.items[?(@.metadata.name=='gitops-reverser-validating-webhook-configuration')]" + - ".webhooks[*].name}" - output, err := kubectlRun("get", "validatingwebhookconfigurations", "-o", jsonPath) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("gitops-reverser.configbutler.ai"), - "Event webhook should be registered") - } - Eventually(verifyWebhook).Should(Succeed()) - }) - It("should ensure the metrics endpoint is serving metrics", func() { By("validating that the controller service is available for metrics") _, err := kubectlRunInNamespace(namespace, "get", "service", controllerServiceName) @@ -299,46 +278,6 @@ var _ = Describe("Manager", Ordered, func() { fmt.Printf("📊 Inspect metrics: %s\n", getPrometheusURL()) }) - It("should receive webhook calls and process them successfully", func() { - By("recording baseline webhook event count") - baselineEvents, err := queryPrometheus("sum(gitopsreverser_events_received_total) or vector(0)") - Expect(err).NotTo(HaveOccurred()) - fmt.Printf("📊 Baseline events: %.0f\n", baselineEvents) - - By("creating a ConfigMap to trigger webhook call") - _, err = kubectlRunInNamespace( - namespace, - "create", - "configmap", - "webhook-test-cm", - "--from-literal=test=webhook", - ) - Expect(err).NotTo(HaveOccurred(), "ConfigMap creation should succeed with working webhook") - - By("waiting for webhook event metric to increment") - waitForMetric("sum(gitopsreverser_events_received_total) or vector(0)", - func(v float64) bool { return v > baselineEvents }, - "webhook events should increment") - - By("verifying webhook events were received") - currentEvents, err := queryPrometheus("sum(gitopsreverser_events_received_total) or vector(0)") - Expect(err).NotTo(HaveOccurred()) - Expect(currentEvents).To(BeNumerically(">", baselineEvents), "Controller should process webhook events") - fmt.Printf("✅ Controller processed %.0f events\n", currentEvents-baselineEvents) - - fmt.Printf("✅ Webhook routing validated\n") - fmt.Printf("📊 Inspect metrics: %s\n", getPrometheusURL()) - - By("cleaning up webhook test resources") - _, _ = kubectlRunInNamespace( - namespace, - "delete", - "configmap", - "webhook-test-cm", - "--ignore-not-found=true", - ) - }) - It("should receive audit webhook events from kube-apiserver", func() { By("recording baseline audit event count") baselineAuditEvents, err := queryPrometheus("sum(gitopsreverser_audit_events_received_total) or vector(0)") @@ -1599,6 +1538,20 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyFileDeleted, "60s", "1s").Should(Succeed()) + By("verifying the deleted CRD file does not reappear after terminating updates") + Consistently(func(g Gomega) { + pullCmd := exec.Command("git", "pull") + pullCmd.Dir = checkoutDir + _, _ = pullCmd.CombinedOutput() + + expectedFile := filepath.Join(checkoutDir, + "e2e/crd-delete-test", + "apiextensions.k8s.io/v1/customresourcedefinitions/icecreamorders.shop.example.com.yaml") + _, statErr := os.Stat(expectedFile) + g.Expect(statErr).To(HaveOccurred(), "CRD file must stay deleted after CRD termination updates") + g.Expect(os.IsNotExist(statErr)).To(BeTrue(), "CRD file must not reappear in Git") + }, "15s", "1s").Should(Succeed()) + By("cleaning up test resources") cleanupClusterWatchRule(clusterWatchRuleName) cleanupGitTarget(destName, testNs) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 264e05b58..fb62fabc0 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -46,8 +46,8 @@ var namespace = resolveE2ENamespace() //nolint:gochecknoglobals // used across e const metricWaitDefaultTimeout = 30 * time.Second const e2eEncryptionRefName = "sops-age-key" -// controllerServiceName is the single Service name used by the controller. const controllerServiceName = "gitops-reverser-service" +const auditServiceName = "gitops-reverser-audit" const controllerPodLabelSelector = "control-plane=gitops-reverser" // promAPI is the Prometheus API client instance diff --git a/test/e2e/setup/flux/kustomization.yaml b/test/e2e/setup/flux/kustomization.yaml index 826c93f1f..fae9a67bb 100644 --- a/test/e2e/setup/flux/kustomization.yaml +++ b/test/e2e/setup/flux/kustomization.yaml @@ -32,3 +32,11 @@ configMapGenerator: namespace: valkey-e2e files: - values.yaml=values/valkey-values.yaml + +secretGenerator: + - name: valkey-auth + namespace: valkey-e2e + literals: + - password=e2e-valkey-password + options: + disableNameSuffixHash: true diff --git a/test/e2e/setup/flux/values/valkey-values.yaml b/test/e2e/setup/flux/values/valkey-values.yaml index 461a49408..8744f9437 100644 --- a/test/e2e/setup/flux/values/valkey-values.yaml +++ b/test/e2e/setup/flux/values/valkey-values.yaml @@ -1,5 +1,5 @@ -# Valkey Helm values for e2e testing -# Standalone mode, no persistence, no auth, minimal resources +# Valkey Helm values for e2e testing. +# Standalone mode, no persistence, minimal resources, auth enabled. architecture: standalone @@ -13,6 +13,12 @@ primary: requests: memory: "128Mi" -# No authentication needed for e2e testing +# Auth enabled. Password from pre-created valkey-auth secret. +# The fixed password is intentional for test environments — not a security concern. auth: - enabled: false + enabled: true + usersExistingSecret: valkey-auth + aclUsers: + default: + permissions: "~* &* +@all" + passwordKey: password diff --git a/test/e2e/suite_state.go b/test/e2e/suite_state.go index 28c1649fc..c8e6e623e 100644 --- a/test/e2e/suite_state.go +++ b/test/e2e/suite_state.go @@ -99,6 +99,7 @@ func e2ePreservationSummary() string { } func initE2ECommandContext() { + //nolint:gosec // cancel is stored for suite-lifetime interrupt handling e2eExecutionContext, cancel := context.WithCancel(context.Background()) e2eCommandCancel = cancel setE2ECommandContext(e2eExecutionContext)