Taskfile.e2e.yml

# yaml-language-server: $schema=https://taskfile.dev/schema.json
version: "3"

vars:
  CTX: '{{.CTX | default "k3d-audit-pass-through-e2e"}}'
  CLUSTER_NAME:
    sh: |
      ctx='{{.CTX}}'
      ctx="${ctx#k3d-}"
      ctx="${ctx#kind-}"
      printf '%s' "${ctx}"
  DOCKER: '{{.DOCKER | default "docker"}}'
  FLUX: '{{.FLUX | default "flux"}}'
  HELM: '{{.HELM | default "helm"}}'
  K3D: '{{.K3D | default "k3d"}}'
  KUBECTL: '{{.KUBECTL | default "kubectl"}}'
  E2E_PROXY_IMAGE: '{{.E2E_PROXY_IMAGE | default "apiservice-audit-proxy:e2e-local"}}'
  E2E_PROXY_IMAGE_REPOSITORY:
    sh: |
      ref='{{.E2E_PROXY_IMAGE}}'
      repo="${ref%:*}"
      tag="${ref##*:}"
      if [ "${repo}" = "${tag}" ]; then
        printf '%s' "${ref}"
      else
        printf '%s' "${repo}"
      fi
  E2E_PROXY_IMAGE_TAG:
    sh: |
      ref='{{.E2E_PROXY_IMAGE}}'
      repo="${ref%:*}"
      tag="${ref##*:}"
      if [ "${repo}" = "${tag}" ]; then
        printf 'latest'
      else
        printf '%s' "${tag}"
      fi
  E2E_PROXY_NAMESPACE: '{{.E2E_PROXY_NAMESPACE | default "wardle"}}'
  E2E_RELEASE_NAME: '{{.E2E_RELEASE_NAME | default "apiservice-audit-proxy"}}'
  E2E_BACKEND_DEPLOYMENT: '{{.E2E_BACKEND_DEPLOYMENT | default "api-backend"}}'
  E2E_PROXY_VALUES_FILE: '{{.E2E_PROXY_VALUES_FILE | default "test/e2e/values/proxy-cert-manager.yaml"}}'
  FLUX_NAMESPACE: '{{.FLUX_NAMESPACE | default "flux-system"}}'
  FLUX_WAIT_TIMEOUT: '{{.FLUX_WAIT_TIMEOUT | default "900s"}}'
  STAMPS_DIR: .stamps/e2e/{{.CLUSTER_NAME}}
  AUDIT_ASSET_SOURCE_DIR: test/e2e/cluster/audit
  AUDIT_ASSET_DIR: '{{.STAMPS_DIR}}/audit'
  AUDIT_POLICY_SOURCE: '{{.AUDIT_ASSET_SOURCE_DIR}}/policy.yaml'
  AUDIT_WEBHOOK_BOOTSTRAP_SOURCE: '{{.AUDIT_ASSET_SOURCE_DIR}}/webhook-config.yaml'
  AUDIT_POLICY_PATH: '{{.AUDIT_ASSET_DIR}}/policy.yaml'
  AUDIT_WEBHOOK_CONFIG_PATH: '{{.AUDIT_ASSET_DIR}}/webhook-config.yaml'
  WEBHOOK_TESTER_LOCAL_PORT: '{{.WEBHOOK_TESTER_LOCAL_PORT | default "18090"}}'
  WEBHOOK_TESTER_SERVICE_PORT: '{{.WEBHOOK_TESTER_SERVICE_PORT | default "8080"}}'
  WEBHOOK_TESTER_BASE_URL: 'http://127.0.0.1:{{.WEBHOOK_TESTER_LOCAL_PORT}}'

tasks:
  e2e:doctor:
    desc: Verify the local machine can run the standalone e2e flow
    cmds:
      - bash hack/e2e/check-docker.sh
      - "{{.K3D}} version"
      - "{{.FLUX}} --version"
      - "{{.HELM}} version --short"
      - "{{.KUBECTL}} version --client=true"

  e2e:cluster-up:
    desc: Create or reuse the local k3d cluster for this project
    cmds:
      - task: e2e:_cluster-ready

  e2e:flux-bootstrap:
    desc: Install Flux and the shared add-ons used by the standalone prototype
    cmds:
      - task: e2e:_services-ready

  e2e:build-image:
    desc: Build the proxy image used for local e2e (no-op when E2E_PROXY_IMAGE is externally provided)
    status:
      - '[ "{{.E2E_PROXY_IMAGE}}" != "apiservice-audit-proxy:e2e-local" ]'
    cmds:
      - task: e2e:_proxy-image-built

  e2e:load-image:
    desc: Load the proxy image into the k3d cluster
    cmds:
      - task: e2e:build-image
      - task: e2e:_proxy-image-loaded

  e2e:deploy-proxy:
    desc: Install the sample backend, pass-through proxy, and webhook-tester with cert-manager-backed TLS
    cmds:
      - task: e2e:_proxy-with-webhook-tester-installed

  e2e:prepare:
    desc: Build, load, and install the full standalone smoke environment
    cmds:
      - task: e2e:deploy-proxy

  e2e:test-taskfile:
    desc: Validate Taskfile build behavior (no cluster required; needs Docker)
    cmds:
      - go test -tags=e2e -run 'TestBuildImageTask' -count=1 -v ./test/e2e

  e2e:test-smoke:
    desc: Run the standalone k3d smoke scenario against the aggregated API path
    deps:
      - e2e:portforward-webhook-tester
    cmds:
      - "CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} go test -tags=e2e -run 'TestSmoke' -count=1 -v ./test/e2e"

  e2e:test-smoke-backend-ca:
    desc: Run the smoke scenario with backend TLS validated by an explicit CA bundle
    cmds:
      # E2E_PROXY_VALUES_FILE must be exported into a fresh task process: a
      # `vars:` override on a dep does not propagate to that dep's own nested
      # dependencies, so the deploy step would otherwise use the default values.
      - E2E_PROXY_VALUES_FILE=test/e2e/values/proxy-cert-manager-backend-ca.yaml task e2e:portforward-webhook-tester
      - "CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} go test -tags=e2e -run 'TestSmoke' -count=1 -v ./test/e2e"

  e2e:test-watch-streams:
    desc: Prove long aggregated API watches survive beyond the proxy write-timeout regression window
    deps:
      - e2e:portforward-webhook-tester
    cmds:
      - "CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} go test -tags=e2e -run 'TestWatchStaysOpenThroughProxy' -count=1 -v ./test/e2e"

  e2e:test-metrics-scrape:
    desc: Prove /metrics is scrapeable in-cluster and Kubernetes-aware stream classification is correct (list=false, watch=true)
    deps:
      - e2e:_proxy-with-webhook-tester-installed
    cmds:
      - "CTX={{.CTX}} E2E_PROXY_NAMESPACE={{.E2E_PROXY_NAMESPACE}} E2E_RELEASE_NAME={{.E2E_RELEASE_NAME}} go test -tags=e2e -run 'TestProxyMetricsScrapeAfterWatch' -count=1 -v ./test/e2e"

  e2e:test-impersonation:
    desc: |
      Run the impersonation-mode scenarios (audited write, non-audited
      passthrough, apiserver-injected extras) with operator-supplied RBAC.
    cmds:
      # E2E_PROXY_VALUES_FILE must be exported into a fresh task process: a
      # `vars:` override on a dep does not propagate to that dep's own nested
      # dependencies, so the deploy step would otherwise use the default values.
      - E2E_PROXY_VALUES_FILE=test/e2e/values/proxy-impersonation.yaml task e2e:portforward-webhook-tester
      - |
        CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} \
          go test -tags=e2e \
          -run 'TestImpersonationWrite|TestImpersonationPassthrough|TestImpersonationApiserverExtras' \
          -count=1 -v ./test/e2e

  e2e:test-impersonation-no-rbac:
    desc: |
      Run the impersonation-mode missing-RBAC scenario: a write must fail with a
      clean 403 cannot-impersonate error proxied from the backend.
    cmds:
      - E2E_PROXY_VALUES_FILE=test/e2e/values/proxy-impersonation-no-rbac.yaml task e2e:portforward-webhook-tester
      - |
        CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} \
          go test -tags=e2e -run 'TestImpersonationRBACMissing' -count=1 -v ./test/e2e

  e2e:test-requestheader-trust:
    desc: |
      Verify inbound requestheader trust is sourced from the cluster ConfigMap:
      no CA Secret is installed, an audited write succeeds, and deleting the
      kube-system auth-reader RoleBinding fails the proxy's startup.
    deps:
      - e2e:portforward-webhook-tester
    cmds:
      - "CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} go test -tags=e2e -run 'TestRequestHeaderTrustFromCluster' -count=1 -v ./test/e2e"

  e2e:test-image-refresh:
    desc: Validate image load → pod digest chain (cluster required; runs after smoke)
    cmds:
      - "CTX={{.CTX}} E2E_PROXY_NAMESPACE={{.E2E_PROXY_NAMESPACE}} go test -tags=e2e -run 'TestImageRefresh' -count=1 -v ./test/e2e"

  e2e:test-smoke-full:
    desc: Run the full e2e suite in order (smoke first, then image-refresh)
    cmds:
      - task: e2e:test-smoke
      - task: e2e:test-image-refresh

  e2e:test-all:
    desc: |
      Run every e2e suite this repository ships, in the order CI expects.
      One target so CI can call a single command; if you want a faster local
      iteration, invoke a sub-target directly (e.g. e2e:test-watch-streams).
    cmds:
      - task: e2e:test-smoke-full
      - task: e2e:test-watch-streams
      - task: e2e:test-metrics-scrape
      - task: e2e:test-smoke-backend-ca
      - task: e2e:test-audit-gap
      - task: e2e:test-delete-collection
      - task: e2e:test-requestheader-trust
      - task: e2e:test-impersonation
      - task: e2e:test-impersonation-no-rbac

  e2e:deploy-with-webhook-tester:
    desc: Deploy the Helm-managed sample backend, proxy, and webhook-tester
    cmds:
      - task: e2e:_proxy-with-webhook-tester-installed

  e2e:portforward-webhook-tester:
    desc: Ensure a local port-forward to webhook-tester is running
    deps:
      - e2e:deploy-with-webhook-tester
    status:
      - 'curl -fsS {{.WEBHOOK_TESTER_BASE_URL}}/healthz >/dev/null 2>&1'
    cmds:
      - |
        export CTX="{{.CTX}}"
        export KUBECTL="{{.KUBECTL}}"
        export STAMPS_DIR="{{.STAMPS_DIR}}"
        export WEBHOOK_TESTER_NAMESPACE="{{.E2E_PROXY_NAMESPACE}}"
        export E2E_RELEASE_NAME="{{.E2E_RELEASE_NAME}}"
        export WEBHOOK_TESTER_LOCAL_PORT="{{.WEBHOOK_TESTER_LOCAL_PORT}}"
        export WEBHOOK_TESTER_SERVICE_PORT="{{.WEBHOOK_TESTER_SERVICE_PORT}}"
        bash hack/e2e/ensure-webhook-tester-port-forward.sh

  e2e:portforward-stop:
    desc: Stop the local webhook-tester port-forward
    cmds:
      - |
        pid_file="{{.STAMPS_DIR}}/webhook-tester-{{.WEBHOOK_TESTER_LOCAL_PORT}}.pid"
        if [ -f "${pid_file}" ]; then
          pid="$(tr -d '\n\r' < "${pid_file}" || true)"
          if [ -n "${pid}" ]; then
            kill "${pid}" 2>/dev/null || true
          fi
          rm -f "${pid_file}"
        fi
        pkill -f "kubectl.*port-forward.*svc/{{.E2E_RELEASE_NAME}}-test-webhook-receiver.*{{.WEBHOOK_TESTER_LOCAL_PORT}}:{{.WEBHOOK_TESTER_SERVICE_PORT}}" 2>/dev/null || true

  e2e:test-audit-gap:
    desc: |
      Assert the audit gap for aggregated API requests.
      Lane A (kube-apiserver native) must have a hollow event; Lane B (proxy) must have the full event.
      Requires: cluster created with audit webhook baked in + webhookTester.enabled=true.
    deps:
      - e2e:portforward-webhook-tester
    cmds:
      - |
        CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} \
          go test -tags=e2e -run 'TestAggregatedAPIAuditGap' -count=1 -v ./test/e2e

  e2e:test-delete-collection:
    desc: |
      Assert the proxy emits a complete audit event for a collection delete
      (the deletecollection verb) of an aggregated resource.
      Requires: cluster created with audit webhook baked in + webhookTester.enabled=true.
    deps:
      - e2e:portforward-webhook-tester
    cmds:
      - |
        CTX={{.CTX}} WEBHOOK_TESTER_BASE_URL={{.WEBHOOK_TESTER_BASE_URL}} \
          go test -tags=e2e -run 'TestAggregatedAPIDeleteCollectionAudit' -count=1 -v ./test/e2e

  e2e:cluster-down:
    desc: Delete the standalone k3d cluster
    cmds:
      - "{{.K3D}} cluster delete {{.CLUSTER_NAME}}"
      - rm -rf {{.STAMPS_DIR}}

  e2e:_cluster-ready:
    method: timestamp
    sources:
      - test/e2e/cluster/start-cluster.sh
      - '{{.AUDIT_POLICY_SOURCE}}'
      - '{{.AUDIT_WEBHOOK_BOOTSTRAP_SOURCE}}'
    generates:
      - '{{.STAMPS_DIR}}/ready'
      - '{{.AUDIT_POLICY_PATH}}'
      - '{{.AUDIT_WEBHOOK_CONFIG_PATH}}'
    status:
      - '{{.K3D}} cluster list -o json 2>/dev/null | grep -q "\"name\":\"{{.CLUSTER_NAME}}\""'
      - '{{.KUBECTL}} --context {{.CTX}} --request-timeout=10s get ns >/dev/null 2>&1'
    cmds:
      - task: e2e:doctor
      - mkdir -p {{.STAMPS_DIR}} {{.AUDIT_ASSET_DIR}}
      - cp {{.AUDIT_POLICY_SOURCE}} {{.AUDIT_POLICY_PATH}}
      - cp {{.AUDIT_WEBHOOK_BOOTSTRAP_SOURCE}} {{.AUDIT_WEBHOOK_CONFIG_PATH}}
      - |
        export CTX="{{.CTX}}"
        export CLUSTER_NAME="{{.CLUSTER_NAME}}"
        export K3D="{{.K3D}}"
        export KUBECTL="{{.KUBECTL}}"
        export AUDIT_DIR_REL="{{.AUDIT_ASSET_DIR}}"
        bash test/e2e/cluster/start-cluster.sh
      - touch {{.STAMPS_DIR}}/ready

  e2e:_flux-installed:
    method: timestamp
    sources:
      - '{{.STAMPS_DIR}}/ready'
    generates:
      - '{{.STAMPS_DIR}}/flux.installed'
    status:
      - '{{.KUBECTL}} --context {{.CTX}} -n {{.FLUX_NAMESPACE}} get deploy/source-controller deploy/kustomize-controller deploy/helm-controller >/dev/null 2>&1'
    cmds:
      - task: e2e:_cluster-ready
      - "{{.FLUX}} install --context {{.CTX}} --namespace {{.FLUX_NAMESPACE}}"
      - touch {{.STAMPS_DIR}}/flux.installed

  e2e:_flux-setup-ready:
    method: timestamp
    sources:
      - '{{.STAMPS_DIR}}/flux.installed'
      - test/e2e/setup/flux/**
      - hack/e2e/wait-flux-resources.sh
    generates:
      - '{{.STAMPS_DIR}}/flux-setup.ready'
    status:
      - '{{.KUBECTL}} --context {{.CTX}} get helmreleases.helm.toolkit.fluxcd.io --all-namespaces >/dev/null 2>&1'
      - '{{.KUBECTL}} --context {{.CTX}} get kustomizations.kustomize.toolkit.fluxcd.io --all-namespaces >/dev/null 2>&1'
    cmds:
      - task: e2e:_flux-installed
      - "{{.KUBECTL}} --context {{.CTX}} apply -k test/e2e/setup/flux"
      - |
        export CTX="{{.CTX}}"
        export FLUX_SERVICES_WAIT_TIMEOUT="{{.FLUX_WAIT_TIMEOUT}}"
        bash hack/e2e/wait-flux-resources.sh
      - touch {{.STAMPS_DIR}}/flux-setup.ready

  e2e:_services-ready:
    method: timestamp
    sources:
      - '{{.STAMPS_DIR}}/flux-setup.ready'
      - test/e2e/setup/manifests/kustomization.yaml
      - test/e2e/setup/manifests/prometheus/**
    generates:
      - '{{.STAMPS_DIR}}/services.ready'
    status:
      - '{{.KUBECTL}} --context {{.CTX}} wait --for=condition=Established crd/prometheuses.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com --timeout=10s >/dev/null 2>&1'
      - '{{.KUBECTL}} --context {{.CTX}} -n prometheus-operator get prometheus/prometheus-e2e >/dev/null 2>&1'
    cmds:
      - task: e2e:_flux-setup-ready
      - |
        {{.KUBECTL}} --context {{.CTX}} wait --for=condition=Established \
          crd/prometheuses.monitoring.coreos.com \
          crd/servicemonitors.monitoring.coreos.com \
          --timeout=180s
        {{.KUBECTL}} --context {{.CTX}} apply -k test/e2e/setup/manifests
      - touch {{.STAMPS_DIR}}/services.ready

  e2e:_proxy-image-built:
    method: checksum
    sources:
      - Dockerfile
      - cmd/**/*.go
      - pkg/**/*.go
      - go.mod
      - go.sum
      - exclude: cmd/**/*_test.go
      - exclude: pkg/**/*_test.go
    generates:
      - '{{.STAMPS_DIR}}/proxy-image-build.id'
    status:
      - '{{.DOCKER}} image inspect "{{.E2E_PROXY_IMAGE}}" >/dev/null 2>&1'
      - test "$(cat "{{.STAMPS_DIR}}/proxy-image-build.id" 2>/dev/null)" = "$({{.DOCKER}} image inspect --format='{{"{{"}}.Id{{"}}"}}' "{{.E2E_PROXY_IMAGE}}")"
    cmds:
      - mkdir -p {{.STAMPS_DIR}}
      - "{{.DOCKER}} build -t {{.E2E_PROXY_IMAGE}} ."
      - '{{.DOCKER}} image inspect --format=''{{"{{"}}.Id{{"}}"}}'' "{{.E2E_PROXY_IMAGE}}" > {{.STAMPS_DIR}}/proxy-image-build.id'

  e2e:_proxy-image-loaded:
    method: timestamp
    sources:
      - Dockerfile
      - cmd/**/*.go
      - pkg/**/*.go
      - go.mod
      - go.sum
      - exclude: cmd/**/*_test.go
      - exclude: pkg/**/*_test.go
    generates:
      - '{{.STAMPS_DIR}}/proxy-image.stamp'
    status:
      - '{{.DOCKER}} image inspect "{{.E2E_PROXY_IMAGE}}" >/dev/null 2>&1'
      - case "$(cat "{{.STAMPS_DIR}}/proxy-image.stamp" 2>/dev/null)" in "{{.E2E_PROXY_IMAGE}}"@*) exit 0 ;; *) exit 1 ;; esac
      - '{{.K3D}} cluster list -o json 2>/dev/null | grep -q "\"name\":\"{{.CLUSTER_NAME}}\""'
    cmds:
      - task: e2e:_cluster-ready
      - mkdir -p {{.STAMPS_DIR}}
      - |
        export CLUSTER_NAME="{{.CLUSTER_NAME}}"
        export IMAGE="{{.E2E_PROXY_IMAGE}}"
        export K3D="{{.K3D}}"
        export DOCKER="{{.DOCKER}}"
        export STAMP_FILE="{{.STAMPS_DIR}}/proxy-image.stamp"
        bash hack/e2e/load-image.sh
      - rm -f {{.STAMPS_DIR}}/proxy-test-webhook-receiver.installed

  e2e:_proxy-with-webhook-tester-installed:
    method: timestamp
    sources:
      - '{{.STAMPS_DIR}}/proxy-image.stamp'
      - '{{.STAMPS_DIR}}/services.ready'
      - '{{.E2E_PROXY_VALUES_FILE}}'
      - charts/apiservice-audit-proxy/**
    generates:
      - '{{.STAMPS_DIR}}/proxy-test-webhook-receiver.installed'
    status:
      - '{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_RELEASE_NAME}} --timeout=10s >/dev/null 2>&1'
      - '{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_BACKEND_DEPLOYMENT}} --timeout=10s >/dev/null 2>&1'
      - '{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_RELEASE_NAME}}-test-webhook-receiver --timeout=10s >/dev/null 2>&1'
      - |
        expected="$(printf 'mode=test-webhook-receiver\nimage=%s\nvalues=%s\n' "$(cat "{{.STAMPS_DIR}}/proxy-image.stamp" 2>/dev/null)" "{{.E2E_PROXY_VALUES_FILE}}")"
        test "$(cat "{{.STAMPS_DIR}}/proxy-test-webhook-receiver.installed" 2>/dev/null)" = "${expected}"
    cmds:
      - task: e2e:load-image
      - task: e2e:_services-ready
      - |
        {{.HELM}} upgrade --install {{.E2E_RELEASE_NAME}} ./charts/apiservice-audit-proxy \
          --kube-context {{.CTX}} \
          --namespace {{.E2E_PROXY_NAMESPACE}} \
          --create-namespace \
          --values {{.E2E_PROXY_VALUES_FILE}} \
          --set image.repository={{.E2E_PROXY_IMAGE_REPOSITORY}} \
          --set image.tag={{.E2E_PROXY_IMAGE_TAG}} \
          --set audit.testWebhookReceiver.enabled=true
      - "{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_BACKEND_DEPLOYMENT}} --timeout=180s"
      - "{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_RELEASE_NAME}}-test-webhook-receiver --timeout=180s"
      - "{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout restart deploy/{{.E2E_RELEASE_NAME}}"
      - "{{.KUBECTL}} --context {{.CTX}} -n {{.E2E_PROXY_NAMESPACE}} rollout status deploy/{{.E2E_RELEASE_NAME}} --timeout=180s"
      - |
        printf 'mode=test-webhook-receiver\nimage=%s\nvalues=%s\n' \
          "$(<"{{.STAMPS_DIR}}/proxy-image.stamp")" \
          "{{.E2E_PROXY_VALUES_FILE}}" \
          > {{.STAMPS_DIR}}/proxy-test-webhook-receiver.installed