e2e.yml

name: e2e

on:
  pull_request:
    paths:
      - 'charts/llama-8b/**'
      - 'policies/**'
      - 'httproutes/**'
      - 'gateway/**'
      - '.github/workflows/e2e.yml'
  push:
    branches: [main]
  workflow_dispatch:

jobs:
  kind-install:
    name: kind cluster + chart install + admission checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: install kind cluster
        uses: helm/kind-action@v1.10.0
        with:
          cluster_name: e2e
          version: v0.24.0
          node_image: kindest/node:v1.30.4

      - name: install Gateway API v1 CRDs (experimental channel)
        run: |
          kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/experimental-install.yaml
          for crd in gateways httproutes tlsroutes backendtlspolicies; do
            for i in $(seq 1 30); do
              status=$(kubectl get crd ${crd}.gateway.networking.k8s.io \
                -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true)
              [ "$status" = "True" ] && break
              sleep 2
            done
            kubectl get crd ${crd}.gateway.networking.k8s.io \
              -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' | grep -q True
          done

      - name: install KEDA CRDs only
        run: |
          kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.15.1/keda-2.15.1-crds.yaml
          # Poll until Established shows up — bare `kubectl wait` races with
          # a nil status.conditions and errors "<nil> is of the type <nil>".
          for i in $(seq 1 30); do
            status=$(kubectl get crd scaledobjects.keda.sh \
              -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true)
            [ "$status" = "True" ] && break
            sleep 2
          done
          kubectl get crd scaledobjects.keda.sh \
            -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' | grep -q True

      - name: install Prometheus Operator CRDs (for ServiceMonitor)
        run: |
          kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.76.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
          kubectl wait --for condition=established --timeout=60s crd/servicemonitors.monitoring.coreos.com

      - name: install Gateway API Inference Extension CRDs
        run: |
          # v1.5.0 config/crd/bases has: inferencepools.inference.networking.k8s.io (v1)
          # and inferenceobjectives.inference.networking.x-k8s.io (v1alpha2).
          kubectl apply --server-side -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/v1.5.0/config/crd/bases/inference.networking.k8s.io_inferencepools.yaml
          for i in $(seq 1 30); do
            status=$(kubectl get crd inferencepools.inference.networking.k8s.io \
              -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true)
            [ "$status" = "True" ] && break
            sleep 2
          done

      - name: install minimal Argo Workflows CRD (referenced by mutate-workflow-ttl policy)
        # The upstream v3.6.2 CRD uses schemaless fields that k8s 1.30 rejects
        # for structural-schema violations. We don't need the full schema in
        # e2e — Kyverno only needs to resolve group/version/resource. This
        # inline CRD satisfies both.
        run: |
          cat <<'EOF' | kubectl apply --server-side -f -
          apiVersion: apiextensions.k8s.io/v1
          kind: CustomResourceDefinition
          metadata:
            name: workflows.argoproj.io
          spec:
            group: argoproj.io
            scope: Namespaced
            names:
              kind: Workflow
              plural: workflows
              singular: workflow
              shortNames: [wf]
            versions:
              - name: v1alpha1
                served: true
                storage: true
                schema:
                  openAPIV3Schema:
                    type: object
                    x-kubernetes-preserve-unknown-fields: true
          EOF
          for i in $(seq 1 30); do
            status=$(kubectl get crd workflows.argoproj.io \
              -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true)
            [ "$status" = "True" ] && break
            sleep 2
          done

      - name: install Kyverno (admission enforcement path)
        run: |
          helm repo add kyverno https://kyverno.github.io/kyverno/
          helm install kyverno kyverno/kyverno --version 3.3.0 \
            --namespace kyverno --create-namespace \
            --set admissionController.replicas=1 \
            --set backgroundController.enabled=false \
            --set reportsController.enabled=false \
            --set cleanupController.enabled=false \
            --wait --timeout=5m

      - name: apply our ClusterPolicies
        run: |
          kubectl apply -f policies/
          # Give Kyverno a moment to register the webhook configurations
          sleep 15

      - name: create supporting resources (fake secrets + namespaces + nvidia RuntimeClass stub)
        run: |
          kubectl create namespace llama
          kubectl create secret generic hf-token         -n llama --from-literal=token=fake
          kubectl create secret generic vllm-api-key     -n llama --from-literal=token=fake-api-key
          kubectl create namespace gateway || true
          kubectl create namespace monitoring || true
          kubectl create namespace argo || true
          # Stub RuntimeClass so pod admission passes on kind (which has no
          # nvidia-container-runtime). Handler=runc means pods just run under
          # normal runc — sufficient for admission testing.
          cat <<'EOF' | kubectl apply -f -
          apiVersion: node.k8s.io/v1
          kind: RuntimeClass
          metadata:
            name: nvidia
          handler: runc
          EOF

      - name: helm install the chart (with fake image so pod won't try to pull real vLLM)
        run: |
          helm install llama charts/llama-8b \
            --namespace llama \
            --set image=nginx:alpine \
            --set otlp.installSdk=false \
            --set otlp.tracesEndpoint="" \
            --set persistence.hfCache.storageClass=standard \
            --set rolloutGate.enabled=false \
            --set autoscaling.enabled=false \
            --wait --timeout=2m || true
          # ^^ pods won't be Ready (nginx doesn't serve /health on 8000), but the manifests must apply

      - name: verify every expected resource exists
        run: |
          set -euxo pipefail
          kubectl -n llama get deployment llama-llama-8b
          kubectl -n llama get service    llama-llama-8b
          kubectl -n llama get pvc        llama-llama-8b-hf-cache
          kubectl -n llama get networkpolicy llama-llama-8b-default-deny
          kubectl -n llama get networkpolicy llama-llama-8b-allow
          kubectl -n llama get servicemonitor llama-llama-8b
          kubectl get priorityclass gpu-inference

      - name: verify chart renders runtimeClassName correctly on Deployment template
        run: |
          rc=$(kubectl -n llama get deploy llama-llama-8b \
            -o jsonpath='{.spec.template.spec.runtimeClassName}')
          [ "$rc" = "nvidia" ] || { echo "expected nvidia, got $rc"; exit 1; }

      - name: verify Kyverno mutate policy actually fires on a bare Pod
        # Apply a Pod that requests GPU but has NO runtimeClassName. Kyverno's
        # mutate-nvidia-runtime-class should stamp it before admission finishes.
        run: |
          # Wait for Kyverno webhook to be responsive
          for i in $(seq 1 60); do
            kubectl -n kyverno get validatingwebhookconfiguration kyverno-policy-validating-webhook-cfg >/dev/null 2>&1 && break
            sleep 2
          done
          sleep 10

          cat <<'EOF' | kubectl apply -f -
          apiVersion: v1
          kind: Pod
          metadata:
            name: mutation-test
            namespace: default
          spec:
            containers:
              - name: c
                image: nginx:alpine
                resources:
                  requests: {nvidia.com/gpu: 1}
                  limits:   {nvidia.com/gpu: 1}
          EOF

          rc=$(kubectl get pod mutation-test -o jsonpath='{.spec.runtimeClassName}')
          echo "runtimeClassName after Kyverno mutation: $rc"
          [ "$rc" = "nvidia" ] || { echo "Kyverno mutation did not fire"; kubectl -n kyverno logs -l app.kubernetes.io/component=admission-controller --tail=50; exit 1; }
          kubectl delete pod mutation-test

      - name: verify no Kyverno policy is denying anything unexpected
        run: |
          # If Kyverno reports blocking violations on the chart, dump them and fail.
          # In audit mode our validate policies produce PolicyReports; grab them.
          kubectl -n llama get policyreport -o yaml 2>/dev/null | tee /tmp/policyreports.yaml || true
          # Fail only if a policy is in enforce mode and a chart resource was rejected.
          # (We already applied resources above; if enforce policies had rejected them,
          #  the previous kubectl get calls would have returned NotFound.)

      - name: dump diagnostics if anything failed
        if: failure()
        run: |
          echo "=== events ==="
          kubectl -n llama get events --sort-by=.lastTimestamp | tail -50
          echo "=== pods ==="
          kubectl -n llama describe pods
          echo "=== kyverno logs ==="
          kubectl -n kyverno logs -l app.kubernetes.io/component=admission-controller --tail=100 || true