loadtests: pass vLLM API key via OPENAI_API_KEY env (bench was hittin… #36
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e | |
| on: | |
| pull_request: | |
| paths: | |
| - 'charts/llama-8b/**' | |
| - 'policies/**' | |
| - 'httproutes/**' | |
| - 'gateway/**' | |
| - '.github/workflows/e2e.yml' | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| jobs: | |
| kind-install: | |
| name: kind cluster + chart install + admission checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install kind cluster | |
| uses: helm/kind-action@v1.10.0 | |
| with: | |
| cluster_name: e2e | |
| version: v0.24.0 | |
| node_image: kindest/node:v1.30.4 | |
| - name: install Gateway API v1 CRDs (experimental channel) | |
| run: | | |
| kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/experimental-install.yaml | |
| for crd in gateways httproutes tlsroutes backendtlspolicies; do | |
| for i in $(seq 1 30); do | |
| status=$(kubectl get crd ${crd}.gateway.networking.k8s.io \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true) | |
| [ "$status" = "True" ] && break | |
| sleep 2 | |
| done | |
| kubectl get crd ${crd}.gateway.networking.k8s.io \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' | grep -q True | |
| done | |
| - name: install KEDA CRDs only | |
| run: | | |
| kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.15.1/keda-2.15.1-crds.yaml | |
| # Poll until Established shows up — bare `kubectl wait` races with | |
| # a nil status.conditions and errors "<nil> is of the type <nil>". | |
| for i in $(seq 1 30); do | |
| status=$(kubectl get crd scaledobjects.keda.sh \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true) | |
| [ "$status" = "True" ] && break | |
| sleep 2 | |
| done | |
| kubectl get crd scaledobjects.keda.sh \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' | grep -q True | |
| - name: install Prometheus Operator CRDs (for ServiceMonitor) | |
| run: | | |
| kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.76.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml | |
| kubectl wait --for condition=established --timeout=60s crd/servicemonitors.monitoring.coreos.com | |
| - name: install Gateway API Inference Extension CRDs | |
| run: | | |
| # v1.5.0 config/crd/bases has: inferencepools.inference.networking.k8s.io (v1) | |
| # and inferenceobjectives.inference.networking.x-k8s.io (v1alpha2). | |
| kubectl apply --server-side -f https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/v1.5.0/config/crd/bases/inference.networking.k8s.io_inferencepools.yaml | |
| for i in $(seq 1 30); do | |
| status=$(kubectl get crd inferencepools.inference.networking.k8s.io \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true) | |
| [ "$status" = "True" ] && break | |
| sleep 2 | |
| done | |
| - name: install minimal Argo Workflows CRD (referenced by mutate-workflow-ttl policy) | |
| # The upstream v3.6.2 CRD uses schemaless fields that k8s 1.30 rejects | |
| # for structural-schema violations. We don't need the full schema in | |
| # e2e — Kyverno only needs to resolve group/version/resource. This | |
| # inline CRD satisfies both. | |
| run: | | |
| cat <<'EOF' | kubectl apply --server-side -f - | |
| apiVersion: apiextensions.k8s.io/v1 | |
| kind: CustomResourceDefinition | |
| metadata: | |
| name: workflows.argoproj.io | |
| spec: | |
| group: argoproj.io | |
| scope: Namespaced | |
| names: | |
| kind: Workflow | |
| plural: workflows | |
| singular: workflow | |
| shortNames: [wf] | |
| versions: | |
| - name: v1alpha1 | |
| served: true | |
| storage: true | |
| schema: | |
| openAPIV3Schema: | |
| type: object | |
| x-kubernetes-preserve-unknown-fields: true | |
| EOF | |
| for i in $(seq 1 30); do | |
| status=$(kubectl get crd workflows.argoproj.io \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null || true) | |
| [ "$status" = "True" ] && break | |
| sleep 2 | |
| done | |
| - name: install Kyverno (admission enforcement path) | |
| run: | | |
| helm repo add kyverno https://kyverno.github.io/kyverno/ | |
| helm install kyverno kyverno/kyverno --version 3.3.0 \ | |
| --namespace kyverno --create-namespace \ | |
| --set admissionController.replicas=1 \ | |
| --set backgroundController.enabled=false \ | |
| --set reportsController.enabled=false \ | |
| --set cleanupController.enabled=false \ | |
| --wait --timeout=5m | |
| - name: apply our ClusterPolicies | |
| run: | | |
| kubectl apply -f policies/ | |
| # Give Kyverno a moment to register the webhook configurations | |
| sleep 15 | |
| - name: create supporting resources (fake secrets + namespaces + nvidia RuntimeClass stub) | |
| run: | | |
| kubectl create namespace llama | |
| kubectl create secret generic hf-token -n llama --from-literal=token=fake | |
| kubectl create secret generic vllm-api-key -n llama --from-literal=token=fake-api-key | |
| kubectl create namespace gateway || true | |
| kubectl create namespace monitoring || true | |
| kubectl create namespace argo || true | |
| # Stub RuntimeClass so pod admission passes on kind (which has no | |
| # nvidia-container-runtime). Handler=runc means pods just run under | |
| # normal runc — sufficient for admission testing. | |
| cat <<'EOF' | kubectl apply -f - | |
| apiVersion: node.k8s.io/v1 | |
| kind: RuntimeClass | |
| metadata: | |
| name: nvidia | |
| handler: runc | |
| EOF | |
| - name: helm install the chart (with fake image so pod won't try to pull real vLLM) | |
| run: | | |
| helm install llama charts/llama-8b \ | |
| --namespace llama \ | |
| --set image=nginx:alpine \ | |
| --set otlp.installSdk=false \ | |
| --set otlp.tracesEndpoint="" \ | |
| --set persistence.hfCache.storageClass=standard \ | |
| --set rolloutGate.enabled=false \ | |
| --set autoscaling.enabled=false \ | |
| --wait --timeout=2m || true | |
| # ^^ pods won't be Ready (nginx doesn't serve /health on 8000), but the manifests must apply | |
| - name: verify every expected resource exists | |
| run: | | |
| set -euxo pipefail | |
| kubectl -n llama get deployment llama-llama-8b | |
| kubectl -n llama get service llama-llama-8b | |
| kubectl -n llama get pvc llama-llama-8b-hf-cache | |
| kubectl -n llama get networkpolicy llama-llama-8b-default-deny | |
| kubectl -n llama get networkpolicy llama-llama-8b-allow | |
| kubectl -n llama get servicemonitor llama-llama-8b | |
| kubectl get priorityclass gpu-inference | |
| - name: verify chart renders runtimeClassName correctly on Deployment template | |
| run: | | |
| rc=$(kubectl -n llama get deploy llama-llama-8b \ | |
| -o jsonpath='{.spec.template.spec.runtimeClassName}') | |
| [ "$rc" = "nvidia" ] || { echo "expected nvidia, got $rc"; exit 1; } | |
| - name: verify Kyverno mutate policy actually fires on a bare Pod | |
| # Apply a Pod that requests GPU but has NO runtimeClassName. Kyverno's | |
| # mutate-nvidia-runtime-class should stamp it before admission finishes. | |
| run: | | |
| # Wait for Kyverno webhook to be responsive | |
| for i in $(seq 1 60); do | |
| kubectl -n kyverno get validatingwebhookconfiguration kyverno-policy-validating-webhook-cfg >/dev/null 2>&1 && break | |
| sleep 2 | |
| done | |
| sleep 10 | |
| cat <<'EOF' | kubectl apply -f - | |
| apiVersion: v1 | |
| kind: Pod | |
| metadata: | |
| name: mutation-test | |
| namespace: default | |
| spec: | |
| containers: | |
| - name: c | |
| image: nginx:alpine | |
| resources: | |
| requests: {nvidia.com/gpu: 1} | |
| limits: {nvidia.com/gpu: 1} | |
| EOF | |
| rc=$(kubectl get pod mutation-test -o jsonpath='{.spec.runtimeClassName}') | |
| echo "runtimeClassName after Kyverno mutation: $rc" | |
| [ "$rc" = "nvidia" ] || { echo "Kyverno mutation did not fire"; kubectl -n kyverno logs -l app.kubernetes.io/component=admission-controller --tail=50; exit 1; } | |
| kubectl delete pod mutation-test | |
| - name: verify no Kyverno policy is denying anything unexpected | |
| run: | | |
| # If Kyverno reports blocking violations on the chart, dump them and fail. | |
| # In audit mode our validate policies produce PolicyReports; grab them. | |
| kubectl -n llama get policyreport -o yaml 2>/dev/null | tee /tmp/policyreports.yaml || true | |
| # Fail only if a policy is in enforce mode and a chart resource was rejected. | |
| # (We already applied resources above; if enforce policies had rejected them, | |
| # the previous kubectl get calls would have returned NotFound.) | |
| - name: dump diagnostics if anything failed | |
| if: failure() | |
| run: | | |
| echo "=== events ===" | |
| kubectl -n llama get events --sort-by=.lastTimestamp | tail -50 | |
| echo "=== pods ===" | |
| kubectl -n llama describe pods | |
| echo "=== kyverno logs ===" | |
| kubectl -n kyverno logs -l app.kubernetes.io/component=admission-controller --tail=100 || true |