From 5c7e9fed40958c2d4cdc768d0e7d0ed9d6ff49cc Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Tue, 28 Jul 2026 17:16:37 -0400 Subject: [PATCH] Add autoscaled-workerpool demo using HPA and prometheus-adapter on Kind --- demos/autoscaled-workerpool/README.md | 174 +++++++++++++ .../autoscaled-workerpool.yaml.tmpl | 73 ++++++ demos/autoscaled-workerpool/hpa-kind.yaml | 61 +++++ .../prometheus-adapter.yaml | 241 ++++++++++++++++++ hack/install-ate.sh | 1 + hack/install-demo-autoscaled-workerpool.sh | 83 ++++++ 6 files changed, 633 insertions(+) create mode 100644 demos/autoscaled-workerpool/README.md create mode 100644 demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl create mode 100644 demos/autoscaled-workerpool/hpa-kind.yaml create mode 100644 demos/autoscaled-workerpool/prometheus-adapter.yaml create mode 100644 hack/install-demo-autoscaled-workerpool.sh diff --git a/demos/autoscaled-workerpool/README.md b/demos/autoscaled-workerpool/README.md new file mode 100644 index 000000000..d1652b86f --- /dev/null +++ b/demos/autoscaled-workerpool/README.md @@ -0,0 +1,174 @@ + + +# Autoscaling WorkerPools with an HPA + +This describes how to autoscale a `WorkerPool` on how many of its workers are +currently assigned, using a Kubernetes HorizontalPodAutoscaler (HPA) fed by the +`ate_workerpool_workers` metric through prometheus-adapter. + +## Prerequisites + +- A local kind cluster with Agent Substrate installed (`./hack/install-ate-kind.sh --deploy-ate-system`). + Note: this demo is currently only supported on kind. +- `ko` installed for building images. +- A GCS bucket for storing snapshots (configured via `BUCKET_NAME` env var). + +## Architecture (kind) + +``` +ate-api-server :9090/metrics (ate_workerpool_workers, OTel -> Prometheus gauge) + │ scrape (prometheus.io/scrape annotation, 15s) + ▼ +Prometheus manifests/ate-install/monitoring/prometheus.yaml + │ PromQL + ▼ +prometheus-adapter demos/autoscaled-workerpool/prometheus-adapter.yaml + │ external.metrics.k8s.io + ▼ +HorizontalPodAutoscaler + │ /scale (writes WorkerPool.spec.replicas) + ▼ +atecontroller ──► Deployment.spec.replicas ──► worker pods +``` +## HPA Configuration (External + AverageValue) + +The example HPA uses an **External** metric with target type **AverageValue**: + +``` +desiredReplicas = ceil( metricValue / target.averageValue ) +``` + +where `metricValue = max(ate_workerpool_workers{name=, state=assigned})`: +the pool's assigned-worker count. + +`averageValue` is the target **assigned-workers-per-replica**: + +| `averageValue` | Meaning | Example (`assigned=7`) | +| -------------- | ------------------------------------------- | ---------------------- | +| `"0.7"` (700m) | ~70% assigned / 30% idle headroom (default) | `ceil(7/0.7) = 10` | +| `"1"` | pack to 100%, no idle headroom | `ceil(7/1) = 7` | +| `"0.5"` (500m) | lots of headroom, ~2× replicas | `ceil(7/0.5) = 14` | + +Lower `averageValue` → more idle headroom → more replicas. + +### Scale-up/scale-down behavior + +The example HPAs set an aggressive `scaleUp` (no stabilization window, +`selectPolicy: Max`, `Percent 100` / `Pods 10` steps) so a burst is served in a +**batch** rather than creeping up one worker at a time, and a slow `scaleDown` +(300s stabilization) to avoid flapping. `behavior` only sets the _rate of change_; +the HPA still scales to what the metric dictates. + +## How to Run on Agent Substrate + +### 1. Build and Deploy + +```sh +# Local dev (kind) +./hack/install-ate-kind.sh --deploy-demo-autoscaled-workerpool +``` + +This command will: + +- Deploy `prometheus-adapter` into `ate-demo-autoscaled-workerpool` to serve `ate_workerpool_workers` on `external.metrics.k8s.io`. +- Create the `ate-demo-autoscaled-workerpool` namespace. +- Create one `WorkerPool` (`counter`, starting with 5 replicas), one `ActorTemplate` (`counter`), and one `HorizontalPodAutoscaler` (`counter`). +- Wait until the template is `Ready` and the pool is rolled out. + +### 2. Verify Monitoring Stack & External Metric + +Confirm that `prometheus-adapter` is serving the external metric: + +```sh +# 1. Adapter is serving the External Metrics API +kubectl get apiservice v1beta1.external.metrics.k8s.io # Available=True + +# 2. The external metric resolves for the counter pool +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/ate-demo-autoscaled-workerpool/ate_workerpool_workers?labelSelector=ate_worker_state%3Dassigned,ate_workerpool_name%3Dcounter" +``` + +## How to Use + +We can trigger autoscaling by creating an atespace, spawning multiple actors, and sending traffic to assign workers in the pool: + +### 1. Create an atespace and spawn load actors + +```sh +# Install the CLI as a kubectl plugin if not already installed +go install ./cmd/kubectl-ate + +# Create an atespace for the actors +kubectl ate create atespace demo + +# Create 15 actors to generate load +for i in {001..015}; do + kubectl ate create actor c$i -a demo --template ate-demo-autoscaled-workerpool/counter +done +``` + +### 2. Port-forward the atenet router and send traffic + +```sh +kubectl port-forward -n ate-system svc/atenet-router 8000:80 & +``` + +In a separate terminal, send requests in a retry loop across all hosts to activate the actors and keep sessions active while the pool scales up: + +```sh +for attempt in {1..10}; do + for i in {001..015}; do + curl -s -H "Host: c$i.demo.actors.resources.substrate.ate.dev" http://localhost:8000 >/dev/null + done + sleep 2 +done +``` + +### 3. Watch the HPA scale up + +As the assigned worker count increases, watch the HPA scale up the pool's replicas: + +```sh +kubectl -n ate-demo-autoscaled-workerpool get hpa counter -w +kubectl -n ate-demo-autoscaled-workerpool get workerpool counter -w +``` + +### 4. Trigger scale-down + +Suspend the actors to drop the assigned worker count. After the 300s stabilization window, the HPA will scale down the pool: + +```sh +for i in {001..015}; do + kubectl ate suspend actor c$i -a demo +done +``` + +## How to Uninstall + +First clean up the actors and atespace: + +```sh +for i in {001..015}; do + kubectl ate delete actor c$i -a demo +done +kubectl ate delete atespace demo +``` + +Then remove the demo resources and namespace: + +```sh +./hack/install-ate-kind.sh --delete-demo-autoscaled-workerpool +``` diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl new file mode 100644 index 000000000..4880383c8 --- /dev/null +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -0,0 +1,73 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This demo demonstrates autoscaling an Agent Substrate WorkerPool using a +# HorizontalPodAutoscaler (HPA) and an External Metric (ate_workerpool_workers). +# The pool runs capacity for the "counter" workload; the HPA scales the pool +# dynamically based on the number of assigned workers. +# +# See hpa-workerpool-autoscaling.md for the design and math details. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-autoscaled-workerpool + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: counter + namespace: ate-demo-autoscaled-workerpool + labels: + workload: counter-autoscaled +spec: + replicas: 5 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: counter + namespace: ate-demo-autoscaled-workerpool +spec: + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + containers: + - name: counter + image: ko://github.com/agent-substrate/substrate/demos/counter + command: + - /ko-app/counter +${VALIDATE_EXISTING_FILE_PATH_ARG} + readyz: + httpGet: + path: /readyz + port: 80 + volumeMounts: + - name: data + mountPath: /home/counter +${EXTERNAL_VOLUME_MOUNTS} + workerSelector: + matchLabels: + workload: counter-autoscaled + snapshotsConfig: + onPause: Full + onCommit: Data + location: gs://${BUCKET_NAME}/ate-demo-autoscaled-workerpool/ + volumes: + - name: data + durableDir: {} +${EXTERNAL_VOLUMES} diff --git a/demos/autoscaled-workerpool/hpa-kind.yaml b/demos/autoscaled-workerpool/hpa-kind.yaml new file mode 100644 index 000000000..84ff76c2d --- /dev/null +++ b/demos/autoscaled-workerpool/hpa-kind.yaml @@ -0,0 +1,61 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autoscale the counter WorkerPool based on how many of its workers are assigned. +# The metric ate_workerpool_workers (emitted by ate-api-server) is served to the +# External Metrics API by prometheus-adapter (demos/autoscaled-workerpool). +# External AverageValue math: desiredReplicas = ceil(assigned / averageValue). +# With averageValue=0.7 the pool targets ~70% assignment / 30% idle headroom. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: counter + namespace: ate-demo-autoscaled-workerpool +spec: + scaleTargetRef: + apiVersion: ate.dev/v1alpha1 + kind: WorkerPool + name: counter + minReplicas: 2 # warm floor: keep at least 2 workers ready + maxReplicas: 25 + metrics: + - type: External + external: + metric: + name: ate_workerpool_workers + selector: + matchLabels: + ate_workerpool_name: counter + ate_worker_state: assigned + target: + type: AverageValue + averageValue: "0.7" # 70% utilization + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + selectPolicy: Max + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 10 + periodSeconds: 15 + scaleDown: + stabilizationWindowSeconds: 300 + selectPolicy: Max + policies: + - type: Pods + value: 2 + periodSeconds: 60 diff --git a/demos/autoscaled-workerpool/prometheus-adapter.yaml b/demos/autoscaled-workerpool/prometheus-adapter.yaml new file mode 100644 index 000000000..9e89d3615 --- /dev/null +++ b/demos/autoscaled-workerpool/prometheus-adapter.yaml @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# prometheus-adapter exposes the self-hosted Prometheus (prometheus.yaml) to the +# Kubernetes External Metrics API (external.metrics.k8s.io) so an HPA can scale a +# WorkerPool on ate_workerpool_workers. +# This is a standard install (github.com/kubernetes-sigs/prometheus-adapter) +# trimmed to the external-metrics path and pointed at the in-cluster Prometheus. +#Runs in ate-demo-autoscaled-workerpool. +# +# Flow: ate-api-server:9090/metrics --scrape--> Prometheus --query--> adapter +# --external.metrics.k8s.io--> HPA --/scale--> WorkerPool -> Deployment. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool +--- +# The external rule maps the Prometheus series ate_workerpool_workers to an +# external metric of the same name. An HPA's metric.selector.matchLabels are +# substituted into <<.LabelMatchers>>, so a selector of +# {ate_workerpool_name: shared-pool, ate_worker_state: assigned} yields +# max(ate_workerpool_workers{ate_workerpool_name="shared-pool",ate_worker_state="assigned"}) +# — the pool's assigned-worker count. + +# We use max() rather than sum() because ate-api-server is sharded; +# every replica reports the global worker tally. +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-adapter-config + namespace: ate-demo-autoscaled-workerpool +data: + config.yaml: | + externalRules: + - seriesQuery: '{__name__="ate_workerpool_workers"}' + resources: + namespaced: false + name: + matches: "^(.*)$" + as: "$1" + # max() across ate-api-server replicas since each replica reports the + # global count. + metricsQuery: 'max(<<.Series>>{<<.LabelMatchers>>}) by (ate_workerpool_name, ate_worker_state)' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool + labels: + app: prometheus-adapter +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-adapter + template: + metadata: + labels: + app: prometheus-adapter + spec: + serviceAccountName: prometheus-adapter + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: prometheus-adapter + image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0 + args: + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --prometheus-url=http://prometheus.otel-system.svc:9090/ + - --metrics-relist-interval=30s + - --config=/etc/adapter/config.yaml + - --v=4 + ports: + - name: https + containerPort: 6443 + readinessProbe: + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + livenessProbe: + httpGet: + path: /livez + port: https + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /etc/adapter/ + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: prometheus-adapter-config + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool + labels: + app: prometheus-adapter +spec: + type: ClusterIP + selector: + app: prometheus-adapter + ports: + - name: https + port: 443 + targetPort: https +--- +# Register the adapter as the backend for external.metrics.k8s.io. insecureSkipTLSVerify +# is fine for dev (the adapter self-signs its serving cert via --cert-dir); for prod, +# supply a CA bundle instead. NOTE: only one adapter can own this APIService per +# cluster — the GMP custom-metrics-stackdriver-adapter registers the same group, so +# the two non-GMP / GMP variants must run in separate clusters. +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool + port: 443 + group: external.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +# Let the adapter authenticate/authorize incoming aggregated-API requests. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-adapter-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-adapter-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool +--- +# The adapter discovers namespaces/pods/services to resolve metric associations. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-adapter-resource-reader +rules: +- apiGroups: [""] + resources: ["namespaces", "pods", "services", "configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-adapter-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter-resource-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: ate-demo-autoscaled-workerpool +--- +# Allow the built-in HPA controller to read external metrics from the aggregated API. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-adapter-external-metrics-reader +rules: +- apiGroups: ["external.metrics.k8s.io"] + resources: ["*"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter-external-metrics-reader +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system diff --git a/hack/install-ate.sh b/hack/install-ate.sh index e53f8c5de..b8c871020 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -44,6 +44,7 @@ source "${ROOT}"/hack/install-demo-sandbox.sh source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh source "${ROOT}"/hack/install-demo-multi-template.sh source "${ROOT}"/hack/install-demo-parking.sh +source "${ROOT}"/hack/install-demo-autoscaled-workerpool.sh # ANSI color codes for prettier output COLOR_CYAN='\033[1;36m' diff --git a/hack/install-demo-autoscaled-workerpool.sh b/hack/install-demo-autoscaled-workerpool.sh new file mode 100644 index 000000000..a35e95da8 --- /dev/null +++ b/hack/install-demo-autoscaled-workerpool.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is sourced as part of install-ate.sh. Do not run directly. + +ATE_DEMOS+=(demo-autoscaled-workerpool) # register demo-autoscaled-workerpool + +demo-autoscaled-workerpool_usage() { + echo " --deploy-demo-autoscaled-workerpool Deploy autoscaled-workerpool demo (HPA + prometheus-adapter + counter workload)" +} + +demo-autoscaled-workerpool_cmdline() { + case "${1}" in + --deploy-demo-autoscaled-workerpool) demo-autoscaled-workerpool_deploy ;; + --delete-demo-autoscaled-workerpool) demo-autoscaled-workerpool_delete ;; + *) + return 1 + ;; + esac + return 0 +} + +demo-autoscaled-workerpool_deploy() { + log_step "demo-autoscaled-workerpool_deploy" + if [[ "${ATE_INSTALL_KIND:-false}" == "false" ]]; then + echo "Error: --deploy-demo-autoscaled-workerpool is not supported on GKE yet" >&2 + exit 1 + fi + ensure_crds + + # Ensure namespace exists before deploying adapter/workload + run_kubectl create namespace ate-demo-autoscaled-workerpool --dry-run=client -o yaml | run_kubectl apply -f - + + # Deploy common workload (Namespace, WorkerPool, ActorTemplate) + log_step "Deploying autoscaled-workerpool workload..." + sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ + -e "/\${VALIDATE_EXISTING_FILE_PATH_ARG}/d" \ + -e "/\${EXTERNAL_VOLUME_MOUNTS}/d" \ + -e "/\${EXTERNAL_VOLUMES}/d" \ + demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl \ + | run_ko apply -f - + + log_step "Deploying prometheus-adapter and HPA for kind..." + run_kubectl apply -f demos/autoscaled-workerpool/prometheus-adapter.yaml + run_kubectl rollout status deployment/prometheus-adapter -n ate-demo-autoscaled-workerpool --timeout=120s + run_kubectl apply -f demos/autoscaled-workerpool/hpa-kind.yaml + + log_step "Waiting for autoscaled-workerpool demo to be ready..." + run_kubectl rollout status deployment/counter -n ate-demo-autoscaled-workerpool --timeout=300s + run_kubectl wait --for=condition=Ready actortemplate/counter -n ate-demo-autoscaled-workerpool --timeout=300s +} + +demo-autoscaled-workerpool_delete() { + log_step "demo-autoscaled-workerpool_delete" + if [[ "${ATE_INSTALL_KIND:-false}" != "true" ]]; then + echo "Error: --delete-demo-autoscaled-workerpool is not supported on GKE" >&2 + exit 1 + fi + delete_demo_actors ate-demo-autoscaled-workerpool counter + + run_kubectl delete --ignore-not-found -f demos/autoscaled-workerpool/hpa-kind.yaml + run_kubectl delete --ignore-not-found -f demos/autoscaled-workerpool/prometheus-adapter.yaml + + sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ + -e "/\${VALIDATE_EXISTING_FILE_PATH_ARG}/d" \ + -e "/\${EXTERNAL_VOLUME_MOUNTS}/d" \ + -e "/\${EXTERNAL_VOLUMES}/d" \ + demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl \ + | run_kubectl delete --ignore-not-found -f - +}