diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a29a42..13f2a3a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -164,7 +164,7 @@ jobs:
# --target-branch defaults to ``main``; explicit for clarity.
run: |
ct lint \
- --charts charts/promforecast,charts/promforecast-stack \
+ --charts charts/promforecast,charts/promforecast-stack,charts/promforecast-metrics-adapter \
--validate-maintainers=false \
--check-version-increment=false \
--target-branch ${{ github.event.repository.default_branch }}
@@ -176,6 +176,7 @@ jobs:
run: |
helm lint --strict charts/promforecast
helm lint --strict charts/promforecast-stack
+ helm lint --strict charts/promforecast-metrics-adapter
- name: Install kubeconform
run: |
@@ -186,8 +187,9 @@ jobs:
- name: kubeconform
run: |
- helm template charts/promforecast | kubeconform -strict -summary -ignore-missing-schemas
- helm template charts/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas
+ helm template charts/promforecast | kubeconform -strict -summary -ignore-missing-schemas
+ helm template charts/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas
+ helm template charts/promforecast-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas
# Also validate the bundled dashboards + PrometheusRule path so CI
# catches regressions in the templated assets, not only the default
# off path.
diff --git a/Makefile b/Makefile
index 721990e..ba73518 100644
--- a/Makefile
+++ b/Makefile
@@ -61,8 +61,10 @@ chart-deps: ## Resolve umbrella subchart dependencies.
chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests.
helm lint $(CHART_DIR)/promforecast
helm lint $(CHART_DIR)/promforecast-stack
- helm template $(CHART_DIR)/promforecast | kubeconform -strict -summary -ignore-missing-schemas
- helm template $(CHART_DIR)/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas
+ helm lint $(CHART_DIR)/promforecast-metrics-adapter
+ helm template $(CHART_DIR)/promforecast | kubeconform -strict -summary -ignore-missing-schemas
+ helm template $(CHART_DIR)/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas
+ helm template $(CHART_DIR)/promforecast-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas
helm template $(CHART_DIR)/promforecast-stack \
--set dashboards.enabled=true \
--set prometheusRules.enabled=true \
diff --git a/charts/promforecast-metrics-adapter/Chart.yaml b/charts/promforecast-metrics-adapter/Chart.yaml
new file mode 100644
index 0000000..615f5ce
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: promforecast-metrics-adapter
+description: Kubernetes custom-metrics adapter that exposes promforecast forecast metrics for HorizontalPodAutoscaler.
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+kubeVersion: ">=1.27.0-0"
+home: https://github.com/esops-dev/promforecast
+sources:
+ - https://github.com/esops-dev/promforecast
+keywords:
+ - prometheus
+ - forecasting
+ - kubernetes
+ - autoscaling
+ - hpa
+ - custom-metrics
+maintainers:
+ - name: esops-dev
+ url: https://github.com/esops-dev
+icon: https://raw.githubusercontent.com/esops-dev/promforecast/main/docs/assets/logo.png
+annotations:
+ artifacthub.io/license: Apache-2.0
+ artifacthub.io/category: monitoring
diff --git a/charts/promforecast-metrics-adapter/templates/_helpers.tpl b/charts/promforecast-metrics-adapter/templates/_helpers.tpl
new file mode 100644
index 0000000..0a67390
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/_helpers.tpl
@@ -0,0 +1,131 @@
+{{/*
+Common helpers for the promforecast-metrics-adapter chart.
+*/}}
+
+{{- define "promforecast-metrics-adapter.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.labels" -}}
+helm.sh/chart: {{ include "promforecast-metrics-adapter.chart" . }}
+{{ include "promforecast-metrics-adapter.selectorLabels" . }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: promforecast
+app.kubernetes.io/component: metrics-adapter
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "promforecast-metrics-adapter.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+{{- default (include "promforecast-metrics-adapter.fullname" .) .Values.serviceAccount.name -}}
+{{- else -}}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "promforecast-metrics-adapter.image" -}}
+{{- $tag := default .Chart.AppVersion .Values.image.tag -}}
+{{- printf "%s:%s" .Values.image.repository $tag -}}
+{{- end -}}
+
+{{/*
+TLS certificate (self-signed when tls.generate=true).
+
+Stored in a ``hiddenSecret`` flavour so re-rendering on each
+``helm upgrade`` regenerates the cert. Stable across upgrades within
+the same release because the chart pins the CN to the service DNS
+name; only the validity period rolls.
+*/}}
+{{- define "promforecast-metrics-adapter.tlsSecretName" -}}
+{{- if .Values.tls.existingSecret -}}
+{{- .Values.tls.existingSecret -}}
+{{- else -}}
+{{- printf "%s-tls" (include "promforecast-metrics-adapter.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+TLS material generation.
+
+``genCA`` / ``genSignedCert`` are non-deterministic across calls,
+so the cert is generated *once* per Helm render and stashed on
+``$.Values`` for the duration of that render. Subsequent calls
+(e.g. one for the Secret, one for the APIService) pull the
+already-generated material instead of producing a fresh CA.
+
+On ``helm upgrade`` against a live cluster, ``lookup`` finds the
+chart-managed Secret from the previous install and the helper
+returns *that* material — so the cert + CA bundle stays stable
+across upgrades and the APIService caBundle doesn't churn (which
+would otherwise force a rolling restart on every upgrade via the
+``checksum/tls`` annotation and break in-flight HPA polls). The
+fresh-generate path only fires when ``lookup`` returns nothing:
+first install, or offline ``helm template`` (which is fine — the
+golden-file render is deterministic per process but irrelevant to
+a real cluster).
+
+Returns a dict with ``ca``, ``cert``, ``key`` (PEM strings). For
+``tls.existingSecret`` installs the helper uses ``lookup`` against
+the operator-named Secret and returns its data instead.
+*/}}
+{{- define "promforecast-metrics-adapter.tlsMaterial" -}}
+{{- $fullname := include "promforecast-metrics-adapter.fullname" . -}}
+{{- if .Values.tls.existingSecret -}}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.tls.existingSecret -}}
+{{- if $secret -}}
+{{- dict "ca" (index $secret.data "ca.crt" | b64dec) "cert" (index $secret.data "tls.crt" | b64dec) "key" (index $secret.data "tls.key" | b64dec) | toYaml -}}
+{{- else -}}
+{{- /*
+The operator set ``tls.existingSecret`` but the Secret isn't in the
+cluster (or we're rendering offline via ``helm template`` which
+returns nothing from ``lookup``). Failing loud here is the right
+default for a production install — the APIService would otherwise
+fall back to ``insecureSkipTLSVerify: true`` and silently accept any
+upstream cert, which defeats the existingSecret contract.
+
+For offline rendering (CI's ``helm template`` golden-file path),
+either pass ``--set tls.generate=true`` to exercise the self-signed
+path, or pre-create the Secret in a kind cluster and let the lookup
+populate.
+*/}}
+{{- fail (printf "tls.existingSecret=%q is set but Secret %q was not found in namespace %q. Either create the Secret first (it must carry ca.crt/tls.crt/tls.key), or unset existingSecret and use tls.generate=true. Note: ``helm template`` cannot resolve ``lookup`` against a live cluster; pass --set tls.generate=true to render offline." .Values.tls.existingSecret .Values.tls.existingSecret .Release.Namespace) -}}
+{{- end -}}
+{{- else -}}
+{{- if not (hasKey .Values "_tlsMaterial") -}}
+{{- $existing := lookup "v1" "Secret" .Release.Namespace (printf "%s-tls" $fullname) -}}
+{{- if and $existing (index $existing.data "ca.crt") (index $existing.data "tls.crt") (index $existing.data "tls.key") -}}
+{{- $_ := set .Values "_tlsMaterial" (dict "ca" (index $existing.data "ca.crt" | b64dec) "cert" (index $existing.data "tls.crt" | b64dec) "key" (index $existing.data "tls.key" | b64dec)) -}}
+{{- else -}}
+{{- $svcDNS := printf "%s.%s.svc" $fullname .Release.Namespace -}}
+{{- $svcDNSFull := printf "%s.%s.svc.cluster.local" $fullname .Release.Namespace -}}
+{{- $altNames := list $svcDNS $svcDNSFull $fullname -}}
+{{- $ca := genCA (printf "%s-ca" $fullname) (int .Values.tls.validityDays) -}}
+{{- $cert := genSignedCert $svcDNS nil $altNames (int .Values.tls.validityDays) $ca -}}
+{{- $_ := set .Values "_tlsMaterial" (dict "ca" $ca.Cert "cert" $cert.Cert "key" $cert.Key) -}}
+{{- end -}}
+{{- end -}}
+{{- .Values._tlsMaterial | toYaml -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/promforecast-metrics-adapter/templates/apiservice.yaml b/charts/promforecast-metrics-adapter/templates/apiservice.yaml
new file mode 100644
index 0000000..957d868
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/apiservice.yaml
@@ -0,0 +1,45 @@
+{{- if .Values.apiService.enabled -}}
+{{- $fullname := include "promforecast-metrics-adapter.fullname" . -}}
+{{- $tls := include "promforecast-metrics-adapter.tlsMaterial" . | fromYaml -}}
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+ name: v1beta1.external.metrics.k8s.io
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+spec:
+ service:
+ name: {{ $fullname }}
+ namespace: {{ .Release.Namespace }}
+ port: {{ .Values.service.port }}
+ group: external.metrics.k8s.io
+ version: v1beta1
+ versionPriority: 100
+ groupPriorityMinimum: 100
+ {{- if $tls.ca }}
+ caBundle: {{ $tls.ca | b64enc }}
+ {{- else }}
+ insecureSkipTLSVerify: true
+ {{- end }}
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+ name: v1beta1.custom.metrics.k8s.io
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+spec:
+ service:
+ name: {{ $fullname }}
+ namespace: {{ .Release.Namespace }}
+ port: {{ .Values.service.port }}
+ group: custom.metrics.k8s.io
+ version: v1beta1
+ versionPriority: 100
+ groupPriorityMinimum: 100
+ {{- if $tls.ca }}
+ caBundle: {{ $tls.ca | b64enc }}
+ {{- else }}
+ insecureSkipTLSVerify: true
+ {{- end }}
+{{- end }}
diff --git a/charts/promforecast-metrics-adapter/templates/configmap.yaml b/charts/promforecast-metrics-adapter/templates/configmap.yaml
new file mode 100644
index 0000000..75df965
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/configmap.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+data:
+ adapter.yaml: |
+ # promforecast-metrics-adapter configuration.
+ # Schema: docs/predictive-autoscaling.md
+ datasource_url: {{ .Values.datasourceUrl | quote }}
+ listen: ":6443"
+ query_timeout_seconds: 5.0
+ metrics:
+{{- range .Values.metrics }}
+ - external_metric_name: {{ .externalMetricName | quote }}
+ promql: {{ .promql | quote }}
+ namespaced: {{ default true .namespaced }}
+{{- end }}
diff --git a/charts/promforecast-metrics-adapter/templates/deployment.yaml b/charts/promforecast-metrics-adapter/templates/deployment.yaml
new file mode 100644
index 0000000..1abd618
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/deployment.yaml
@@ -0,0 +1,94 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+spec:
+ replicas: {{ .Values.replicaCount }}
+ selector:
+ matchLabels:
+ {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels:
+ {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 8 }}
+ annotations:
+ # Re-trigger pod rollout when the config or the cert
+ # changes — Helm's checksum/config + checksum/secret
+ # annotations are the standard pattern. The TLS material
+ # helper looks up the chart-managed Secret on upgrade so the
+ # cert is *stable* across `helm upgrade` and this checksum
+ # only flips on a real rotation (initial install, the Secret
+ # being deleted out-of-band, or ``existingSecret`` swapped to
+ # different material) — uvicorn loads ssl_certfile once at
+ # boot and never reloads, so the rolling restart is the only
+ # way to pick up a new cert.
+ checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+ checksum/tls: {{ include "promforecast-metrics-adapter.tlsMaterial" . | sha256sum }}
+ spec:
+ serviceAccountName: {{ include "promforecast-metrics-adapter.serviceAccountName" . }}
+ securityContext:
+ {{- toYaml .Values.securityContext | nindent 8 }}
+ containers:
+ - name: adapter
+ image: {{ include "promforecast-metrics-adapter.image" . }}
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args:
+ - adapter
+ - --config=/etc/promforecast-adapter/adapter.yaml
+ - --tls-cert=/var/run/promforecast-adapter/tls.crt
+ - --tls-key=/var/run/promforecast-adapter/tls.key
+ ports:
+ - name: https
+ containerPort: 6443
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ port: https
+ scheme: HTTPS
+ initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }}
+ periodSeconds: {{ .Values.probes.readiness.periodSeconds }}
+ timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }}
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: https
+ scheme: HTTPS
+ initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }}
+ periodSeconds: {{ .Values.probes.liveness.periodSeconds }}
+ timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }}
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
+ volumeMounts:
+ - name: config
+ mountPath: /etc/promforecast-adapter
+ readOnly: true
+ - name: tls
+ mountPath: /var/run/promforecast-adapter
+ readOnly: true
+ volumes:
+ - name: config
+ configMap:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}
+ - name: tls
+ secret:
+ secretName: {{ include "promforecast-metrics-adapter.tlsSecretName" . }}
+ items:
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
diff --git a/charts/promforecast-metrics-adapter/templates/rbac.yaml b/charts/promforecast-metrics-adapter/templates/rbac.yaml
new file mode 100644
index 0000000..e7f61cc
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/rbac.yaml
@@ -0,0 +1,53 @@
+{{- if .Values.rbac.create }}
+# Two bindings ship by default; nothing else. The kube-controller-manager
+# (which runs the HPA control loop) already holds permission on the
+# metrics aggregate APIs via the cluster's built-in
+# ``system:kube-controller-manager`` ClusterRole, and the API aggregator
+# forwards its identity through to this adapter after the
+# ``system:auth-delegator`` handshake below, so no extra ``metrics-reader``
+# ClusterRole / binding is needed for HPA to function. Workloads outside
+# the controller manager that need direct access to
+# ``external.metrics.k8s.io`` / ``custom.metrics.k8s.io`` (custom
+# controllers, Argo Rollouts AnalysisRuns) should bind their own
+# ServiceAccount to a ClusterRole they manage — keeping that policy
+# choice in the operator's hands rather than burying a non-obvious
+# default in this chart.
+#
+# The adapter is fronted by the Kubernetes API aggregator, which
+# forwards already-authenticated requests through. To validate the
+# forwarded tokens we bind the standard ``system:auth-delegator``
+# ClusterRole — the convention every custom-metrics adapter follows.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}-auth-delegator
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: system:auth-delegator
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+---
+# RoleBinding to extension-apiserver-authentication-reader so the
+# adapter can read the front-proxy CA from kube-system — required
+# for validating the aggregator's request signatures.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}-auth-reader
+ namespace: kube-system
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: extension-apiserver-authentication-reader
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/promforecast-metrics-adapter/templates/service.yaml b/charts/promforecast-metrics-adapter/templates/service.yaml
new file mode 100644
index 0000000..a5088ac
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "promforecast-metrics-adapter.fullname" . }}
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+spec:
+ type: {{ .Values.service.type }}
+ ports:
+ - port: {{ .Values.service.port }}
+ targetPort: {{ .Values.service.targetPort }}
+ protocol: TCP
+ name: https
+ selector:
+ {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 4 }}
diff --git a/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml b/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml
new file mode 100644
index 0000000..b8d614c
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml
@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }}
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+ {{- with .Values.serviceAccount.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/charts/promforecast-metrics-adapter/templates/tls-secret.yaml b/charts/promforecast-metrics-adapter/templates/tls-secret.yaml
new file mode 100644
index 0000000..15fad18
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/templates/tls-secret.yaml
@@ -0,0 +1,14 @@
+{{- if and .Values.tls.generate (not .Values.tls.existingSecret) -}}
+{{- $tls := include "promforecast-metrics-adapter.tlsMaterial" . | fromYaml -}}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "promforecast-metrics-adapter.tlsSecretName" . }}
+ labels:
+ {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }}
+type: kubernetes.io/tls
+data:
+ tls.crt: {{ $tls.cert | b64enc }}
+ tls.key: {{ $tls.key | b64enc }}
+ ca.crt: {{ $tls.ca | b64enc }}
+{{- end }}
diff --git a/charts/promforecast-metrics-adapter/values.schema.json b/charts/promforecast-metrics-adapter/values.schema.json
new file mode 100644
index 0000000..a8607ee
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/values.schema.json
@@ -0,0 +1,91 @@
+{
+ "$schema": "https://json-schema.org/draft-07/schema#",
+ "$id": "https://promforecast.io/schemas/metrics-adapter.values.v1.schema.json",
+ "title": "promforecast-metrics-adapter Helm values (v1)",
+ "description": "Schema for the Kubernetes custom-metrics adapter that exposes promforecast forecast metrics to HorizontalPodAutoscaler.",
+ "type": "object",
+ "additionalProperties": true,
+ "properties": {
+ "image": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "repository": { "type": "string", "minLength": 1 },
+ "tag": { "type": "string" },
+ "pullPolicy": {
+ "type": "string",
+ "enum": ["Always", "IfNotPresent", "Never"]
+ }
+ },
+ "required": ["repository"]
+ },
+ "replicaCount": { "type": "integer", "minimum": 1 },
+ "datasourceUrl": { "type": "string", "minLength": 1 },
+ "metrics": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "externalMetricName": {
+ "type": "string",
+ "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"
+ },
+ "promql": { "type": "string", "minLength": 1 },
+ "namespaced": { "type": "boolean" }
+ },
+ "required": ["externalMetricName", "promql"]
+ }
+ },
+ "tls": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "generate": { "type": "boolean" },
+ "existingSecret": { "type": "string" },
+ "validityDays": { "type": "integer", "minimum": 1 }
+ }
+ },
+ "apiService": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "enabled": { "type": "boolean" }
+ }
+ },
+ "resources": { "type": "object" },
+ "probes": { "type": "object" },
+ "service": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "type": { "type": "string", "enum": ["ClusterIP", "NodePort", "LoadBalancer"] },
+ "port": { "type": "integer", "minimum": 1, "maximum": 65535 },
+ "targetPort": { "type": "integer", "minimum": 1, "maximum": 65535 }
+ }
+ },
+ "serviceAccount": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "create": { "type": "boolean" },
+ "name": { "type": "string" },
+ "annotations": { "type": "object" }
+ }
+ },
+ "securityContext": { "type": "object" },
+ "rbac": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "create": { "type": "boolean" }
+ }
+ },
+ "nameOverride": { "type": "string" },
+ "fullnameOverride": { "type": "string" },
+ "nodeSelector": { "type": "object" },
+ "tolerations": { "type": "array" },
+ "affinity": { "type": "object" }
+ },
+ "required": ["image", "datasourceUrl"]
+}
diff --git a/charts/promforecast-metrics-adapter/values.yaml b/charts/promforecast-metrics-adapter/values.yaml
new file mode 100644
index 0000000..febe338
--- /dev/null
+++ b/charts/promforecast-metrics-adapter/values.yaml
@@ -0,0 +1,146 @@
+# promforecast-metrics-adapter — Helm values.
+#
+# The adapter implements the Kubernetes custom-metrics API contracts
+# (external.metrics.k8s.io/v1beta1 and custom.metrics.k8s.io/v1beta1) so
+# HorizontalPodAutoscaler can scale on promforecast forecast metrics.
+# Forecasts are still produced by the main `promforecast` chart and
+# stored in VictoriaMetrics via the sink; this adapter is a thin
+# read-side translator between HPA and PromQL.
+
+image:
+ repository: ghcr.io/esops-dev/promforecast
+ # tag: "" # Defaults to the chart's appVersion when empty.
+ pullPolicy: IfNotPresent
+
+# Single replica is the default. The adapter is stateless and a
+# restart is invisible to HPA (which retries on 503), so HA-style
+# multiple replicas are valid but rarely necessary. Use `replicaCount:
+# >1` only for shops with strict per-pod availability requirements.
+replicaCount: 1
+
+# Promforecast datasource URL the adapter reads from.
+#
+# IMPORTANT — OVERRIDE THIS unless you installed the umbrella chart
+# (``promforecast-stack``) into the ``monitoring`` namespace under the
+# release name ``promforecast-stack``. The default hardcodes both
+# pieces because Helm ``values.yaml`` cannot reference
+# ``.Release.Namespace``; if either differs in your install (a
+# ``monitoring-system`` namespace, a ``pf`` release name, an
+# externally-managed VictoriaMetrics) the adapter will resolve the
+# wrong service and every HPA poll returns 503.
+#
+# Conventional overrides:
+# --set datasourceUrl=http://vm.observability.svc.cluster.local:8428
+# --set datasourceUrl=http://victoria-metrics-cluster-vmselect.vm:8481/select/0/prometheus
+datasourceUrl: "http://promforecast-stack-victoria-metrics-single-server.monitoring.svc.cluster.local:8428"
+
+# Metric mappings — the meat of the chart. Each entry exposes one
+# forecast metric to HPA. Use the `external.metrics.k8s.io` API for
+# service-level forecasts (the common case); pair the configured
+# PromQL with `$namespace` / `$pod` placeholders if you want HPA's
+# namespace / labelSelector arguments to filter the query.
+#
+# Conventional pattern: define one external metric per service +
+# signal pair (e.g. payments-rps-forecast,
+# payments-p95-latency-forecast). HPA references the metric by name.
+metrics: []
+ # - externalMetricName: rps-forecast
+ # promql: 'sum by (namespace)(http_requests_rate_forecast{namespace="$namespace"})'
+ # namespaced: true
+
+# K8s API aggregator requires the adapter to serve TLS — the chart
+# generates a self-signed cert at install time and registers its CA
+# bundle with the APIService. cert-manager is not required.
+tls:
+ # When false the chart does not generate a cert; the operator
+ # provides one via the existingSecret reference instead. Useful
+ # in shops that already run cert-manager and want a managed cert.
+ generate: true
+ existingSecret: ""
+ # Validity of the self-signed cert in days. The chart re-generates
+ # the cert on each `helm upgrade` if `generate: true`, so a long
+ # validity is appropriate.
+ validityDays: 3650
+
+apiService:
+ # Whether to register an APIService for the external and custom
+ # metrics groups. Disable only if the cluster already has another
+ # metrics adapter registered (e.g. prometheus-adapter) and you're
+ # running this one alongside under a different group.
+ enabled: true
+
+resources:
+ requests:
+ cpu: 50m
+ memory: 64Mi
+ limits:
+ cpu: 500m
+ memory: 256Mi
+
+# Liveness / readiness probes against the adapter's /healthz
+# endpoint. Tight readiness keeps HPA from polling a not-yet-ready
+# replica during a rollout.
+probes:
+ readiness:
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ timeoutSeconds: 3
+ liveness:
+ initialDelaySeconds: 30
+ periodSeconds: 30
+ timeoutSeconds: 5
+
+service:
+ type: ClusterIP
+ port: 443
+ # The adapter listens on 6443 by convention (custom-metrics
+ # adapters share that port across implementations). The chart's
+ # service maps 443 -> 6443 so APIService consumers reach a
+ # standard HTTPS port.
+ targetPort: 6443
+
+serviceAccount:
+ create: true
+ name: ""
+ annotations: {}
+
+# Pod security context. The adapter doesn't need anything special
+# — runs as a non-root user, no privileged operations.
+securityContext:
+ runAsNonRoot: true
+ runAsUser: 65532
+ runAsGroup: 65532
+ fsGroup: 65532
+
+# NetworkPolicy — by default we don't ship one because the adapter
+# needs the K8s API aggregator (kube-apiserver) to reach it on its
+# TLS port, and that traffic is from arbitrary cluster IPs depending
+# on how kube-apiserver is deployed. Operators who want a
+# NetworkPolicy should pin it to their cluster's apiserver pod
+# selector or CIDR.
+#
+# IMPORTANT — authentication model. The adapter trusts the K8s API
+# aggregator to authenticate clients before forwarding requests; it
+# does NOT independently verify the forwarded identity via
+# TokenReview / SubjectAccessReview. Direct connections to the
+# Service:port that bypass the aggregator (in-cluster `curl` from
+# another pod, `kubectl port-forward`, a misconfigured ingress) read
+# the configured metrics without authentication. In multi-tenant
+# clusters pair this chart with a NetworkPolicy that restricts
+# ingress to the kube-apiserver only (cluster-specific selector;
+# omitted from defaults because the right selector is per-cluster).
+
+# RBAC scope — needed to authenticate clients via TokenReview/
+# SubjectAccessReview when the API aggregator forwards an
+# authenticated request through to us. The chart binds the
+# system:auth-delegator ClusterRole (the standard custom-metrics
+# adapter requirement).
+rbac:
+ create: true
+
+nameOverride: ""
+fullnameOverride: ""
+
+nodeSelector: {}
+tolerations: []
+affinity: {}
diff --git a/charts/promforecast/values.yaml b/charts/promforecast/values.yaml
index 708138c..5d1b0ea 100644
--- a/charts/promforecast/values.yaml
+++ b/charts/promforecast/values.yaml
@@ -60,6 +60,15 @@ config:
# exec`. The endpoint is unauthenticated like /metrics, so only
# enable it when the metrics port is on a trusted network.
expose_config_endpoint: false
+ # Off by default. Flip to true to expose `POST /forecast/what-if/fleet`
+ # which runs the preview machinery across every query whose
+ # `thresholds:` define a capacity ceiling and returns a ranked list
+ # of services projected to breach soonest under a hypothetical
+ # traffic multiplier. The endpoint is throttled by
+ # `safety.fleet_what_if_min_interval` so polling dashboards can't
+ # starve scheduled fits. Same network-level access-control model as
+ # /metrics and /-/reload.
+ expose_what_if_endpoint: false
safety:
max_series_per_query: 500
@@ -67,6 +76,13 @@ config:
series_overflow: drop_lowest_priority
fit_timeout: 60s
query_timeout: 30s
+ # Minimum interval between successive fleet what-if runs. The
+ # endpoint refits every threshold-bearing query against the
+ # requested traffic multiplier; a single invocation is O(minutes)
+ # on a medium config so unthrottled access would starve scheduled
+ # fits. The default of 5m matches a typical `refresh_interval`.
+ # Has no effect unless `server.expose_what_if_endpoint` is true.
+ fleet_what_if_min_interval: 5m
defaults:
lookback: 14d
diff --git a/dashboards/grafana/promforecast-incident-heatmap.json b/dashboards/grafana/promforecast-incident-heatmap.json
new file mode 100644
index 0000000..c793815
--- /dev/null
+++ b/dashboards/grafana/promforecast-incident-heatmap.json
@@ -0,0 +1,149 @@
+{
+ "title": "promforecast — Incident recurrence heatmap",
+ "uid": "promforecast-incident-heatmap",
+ "description": "Per-(day_of_week, hour_of_day) intensity of recurring incidents for queries opted into `data_profile: events`. Score normalised against the average non-empty bucket; values > 1.0 indicate above-average concentration in that bucket. Pair the heatmap with the interpretation guide in docs/outputs/incident-patterns.md — high score + low variance = preemptive action warranted; high score + high variance = pattern is trending up.",
+ "schemaVersion": 39,
+ "version": 1,
+ "editable": true,
+ "tags": ["promforecast", "incidents", "patterns"],
+ "timezone": "",
+ "time": { "from": "now-30d", "to": "now" },
+ "refresh": "1h",
+ "templating": {
+ "list": [
+ {
+ "name": "datasource",
+ "type": "datasource",
+ "query": "prometheus",
+ "current": { "text": "VictoriaMetrics", "value": "victoriametrics" }
+ },
+ {
+ "name": "group",
+ "label": "Group",
+ "type": "query",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "query": "label_values(forecast_incident_recurrence_score, group)",
+ "refresh": 2,
+ "includeAll": false,
+ "multi": false
+ },
+ {
+ "name": "query",
+ "label": "Query",
+ "type": "query",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "query": "label_values(forecast_incident_recurrence_score{group=\"$group\"}, id)",
+ "refresh": 2,
+ "includeAll": false,
+ "multi": false
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "heatmap",
+ "id": 1,
+ "title": "Incident recurrence intensity — $query ($group)",
+ "description": "Each cell is one (day_of_week, hour_of_day) bucket. Score = bucket event count / mean non-empty bucket count. > 1.0 (warm colour) means this bucket sees more incidents than average. Read columns to find time-of-day patterns; read rows to find day-of-week patterns.",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "gridPos": { "h": 14, "w": 24, "x": 0, "y": 0 },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"}",
+ "instant": true,
+ "format": "heatmap"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "scaleDistribution": { "type": "linear" }
+ },
+ "color": { "mode": "continuous-RdYlGr" },
+ "min": 0,
+ "decimals": 2
+ },
+ "overrides": []
+ },
+ "options": {
+ "calculate": false,
+ "yAxis": { "axisLabel": "day_of_week (UTC; 0=Mon, 6=Sun)" },
+ "color": { "mode": "scheme", "scheme": "RdYlGn", "reverse": true, "steps": 64 },
+ "exemplars": { "color": "rgba(255,0,0,0.7)" },
+ "cellGap": 1,
+ "showValue": "never",
+ "tooltip": { "show": true, "yHistogram": false }
+ }
+ },
+ {
+ "type": "stat",
+ "id": 2,
+ "title": "Top bucket — $query",
+ "description": "The single most-concentrated (day_of_week, hour_of_day) bucket. Pair with the heatmap to focus on the highest-risk window.",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "gridPos": { "h": 5, "w": 8, "x": 0, "y": 14 },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "topk(1, forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"})",
+ "instant": true,
+ "legendFormat": "Day {{day_of_week}} Hour {{hour_of_day}}"
+ }
+ ],
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal",
+ "colorMode": "value",
+ "graphMode": "none",
+ "textMode": "value_and_name"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "none",
+ "decimals": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "green", "value": null },
+ { "color": "yellow", "value": 1.0 },
+ { "color": "orange", "value": 2.0 },
+ { "color": "red", "value": 4.0 }
+ ]
+ }
+ },
+ "overrides": []
+ }
+ },
+ {
+ "type": "stat",
+ "id": 3,
+ "title": "Active buckets",
+ "description": "How many of the 168 (day, hour) buckets had at least one observed incident. A small count means activity is concentrated (worth preemptive action); a large count means activity is diffuse.",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "gridPos": { "h": 5, "w": 8, "x": 8, "y": 14 },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "count(forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"} > 0)",
+ "instant": true,
+ "legendFormat": "non-zero buckets"
+ }
+ ],
+ "options": {
+ "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+ "orientation": "horizontal",
+ "colorMode": "value",
+ "graphMode": "none",
+ "textMode": "value_and_name"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "decimals": 0
+ },
+ "overrides": []
+ }
+ }
+ ]
+}
diff --git a/dashboards/grafana/promforecast-is-it-working.json b/dashboards/grafana/promforecast-is-it-working.json
index 250fa9a..3210dd3 100644
--- a/dashboards/grafana/promforecast-is-it-working.json
+++ b/dashboards/grafana/promforecast-is-it-working.json
@@ -272,6 +272,61 @@
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] }
]
}
+ },
+ {
+ "type": "table",
+ "id": 30,
+ "title": "Fleet capacity stress test — ranked by projected breach",
+ "description": "Ranks every threshold-bearing series by the projected time-to-threshold under *current* load. Pair with `POST /forecast/what-if/fleet` (opt-in via server.expose_what_if_endpoint) to compute the same ranking under a hypothetical traffic multiplier (`{\"traffic_multiplier\": 1.5, \"horizon\": \"90d\"}`) — the endpoint returns the same shape, but with the multiplier applied. The throttle is set by safety.fleet_what_if_min_interval (5m default).",
+ "datasource": { "type": "prometheus", "uid": "${datasource}" },
+ "gridPos": { "h": 9, "w": 24, "x": 0, "y": 27 },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sort({__name__=~\".+_forecast_time_to_threshold_seconds\"})",
+ "format": "table",
+ "instant": true,
+ "legendFormat": "{{__name__}} {{name}}"
+ }
+ ],
+ "transformations": [
+ { "id": "organize",
+ "options": {
+ "excludeByName": { "Time": true, "job": true, "instance": true, "cluster": true, "replica": true },
+ "renameByName": {
+ "Value": "time_to_threshold_seconds",
+ "__name__": "metric",
+ "name": "threshold",
+ "group": "group",
+ "id": "query",
+ "model": "model",
+ "severity": "severity"
+ }
+ }
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": { "align": "left" },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ { "color": "red", "value": null },
+ { "color": "yellow", "value": 3600 },
+ { "color": "green", "value": 86400 }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": { "id": "byName", "options": "time_to_threshold_seconds" },
+ "properties": [
+ { "id": "unit", "value": "s" },
+ { "id": "custom.cellOptions", "value": { "type": "color-text" } }
+ ]
+ }
+ ]
+ }
}
]
}
diff --git a/docs/README.md b/docs/README.md
index 99a5b1a..4ed020d 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -37,7 +37,11 @@ metrics, calibration, explainability, and ad-hoc what-if endpoints.
- [Explainability (decomposition)](outputs/explainability.md) — trend / seasonal / level / residual split of the forecast curve
- [Forecast drift](outputs/drift.md) — prediction-vs-prediction shift telemetry
- [Forecast preview](outputs/preview.md) — opt-in ad-hoc what-if endpoint
+- [Fleet capacity what-if](outputs/what-if-fleet.md) — multi-service stress-test endpoint that ranks projected breaches under a hypothetical traffic multiplier
+- [Deploy risk score](outputs/deploy-safety.md) — deploy-anchored composite of deviation across golden signals; pair with Argo Rollouts / Flagger as a rollback gate
+- [Recurring incident patterns](outputs/incident-patterns.md) — 7 x 24 heatmap of when incidents tend to land; for queries with `data_profile: events`
- [Canary forecasts](outputs/canary.md)
+- [Predictive autoscaling](predictive-autoscaling.md) — Kubernetes custom-metrics adapter that exposes forecast metrics to HorizontalPodAutoscaler for scaling on the projected load curve
## Operations
diff --git a/docs/operations/cardinality.md b/docs/operations/cardinality.md
index db34cfc..7acaebd 100644
--- a/docs/operations/cardinality.md
+++ b/docs/operations/cardinality.md
@@ -48,6 +48,8 @@ Notation:
| `forecast_conditional_band_calibration_offset` | `S × M × L` | `emission.conditional: true` *and* `accuracy.conformal.enabled: true` |
| SLO error budget | `S × M × N_windows × (3 + L)` | `slo:` configured (3 sibling families × N_windows, plus level fan-out on exhausted_in) |
| Recommended maintenance windows | `S × M × count` | `recommended_windows.enabled: true` |
+| `forecast_deploy_risk_score` | `max_active_deploys` per service | One row per active deploy while inside `active_window`; bounded per service by `deploy_signals[].max_active_deploys`. Charged against the *global* budget, not the per-query series cap. |
+| `forecast_incident_recurrence_score` | `168` per opted-in query | Fixed 7 days × 24 hours grid per query with `data_profile: events`. Independent of `S` — the heatmap is the aggregate across whatever the counter PromQL fans out into. |
| Operational + preprocessing families | ~7 per group + ~6 per query | Mostly always on |
## Working example
diff --git a/docs/outputs/deploy-safety.md b/docs/outputs/deploy-safety.md
new file mode 100644
index 0000000..7ff3d03
--- /dev/null
+++ b/docs/outputs/deploy-safety.md
@@ -0,0 +1,100 @@
+# Deploy risk score
+
+`forecast_deploy_risk_score` turns multiple golden-signal deviation ratios into a single deploy-anchored risk number.
+Instead of manually correlating latency, errors, saturation, and traffic during a rollout, you get one gauge per active deploy that SREs can alert on.
+
+The metric only exists while a deploy is inside its **active window** — background noise never pollutes dashboards.
+
+## Configuration
+
+```yaml
+deploy_signals:
+ - service: payments-api
+ anchor:
+ type: regressor
+ query: payments_request_rate
+ regressor_id: deploys
+ threshold: 0 # transition from <=0 to >0
+ signals:
+ - query: payments_request_rate
+ weight: 0.5
+ - query: payments_p95_latency
+ weight: 1.0
+ - query: payments_error_rate
+ weight: 2.0
+ active_window: 30m
+ max_active_deploys: 8
+```
+
+| Field | Default | Meaning |
+|------------------|---------|---------|
+| `service` | — | Human-readable service name |
+| `anchor` | — | Detects deploy start (via regressor transition) |
+| `signals` | — | Weighted golden signals (`errors ≥ latency ≥ saturation ≥ traffic` is conventional) |
+| `active_window` | — | How long the score stays emitted after detection |
+| `max_active_deploys` | `8` | LRU cap per service to bound cardinality |
+
+## Emitted metric
+
+```
+forecast_deploy_risk_score{
+ service="payments-api",
+ deployment_id="payments-api-1715000000",
+ contributing_signals="3"
+} 0.42
+```
+
+- **Value** — weighted mean of absolute `forecast_deviation_ratio` across contributing signals (higher = more risk).
+- `deployment_id` — stable identifier for the current deploy.
+- `contributing_signals` — number of signals that produced a finite deviation this cycle.
+
+## Anchor detection
+
+The anchor watches the configured regressor on the anchor query.
+A deploy is registered when the leading value crosses `threshold` from ≤ threshold to > threshold.
+A persistently high regressor counts as **one** deploy, not repeated triggers.
+
+Typical anchor regressor:
+
+```yaml
+regressors:
+ - id: deploys
+ promql: changes(kube_deployment_status_observed_generation{deployment="payments-api"}[5m])
+```
+
+## Worked examples
+
+**Argo Rollouts**
+Expose rollout phase changes as a counter and reference it as the anchor.
+Argo’s analysis backend can read `forecast_deploy_risk_score` directly and mark the rollout `Failed` if the score exceeds your threshold.
+
+**Flagger**
+Define a `MetricTemplate` that queries the risk score and attach it to your Canary resource.
+A sustained high score triggers automatic rollback.
+
+## Cardinality
+
+At most `max_active_deploys` rows per service at any time.
+With 10 services the worst-case total is ~80 rows — safely bounded by LRU eviction.
+
+## Example alerts
+
+```promql
+# High risk during rollout
+forecast_deploy_risk_score{contributing_signals!~"0|1"} > 0.5
+
+# Critical risk
+forecast_deploy_risk_score{contributing_signals!~"0|1"} > 1.0
+```
+
+## Operational notes
+
+- With HA enabled, only the leader computes scores (followers serve the cached snapshot).
+- Config reloads reset the tracker; active deploys disappear until re-detected.
+- Score is computed from the current snapshot’s deviation rows — a deploy registered before the first signal fit will temporarily show `contributing_signals: 0`.
+
+## See also
+
+- [Deviation metrics](deviation.md) — the signals the score is built from
+- [Drift telemetry](drift.md) — distinct from deploy-anchored risk
+- [Canary / weighted groups](canary.md) — single-metric alternative when you have only one golden signal
diff --git a/docs/outputs/incident-patterns.md b/docs/outputs/incident-patterns.md
new file mode 100644
index 0000000..87dc5fb
--- /dev/null
+++ b/docs/outputs/incident-patterns.md
@@ -0,0 +1,90 @@
+# Recurring incident pattern detection
+
+`forecast_incident_recurrence_score` turns a stream of discrete incident events into a **7×24 heatmap** of recurrence intensity.
+It answers “when do incidents tend to happen?” so you can proactively staff on-call or harden systems around predictable hot spots.
+
+This is discrete-event forecasting (not continuous metrics) and sits at the boundary with the sibling `promanomaly` project.
+
+## Configuration
+
+Mark the query with `data_profile: events`:
+
+```yaml
+groups:
+ - name: incidents
+ queries:
+ - id: pagerduty_incidents_total
+ promql: sum(pagerduty_incidents_total)
+ data_profile: events
+```
+
+The PromQL must return a counter that increments on each incident. The runner aggregates all returned series into a single event stream before bucketing.
+
+Typical sources:
+- `pagerduty_incidents_total`
+- `alertmanager_notifications_total{severity="critical"}`
+- A custom webhook counter
+
+## Emitted metric
+
+A full 168-cell grid is always published:
+
+```
+forecast_incident_recurrence_score{
+ group="incidents",
+ id="pagerduty_incidents_total",
+ day_of_week="0", # 0 = Monday … 6 = Sunday
+ hour_of_day="14"
+} 2.5
+```
+
+- Value = events in this bucket ÷ mean events across all non-empty buckets.
+ `> 1.0` = above-average concentration; `0` = never observed.
+
+## Interpretation guide
+
+| Score | Meaning |
+|-----------|--------------------------------------|
+| `0` | No events observed |
+| `< 1.0` | Below-average activity |
+| `1.0` | Exactly average |
+| `1.5–3.0` | 50–200 % above average |
+| `> 3.0` | Strong, reliable hot spot |
+
+Look at both the score **and** the overall distribution shape:
+- Few high-score buckets → predictable pattern (act on it).
+- Many low-score buckets → random noise (descriptive only).
+
+## Worked examples
+
+**PagerDuty**
+```yaml
+- id: pagerduty_incidents_total
+ promql: sum(pagerduty_incidents_total{service=~"$service_pattern"})
+ data_profile: events
+```
+
+**Alertmanager**
+```yaml
+- id: alertmanager_critical_notifications
+ promql: sum(alertmanager_notifications_total{severity="critical"})
+ data_profile: events
+```
+
+**Custom webhook**
+Increment a Prometheus `Counter("incidents_total")` in your webhook handler and scrape it normally.
+
+## Operational notes
+
+- No forecast curve, bands, or deviation metrics are emitted — only the heatmap.
+- Fixed cardinality: exactly **168 rows** per opted-in query (counted by `promforecast validate`).
+- Time-of-day is always UTC.
+- Counter resets (negative deltas) are treated as zero events.
+
+The bundled `promforecast-incident-heatmap.json` dashboard renders the heatmap with group/query selectors and includes “Active buckets” + “Top bucket” stat panels for quick triage.
+
+## Cross-tool overlay (roadmap)
+
+A future dashboard will overlay this recurrence score with `promanomaly` outputs to answer “did the pattern hold this week, and did the anomaly detector agree?”.
+
+Until then, the heatmap stands alone as a preemptive-staffing signal.
diff --git a/docs/outputs/what-if-fleet.md b/docs/outputs/what-if-fleet.md
new file mode 100644
index 0000000..fb525f2
--- /dev/null
+++ b/docs/outputs/what-if-fleet.md
@@ -0,0 +1,110 @@
+# Fleet capacity what-if endpoint
+
+`POST /forecast/what-if/fleet` answers the **strategic** question:
+“If traffic grows by X% over the next N days, which services will hit their capacity ceilings first?”
+
+It re-uses the single-query preview machinery internally, looping over every query that defines a capacity threshold (`gt` or `lt` comparators only) and ranking the projected ETAs.
+
+## Enabling
+
+```yaml
+server:
+ expose_what_if_endpoint: true # default: false
+
+safety:
+ fleet_what_if_min_interval: 5m # default: 5m
+```
+
+The endpoint follows the same network-level access model as `/forecast/preview` and `/-/reload`.
+
+A rate limit prevents the endpoint from starving scheduled fits. Requests that hit the throttle return `429 Too Many Requests` with a `Retry-After` header and this JSON body:
+
+```json
+{
+ "error": "throttled",
+ "message": "fleet what-if throttled; retry after 287s",
+ "retry_after_seconds": 287.4
+}
+```
+
+## Request body
+
+```json
+{
+ "traffic_multiplier": 1.5,
+ "horizon": "90d",
+ "groups": ["payments", "search"]
+}
+```
+
+| Field | Required | Notes |
+|----------------------|----------|-------|
+| `traffic_multiplier` | Yes | Positive number (`1.0` = baseline ranking) |
+| `horizon` | No | Prometheus duration or seconds; defaults to `defaults.horizon` |
+| `groups` | No | Restrict scan to listed groups (unknown groups → 400) |
+
+The multiplier is applied by:
+1. **Regressor scaling** (preferred) — if the query has a traffic regressor (`traffic`, `rps`, `qps`, `request_rate`, etc.), the value is overridden.
+2. **Baseline fallback** — otherwise the preview runs unchanged and the response row flags `traffic_multiplier_applied: "baseline"`.
+
+## Response
+
+```json
+{
+ "horizon_seconds": 7776000,
+ "traffic_multiplier": 1.5,
+ "served_by": "promforecast-0",
+ "evaluated_query_count": 24,
+ "services": [
+ {
+ "query_id": "payments_db_storage_used_bytes",
+ "group": "infra",
+ "threshold_name": "warning",
+ "threshold_value": 858993459200,
+ "comparator": "gt",
+ "time_to_threshold_seconds": 86400.0,
+ "model": "AutoARIMA",
+ "traffic_multiplier_applied": "regressor"
+ }
+ ]
+}
+```
+
+- Rows are ranked by ascending `time_to_threshold_seconds` (sooner breaches first).
+- Finite ETAs sort ahead of non-finite (`NaN` = no breach within horizon).
+- Failed previews appear with an `"error"` field but do not break the rest of the ranking.
+
+## Worked examples
+
+**Q4 capacity planning**
+```bash
+curl -s -X POST http://promforecast:9091/forecast/what-if/fleet \
+ -H 'content-type: application/json' \
+ -d '{"traffic_multiplier": 1.4, "horizon": "90d"}' \
+ | jq '.services[] | select(.time_to_threshold_seconds < 7776000)'
+```
+
+**Post-acquisition absorption**
+```bash
+curl -s -X POST ... \
+ -d '{"traffic_multiplier": 1.3, "horizon": "90d", "groups": ["api", "ingest"]}'
+```
+
+**Black Friday peak simulation**
+```bash
+curl -s -X POST ... \
+ -d '{"traffic_multiplier": 3.0, "horizon": "7d"}' \
+ | jq '.services[] | select(.time_to_threshold_seconds < 86400)'
+```
+
+## Operational notes
+
+- Throttle is per-replica (in-memory). In HA mode each replica enforces its own limit.
+- The endpoint is stateless and emits no metrics.
+- The bundled “Is it working?” dashboard includes a *Fleet capacity stress test* table that shows the same shape — pair it with a notebook curl for baseline vs. overlay comparisons.
+
+## See also
+
+- [`preview.md`](preview.md) — single-query what-if this endpoint composes over
+- [`capacity.md`](capacity.md) — the underlying `time_to_threshold_seconds` metric
+- [`maintenance-windows.md`](maintenance-windows.md) — ranked quiet windows instead of ranked services
diff --git a/docs/predictive-autoscaling.md b/docs/predictive-autoscaling.md
new file mode 100644
index 0000000..fe4dbfb
--- /dev/null
+++ b/docs/predictive-autoscaling.md
@@ -0,0 +1,146 @@
+# Predictive autoscaling with Kubernetes HPA
+
+The `promforecast-metrics-adapter` sub-component exposes promforecast forecasts through the Kubernetes custom-metrics API.
+HorizontalPodAutoscaler can then scale **on the forecast** of load 15–30 minutes ahead instead of reacting to current load — capacity arrives *before* the spike.
+
+The adapter is a separate deployable (own Helm chart and container entrypoint). The main forecaster is unaffected.
+
+## Architecture
+
+```mermaid
+flowchart TD
+ F["promforecast forecaster"]
+ VM[("VictoriaMetrics
long-term TSDB
(forecasts via sink)")]
+ A["promforecast-metrics-adapter
(this sub-chart)
• external.metrics.k8s.io
• custom.metrics.k8s.io
• serves TLS"]
+ K["kube-apiserver
aggregator front"]
+ H["HorizontalPodAutoscaler"]
+
+ F -->|remote_write forecasts| VM
+ VM -->|PromQL| A
+ A -->|TLS, via APIService| K
+ K -->|polls| H
+```
+
+Stateless: the adapter only reads forecasts that the main forecaster has already written to VictoriaMetrics via the sink.
+
+## Install
+
+```bash
+helm install promforecast-adapter ./charts/promforecast-metrics-adapter \
+ --namespace monitoring \
+ --set datasourceUrl=http://victoriametrics:8428 \
+ --values - <<'EOF'
+metrics:
+ - externalMetricName: rps-forecast
+ promql: 'sum by (service)(http_requests_rate_forecast{service="$service"})'
+ namespaced: true
+ - externalMetricName: cpu-forecast
+ promql: 'avg(node_cpu_busy_pct_forecast)'
+ namespaced: false
+EOF
+```
+
+The chart creates a self-signed cert, registers the two required `APIService` objects, and sets up RBAC. No cert-manager needed.
+
+## Config reference
+
+```yaml
+# values.yaml
+datasourceUrl: http://victoriametrics:8428
+
+metrics:
+ - externalMetricName: rps-forecast
+ promql: |
+ sum by (service)(
+ http_requests_rate_forecast{service="$service"}
+ )
+ namespaced: true
+
+tls:
+ generate: true # self-signed cert
+ validityDays: 3650
+ # existingSecret: my-adapter-tls # optional
+```
+
+**Placeholders** in `promql` are substituted from the HPA’s label selector. Namespaced metrics automatically get `namespace=$namespace`.
+
+## HPA example
+
+```yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: payments-api-predictive
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: payments-api
+ minReplicas: 3
+ maxReplicas: 30
+ metrics:
+ - type: External
+ external:
+ metric:
+ name: rps-forecast
+ selector:
+ matchLabels:
+ service: payments-api
+ target:
+ type: AverageValue
+ averageValue: "50"
+ behavior:
+ scaleUp:
+ stabilizationWindowSeconds: 0
+ scaleDown:
+ stabilizationWindowSeconds: 600
+```
+
+A full example ships in `examples/k8s/predictive-hpa.yaml`.
+
+## Quality gate
+
+Forecast quality is a hard prerequisite for safe predictive scaling.
+Common pattern: run **two HPAs** on the same target:
+
+- **Primary** — predictive HPA (the one above).
+- **Fallback** — classic reactive HPA on CPU / request rate with a higher target.
+
+Kubernetes `selectPolicy: Max` ensures the more aggressive recommendation wins. The predictive HPA usually leads; the reactive one is the safety net.
+
+For tighter control you can gate the predictive HPA on `forecast_quality_score > 0.6` (small operator or custom metric).
+
+## Failure modes
+
+| State | Adapter response | HPA sees |
+|--------------------------------|---------------------------|---------------------------|
+| VM unreachable | 503 ServiceUnavailable | Metric unavailable |
+| Metric not found | 404 NotFound | `failedGetMetric` |
+| PromQL returns no series | 200 with value `0` | Treated as zero |
+| Forecast is NaN / Inf | 200 with value `0` | Treated as zero |
+
+Use `clamp_min(..., 0.001)` in the PromQL if you need to distinguish “no data” from “real zero”.
+
+## Verification
+
+```bash
+# Check pod and APIServices
+kubectl -n monitoring get pods -l app.kubernetes.io/name=promforecast-metrics-adapter
+kubectl get apiservice v1beta1.external.metrics.k8s.io
+
+# Test the metric directly
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/payments/rps-forecast" | jq .
+```
+
+## Operational notes
+
+- **TLS cert rotation** — the chart reuses the existing Secret on `helm upgrade`. To force rotation, delete the `-tls` Secret.
+- **No hot-reload** — metric mappings are part of the Kubernetes API surface, so changes require a Deployment rollout.
+- **Compatibility** — if you already run prometheus-adapter, set `apiService.enabled: false` on this chart.
+- **Single replica is sufficient** — the adapter is stateless and HPA retries on 503.
+- **Security** — the adapter trusts the kube-apiserver aggregator. Use a NetworkPolicy pinned to the apiserver pods in multi-tenant clusters.
+
+See also:
+- [`preview.md`](preview.md) — single-query what-if
+- [`capacity.md`](capacity.md) — the underlying forecast metrics
+- [`maintenance-windows.md`](maintenance-windows.md) — ranked quiet windows
diff --git a/examples/alerts/promforecast-rules.yaml b/examples/alerts/promforecast-rules.yaml
index bcfd15d..8139eff 100644
--- a/examples/alerts/promforecast-rules.yaml
+++ b/examples/alerts/promforecast-rules.yaml
@@ -404,3 +404,46 @@ spec:
above 4h. The queue is being worked off, but the
backlog is too large to clear before downstream SLAs
are at risk. Add consumer capacity or shed load.
+
+ # ----------------------------------------------------------------
+ # Deploy-anchored risk score alerts.
+ # ----------------------------------------------------------------
+ # These rules assume the ``deploy_signals:`` top-level config
+ # block is populated with at least one service. The forecaster
+ # emits ``forecast_deploy_risk_score{deployment_id, service,
+ # contributing_signals}`` per active deploy; an empty
+ # ``deploy_signals`` produces no series and the alerts stay
+ # quiet. ``contributing_signals >= 2`` keeps the alert from
+ # firing on single-signal blips.
+ - name: promforecast.deploy.examples
+ interval: 1m
+ rules:
+ - alert: DeployHighRisk
+ expr: |
+ forecast_deploy_risk_score{contributing_signals!~"0|1"} > 0.5
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Deploy {{ $labels.deployment_id }} on {{ $labels.service }} shows elevated risk"
+ description: |
+ The composite of |forecast_deviation_ratio| across the
+ configured signals for {{ $labels.service }} exceeded
+ 0.5 for 5m after a deploy. Consider rolling back if
+ the deviation persists or worsens — see DeployCriticalRisk.
+
+ - alert: DeployCriticalRisk
+ expr: |
+ forecast_deploy_risk_score{contributing_signals!~"0|1"} > 1.0
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: "Deploy {{ $labels.deployment_id }} on {{ $labels.service }} is materially degrading signals"
+ description: |
+ The composite of |forecast_deviation_ratio| across the
+ configured signals for {{ $labels.service }} exceeded
+ 1.0 for 2m — at least one signal is a full band-width
+ outside its forecast. Roll back unless the deploy is
+ expected to shift the metric (in which case retrain
+ the model with the new regime).
diff --git a/examples/configs/deploy-and-incidents.yaml b/examples/configs/deploy-and-incidents.yaml
new file mode 100644
index 0000000..fdf562c
--- /dev/null
+++ b/examples/configs/deploy-and-incidents.yaml
@@ -0,0 +1,113 @@
+# Example forecaster config exercising the two predictive-ops emission
+# families:
+#
+# 1. ``deploy_signals:`` — per-service composite risk score anchored at
+# each rollout. Pair with the ``DeployHighRisk`` / ``DeployCriticalRisk``
+# example alerts in examples/alerts/promforecast-rules.yaml.
+#
+# 2. ``data_profile: events`` — incident-pattern detection. The runner
+# routes the query to the 7×24 recurrence heatmap path instead of
+# fitting a forecast curve. Pair with the
+# ``promforecast-incident-heatmap`` Grafana dashboard.
+#
+# Both families are off unless explicitly configured. Adapt the metric
+# names and label selectors to your environment.
+
+apiVersion: promforecast.io/v1
+
+datasource:
+ url: http://victoriametrics:8428/
+
+# All queries below aggregate to ~1 series per service via ``sum by``.
+# Lower the per-query cap accordingly so ``promforecast validate``'s
+# cardinality estimate reflects reality rather than the worst case
+# ``safety.max_series_per_query`` default (500). Operators with
+# higher fan-out (per-pod / per-instance breakdowns) should raise this.
+safety:
+ max_series_per_query: 10
+
+defaults:
+ lookback: 14d
+ step: 5m
+ horizon: 6h
+ models: [AutoARIMA, SeasonalNaive]
+ season_length: 288
+ accuracy:
+ evaluate: false
+ auto_select: true
+ fallback_model: SeasonalNaive
+
+groups:
+ - name: deploy_safety
+ priority: 1
+ queries:
+ # Latency (p95). Used by the deploy-risk composite below.
+ - id: payments_p95_latency_seconds
+ promql: |
+ histogram_quantile(
+ 0.95,
+ sum by (service, le) (rate(http_request_duration_seconds_bucket{service="payments"}[5m]))
+ )
+ regressors:
+ # The ``deploys`` regressor below is what the deploy-risk
+ # block anchors against. ``changes(...)`` ticks once per
+ # rollout that bumps the observed generation.
+ - id: deploys
+ promql: |
+ changes(kube_deployment_status_observed_generation{deployment="payments"}[5m])
+
+ # Error rate. Second contributor to the composite.
+ - id: payments_5xx_per_second
+ promql: |
+ sum by (service) (rate(http_requests_total{service="payments", status=~"5.."}[5m]))
+
+ # Saturation proxy (CPU). Third contributor.
+ - id: payments_cpu_seconds_rate
+ promql: |
+ sum by (service) (rate(container_cpu_usage_seconds_total{pod=~"payments-.*"}[5m]))
+
+ - name: incidents
+ priority: 2
+ queries:
+ # PagerDuty incidents bucketed into a 7×24 recurrence heatmap.
+ # Counter resets (target restart) are dropped to zero — see
+ # promforecast.incident_pattern.counter_to_event_counts.
+ - id: pagerduty_incidents_total
+ promql: pagerduty_incidents_total{service=~"payments|checkout|search"}
+ data_profile: events
+
+ # Alertmanager critical notifications, same treatment. Pairs
+ # well with the heatmap dashboard's ``query`` template selector.
+ - id: alertmanager_critical_notifications
+ promql: alertmanager_notifications_total{severity="critical"}
+ data_profile: events
+
+# ----------------------------------------------------------------------
+# Per-service deploy-anchored risk composite.
+#
+# Cardinality: ``max_active_deploys`` rows per service while at least
+# one deploy is inside its ``active_window``. Outside the window the
+# rows disappear from /metrics — the gauge is silent at steady state.
+#
+# Weights are conventionally latency >= traffic >= errors >= saturation
+# (latency moves first under a bad deploy); operators tune per service.
+# ----------------------------------------------------------------------
+deploy_signals:
+ - service: payments
+ anchor:
+ type: regressor
+ query: payments_p95_latency_seconds
+ regressor_id: deploys
+ threshold: 0
+ signals:
+ - query: payments_p95_latency_seconds
+ weight: 2.0
+ - query: payments_5xx_per_second
+ weight: 1.5
+ - query: payments_cpu_seconds_rate
+ weight: 1.0
+ # How long after a detected rollout the gauge stays live.
+ # 30 minutes is the conventional window for Argo Rollouts / Flagger
+ # analysis runs; tune to your rollout duration.
+ active_window: 30m
+ max_active_deploys: 4
diff --git a/examples/k8s/predictive-hpa.yaml b/examples/k8s/predictive-hpa.yaml
new file mode 100644
index 0000000..53dc190
--- /dev/null
+++ b/examples/k8s/predictive-hpa.yaml
@@ -0,0 +1,71 @@
+# Example: predictive HPA scaling on a promforecast forecast metric.
+#
+# The HPA references the external metric ``rps-forecast`` — the
+# metric name as registered with the promforecast-metrics-adapter
+# (see the adapter chart's ``metrics:`` array). The forecast value
+# is delivered to HPA via the Kubernetes custom-metrics API
+# aggregator; the adapter resolves it by querying VictoriaMetrics.
+#
+# **Quality gate.** A bad forecast can produce a runaway scale-up
+# or scale-down recommendation. Pair the predictive metric with a
+# ``forecast_quality_score`` gate so the HPA only acts when the
+# underlying forecast is trustworthy. The conventional pattern is a
+# second HPA scaling on a different signal as a fallback (e.g. raw
+# CPU) so the predictive HPA is the primary and the reactive HPA is
+# the safety net.
+#
+# **Stabilization window.** The ``behavior.scaleDown.stabilizationWindowSeconds``
+# is set deliberately wide (10 minutes) so a transient bad forecast
+# doesn't cause a flap. Predictive scaling is a "scale up early,
+# scale down slowly" workflow — the standard scaling-policy defaults
+# bias the wrong way for a predictive signal.
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: payments-api-predictive
+ namespace: payments
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: payments-api
+ minReplicas: 3
+ maxReplicas: 30
+ metrics:
+ # Primary: predictive forecast metric. The HPA target value is
+ # in the same units as the underlying forecast — e.g. if the
+ # forecast is requests-per-second per pod, set the target to
+ # the desired RPS-per-pod.
+ - type: External
+ external:
+ metric:
+ name: rps-forecast
+ selector:
+ matchLabels:
+ service: payments-api
+ target:
+ type: AverageValue
+ averageValue: "50"
+ behavior:
+ scaleUp:
+ # Scale up promptly — the whole point of predictive scaling is
+ # to provision capacity before the spike arrives.
+ stabilizationWindowSeconds: 0
+ policies:
+ - type: Pods
+ value: 4
+ periodSeconds: 30
+ - type: Percent
+ value: 50
+ periodSeconds: 30
+ selectPolicy: Max
+ scaleDown:
+ # Scale down slowly. A transient bad forecast that
+ # under-projects load briefly should not cause the HPA to
+ # drop pods.
+ stabilizationWindowSeconds: 600
+ policies:
+ - type: Pods
+ value: 1
+ periodSeconds: 60
+ selectPolicy: Min
diff --git a/forecaster/src/promforecast/adapter.py b/forecaster/src/promforecast/adapter.py
new file mode 100644
index 0000000..8572d85
--- /dev/null
+++ b/forecaster/src/promforecast/adapter.py
@@ -0,0 +1,580 @@
+"""Kubernetes custom-metrics adapter for predictive autoscaling.
+
+A separate sub-component that implements the Kubernetes custom-metrics API
+contracts:
+
+* ``external.metrics.k8s.io/v1beta1`` — flat (cluster-scoped or
+ namespace-scoped) metrics referenced by HPA's ``external`` metric
+ type. The HPA's ``targetValue`` is compared against the value
+ returned here.
+* ``custom.metrics.k8s.io/v1beta1`` — pod / object-scoped metrics
+ with selectors. Less common in practice; the predictive
+ autoscaling use-case usually fits the external API better
+ because the forecast is per-service rather than per-pod. The
+ adapter implements a minimal stub for the custom-metrics surface
+ so HPAs that prefer that flavour still get a sensible response.
+
+The adapter reads from VM via PromQL — it does **not** run the
+forecast pipeline. Forecasts are produced by the main forecaster
+and stored in VM via the sink; the adapter just exposes them under
+HPA-compatible names. Stateless, side-effect-free.
+
+Mounted behind the Kubernetes API aggregator via an
+``APIService`` resource. The aggregator requires TLS; the adapter's
+Helm chart generates a self-signed cert at install time so the
+operator doesn't need cert-manager just to stand this up.
+
+**Why a separate process?** The HPA control loop polls the adapter
+every 15 seconds. Embedding the adapter into the forecaster would
+mean the forecaster's HTTP surface had to be cluster-API-shaped
+(TLS, APIService-fronted), which complicates the simple
+``/metrics`` scrape model. A small separate adapter, talking to the
+same VM, keeps each component's contract tight.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import re
+import time
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import httpx
+import structlog
+import yaml
+from fastapi import FastAPI, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, ConfigDict, Field
+
+logger = structlog.get_logger(__name__)
+
+
+class _MetricMapping(BaseModel):
+ """One promforecast metric exposed to HPA.
+
+ ``external_metric_name`` is the name HPA references in its
+ ``external.metric.name`` field. Must conform to the Kubernetes
+ metric-name regex ([a-z0-9-]+) so the API server accepts it.
+
+ ``promql`` is the PromQL query the adapter runs against VM to
+ resolve the metric. The query should return a scalar / single
+ value — multi-series responses are summed across labels (the
+ HPA only consumes a scalar). For per-pod or per-namespace
+ autoscaling, parametrise the PromQL with the conventional
+ ``namespace`` / ``pod`` labels and let the adapter substitute
+ them at query time.
+
+ ``namespaced: true`` (default) routes the metric under the
+ namespaced HPA path. Set false for cluster-scoped metrics that
+ HPA can reference regardless of namespace.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ external_metric_name: str = Field(pattern=r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$")
+ promql: str
+ namespaced: bool = True
+
+
+class _AdapterConfig(BaseModel):
+ """Top-level adapter config.
+
+ Mounted via a ConfigMap in the standard install. The schema is
+ deliberately minimal — the adapter's job is to translate
+ HPA-shaped queries to PromQL; richer behaviour belongs on the
+ forecaster.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ datasource_url: str
+ metrics: list[_MetricMapping] = Field(default_factory=list)
+ # HTTP listen address. The chart's APIService points to the
+ # service's port; the deployment's pod listens on this port.
+ listen: str = ":6443"
+ # Per-PromQL request timeout. The HPA's poll cadence is 15s;
+ # tighter than that and we'd risk consecutive misses cascading
+ # into HPA marking the metric "unavailable".
+ query_timeout_seconds: float = 5.0
+
+
+# ---------------------------------------------------------------------------
+# K8s API JSON shapes. These mirror the upstream Go types; the adapter
+# emits the exact JSON shape the API aggregator expects.
+# ---------------------------------------------------------------------------
+
+
+def _api_group_list() -> dict[str, Any]:
+ """Body for ``GET /apis/`` (group discovery)."""
+ return {
+ "kind": "APIGroupList",
+ "apiVersion": "v1",
+ "groups": [
+ {
+ "name": "external.metrics.k8s.io",
+ "versions": [
+ {
+ "groupVersion": "external.metrics.k8s.io/v1beta1",
+ "version": "v1beta1",
+ }
+ ],
+ "preferredVersion": {
+ "groupVersion": "external.metrics.k8s.io/v1beta1",
+ "version": "v1beta1",
+ },
+ },
+ {
+ "name": "custom.metrics.k8s.io",
+ "versions": [
+ {
+ "groupVersion": "custom.metrics.k8s.io/v1beta1",
+ "version": "v1beta1",
+ }
+ ],
+ "preferredVersion": {
+ "groupVersion": "custom.metrics.k8s.io/v1beta1",
+ "version": "v1beta1",
+ },
+ },
+ ],
+ }
+
+
+def _external_resource_list(metrics: list[_MetricMapping]) -> dict[str, Any]:
+ """Body for ``GET /apis/external.metrics.k8s.io/v1beta1`` (resource listing)."""
+ return {
+ "kind": "APIResourceList",
+ "apiVersion": "v1",
+ "groupVersion": "external.metrics.k8s.io/v1beta1",
+ "resources": [
+ {
+ "name": m.external_metric_name,
+ "singularName": "",
+ "namespaced": m.namespaced,
+ "kind": "ExternalMetricValueList",
+ "verbs": ["get"],
+ }
+ for m in metrics
+ ],
+ }
+
+
+def _custom_resource_list() -> dict[str, Any]:
+ """Body for ``GET /apis/custom.metrics.k8s.io/v1beta1`` (resource listing).
+
+ Minimal placeholder — the adapter doesn't ship per-object
+ custom metrics today. The empty list is a valid response
+ that tells the API aggregator "this group exists, no
+ metrics under it yet" rather than 404, which would make HPA
+ list-time errors noisier than needed.
+ """
+ return {
+ "kind": "APIResourceList",
+ "apiVersion": "v1",
+ "groupVersion": "custom.metrics.k8s.io/v1beta1",
+ "resources": [],
+ }
+
+
+def _external_metric_value_list(
+ *,
+ metric_name: str,
+ value: float,
+ metric_labels: dict[str, str] | None = None,
+) -> dict[str, Any]:
+ """Body for ``GET /apis/external.metrics.k8s.io/v1beta1/.../``.
+
+ The Kubernetes API expects ``value`` as a Quantity string —
+ integers without a unit when the value is whole, decimals when
+ fractional. Render as a plain string and let the API parse it.
+ """
+ return {
+ "kind": "ExternalMetricValueList",
+ "apiVersion": "external.metrics.k8s.io/v1beta1",
+ "metadata": {},
+ "items": [
+ {
+ "metricName": metric_name,
+ "metricLabels": metric_labels or {},
+ "timestamp": datetime.now(tz=UTC)
+ .isoformat(timespec="seconds")
+ .replace("+00:00", "Z"),
+ "value": _format_quantity(value),
+ }
+ ],
+ }
+
+
+def _format_quantity(value: float) -> str:
+ """Render a float as a K8s Quantity string.
+
+ Integers come out without a decimal so HPA's parser accepts them
+ cleanly; fractional values get six digits of precision (matches
+ Prometheus's ``%g``-flavoured exposition).
+
+ **NaN / Inf -> "0" trade-off.** HPA's ``failedGetMetric`` event
+ (the alternative shape: return a non-200) marks the metric
+ unavailable, which keeps the autoscaler on its last good
+ recommendation indefinitely — *fail open* in HPA's terms. For a
+ forecast that hasn't produced a sample yet (cold-start, ramp-up,
+ transient query failure) the operator usually wants the HPA to
+ *fail closed* — i.e. fall back to whatever ``minReplicas`` says
+ rather than freeze in place. Returning ``"0"`` achieves that:
+ a target like ``averageValue: 10`` against a current value of
+ ``0`` triggers a scale-down toward minReplicas. Operators who
+ want the opposite shape (freeze on NaN) should wrap their
+ PromQL with ``clamp_min(..., 0.001)`` and a sentinel value
+ upstream — that's a downstream config decision the adapter
+ can't make safely on its own.
+ """
+ import math # noqa: PLC0415
+
+ if not math.isfinite(value):
+ return "0"
+ if value.is_integer():
+ return str(int(value))
+ return f"{value:.6f}".rstrip("0").rstrip(".")
+
+
+# ---------------------------------------------------------------------------
+# PromQL fetch shim. The adapter uses httpx directly rather than the
+# forecaster's ``PromSource`` because the adapter's contract is
+# substantively narrower — single-value queries with a short timeout
+# — and pulling in the full source would drag scheduling / retry
+# infrastructure the adapter doesn't need.
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class _PromValue:
+ """One scalar result from PromQL."""
+
+ value: float
+ timestamp: datetime
+ labels: dict[str, str]
+
+
+async def _query_promql(
+ *,
+ client: httpx.AsyncClient,
+ base_url: str,
+ promql: str,
+ timeout_seconds: float,
+) -> list[_PromValue]:
+ """Run a PromQL instant query and return the matching values.
+
+ The adapter uses ``/api/v1/query`` (instant) rather than
+ ``/api/v1/query_range`` (range) — HPA wants the most recent
+ value, not a time series. VM accepts the same endpoint shape
+ as Prometheus.
+
+ Returns an empty list when the query has no matching series.
+ Raises ``httpx.HTTPError`` on transport failure; the caller
+ translates that into a 500 / 503 response.
+ """
+ url = base_url.rstrip("/") + "/api/v1/query"
+ response = await client.get(url, params={"query": promql}, timeout=timeout_seconds)
+ response.raise_for_status()
+ body = response.json()
+ if body.get("status") != "success":
+ return []
+ result = body.get("data", {}).get("result", [])
+ values: list[_PromValue] = []
+ for entry in result:
+ metric_labels = {str(k): str(v) for k, v in entry.get("metric", {}).items()}
+ value_pair = entry.get("value")
+ if not value_pair or len(value_pair) != 2: # noqa: PLR2004
+ continue
+ try:
+ ts = datetime.fromtimestamp(float(value_pair[0]), tz=UTC)
+ val = float(value_pair[1])
+ except (TypeError, ValueError):
+ continue
+ values.append(_PromValue(value=val, timestamp=ts, labels=metric_labels))
+ return values
+
+
+# ---------------------------------------------------------------------------
+# FastAPI app construction.
+# ---------------------------------------------------------------------------
+
+
+def _build_app(config: _AdapterConfig) -> FastAPI:
+ """Wire up the adapter HTTP surface against ``config``.
+
+ The app is built once per process — the metric list is static
+ over the process lifetime. Reloading happens via process
+ restart (Helm ``kubectl rollout restart``) so the contract
+ stays as simple as a stateless adapter can be.
+ """
+ metrics_by_name = {m.external_metric_name: m for m in config.metrics}
+
+ @asynccontextmanager
+ async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]:
+ # If a test pre-seeded a client (or a previous lifespan left
+ # one behind), close it before installing the one we own —
+ # otherwise we leak a connection pool on every restart.
+ prior = getattr(fastapi_app.state, "http", None)
+ if prior is not None:
+ with contextlib.suppress(Exception):
+ await prior.aclose()
+ async with httpx.AsyncClient() as client:
+ fastapi_app.state.http = client
+ yield
+
+ app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
+ # Lazily build the http client on first request so tests that
+ # bypass ``lifespan`` (bare ``TestClient(app)`` without a ``with``
+ # block) get a usable client without us leaking a pool that the
+ # lifespan would otherwise replace and never close. Production
+ # boots through ``lifespan`` and never hits the lazy path.
+ app.state.http = None
+
+ @app.get("/healthz")
+ async def healthz() -> dict[str, str]:
+ return {"status": "ok"}
+
+ @app.get("/apis")
+ async def apis() -> Response:
+ return JSONResponse(_api_group_list())
+
+ @app.get("/apis/")
+ async def apis_slash() -> Response:
+ # The API aggregator may probe either form; both serve the
+ # same body so a trailing-slash mismatch doesn't break
+ # discovery.
+ return JSONResponse(_api_group_list())
+
+ @app.get("/apis/external.metrics.k8s.io/v1beta1")
+ async def external_metrics_list() -> Response:
+ return JSONResponse(_external_resource_list(config.metrics))
+
+ @app.get("/apis/external.metrics.k8s.io/v1beta1/")
+ async def external_metrics_list_slash() -> Response:
+ return JSONResponse(_external_resource_list(config.metrics))
+
+ @app.get("/apis/custom.metrics.k8s.io/v1beta1")
+ async def custom_metrics_list() -> Response:
+ return JSONResponse(_custom_resource_list())
+
+ async def _fetch_external_value(metric_name: str, label_selector: str | None) -> Response:
+ mapping = metrics_by_name.get(metric_name)
+ if mapping is None:
+ return JSONResponse(
+ {
+ "kind": "Status",
+ "apiVersion": "v1",
+ "status": "Failure",
+ "message": f"metric {metric_name!r} is not configured",
+ "reason": "NotFound",
+ "code": 404,
+ },
+ status_code=404,
+ )
+ promql = _apply_label_selector(mapping.promql, label_selector)
+ # ``lifespan`` installs the client on app startup; the lazy
+ # path here only fires for tests that skip the lifespan
+ # (bare ``TestClient(app)`` without a ``with`` block). The
+ # lazy client lives for the rest of the app's lifetime; tests
+ # that care about cleanup should still use the context-manager
+ # form of TestClient.
+ if app.state.http is None:
+ app.state.http = httpx.AsyncClient()
+ client = app.state.http
+ try:
+ values = await _query_promql(
+ client=client,
+ base_url=config.datasource_url,
+ promql=promql,
+ timeout_seconds=config.query_timeout_seconds,
+ )
+ except httpx.HTTPError as exc:
+ logger.warning(
+ "adapter_query_failed",
+ metric=metric_name,
+ error=str(exc),
+ )
+ return JSONResponse(
+ {
+ "kind": "Status",
+ "apiVersion": "v1",
+ "status": "Failure",
+ "message": f"upstream query failed: {exc}",
+ "reason": "ServiceUnavailable",
+ "code": 503,
+ },
+ status_code=503,
+ )
+ # Sum across returned series. HPA wants a scalar; multi-
+ # series responses are nearly always per-pod or per-replica
+ # fan-outs that the operator wants aggregated for
+ # autoscaling purposes. Operators who want a different
+ # reduction wrap their PromQL with ``avg()`` / ``max()`` /
+ # ``topk()`` at config time.
+ aggregate = sum(v.value for v in values) if values else 0.0
+ return JSONResponse(
+ _external_metric_value_list(
+ metric_name=metric_name,
+ value=aggregate,
+ )
+ )
+
+ @app.get("/apis/external.metrics.k8s.io/v1beta1/{metric_name}")
+ async def external_metric_cluster(metric_name: str, request: Request) -> Response:
+ return await _fetch_external_value(metric_name, request.query_params.get("labelSelector"))
+
+ @app.get("/apis/external.metrics.k8s.io/v1beta1/namespaces/{namespace}/{metric_name}")
+ async def external_metric_namespaced(
+ namespace: str, metric_name: str, request: Request
+ ) -> Response:
+ selector = request.query_params.get("labelSelector")
+ # The URL path's namespace is authoritative — the K8s API
+ # server has already enforced RBAC against it. Strip any
+ # caller-supplied ``namespace=`` pair from the labelSelector
+ # before appending our own so a caller can't bypass scoping by
+ # passing ``labelSelector=namespace=other``. Other pairs pass
+ # through.
+ cleaned = _strip_namespace_pair(selector)
+ ns_selector = f"namespace={namespace}"
+ merged = ns_selector if not cleaned else f"{cleaned},{ns_selector}"
+ return await _fetch_external_value(metric_name, merged)
+
+ return app
+
+
+def _strip_namespace_pair(selector: str | None) -> str | None:
+ """Drop any ``namespace=...`` pair from a labelSelector.
+
+ The namespaced endpoint reads the namespace from the URL path
+ (already authenticated and authorised by the K8s API server) and
+ refuses to let a caller-supplied ``namespace=`` pair override it
+ — otherwise a workload in namespace A could read metrics scoped
+ to namespace B. Other pairs are preserved verbatim.
+ """
+ if not selector:
+ return selector
+ kept = [p for p in selector.split(",") if p.partition("=")[0].strip() != "namespace"]
+ return ",".join(kept) if kept else None
+
+
+def _apply_label_selector(promql: str, selector: str | None) -> str:
+ """Substitute ``$