From 5a5f934178130df415322afcd42e02930658d398 Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Sun, 24 May 2026 20:40:28 +0200 Subject: [PATCH] Predictive ops integrations --- .github/workflows/ci.yml | 8 +- Makefile | 6 +- .../promforecast-metrics-adapter/Chart.yaml | 24 + .../templates/_helpers.tpl | 131 +++ .../templates/apiservice.yaml | 45 + .../templates/configmap.yaml | 19 + .../templates/deployment.yaml | 94 +++ .../templates/rbac.yaml | 53 ++ .../templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../templates/tls-secret.yaml | 14 + .../values.schema.json | 91 ++ .../promforecast-metrics-adapter/values.yaml | 146 ++++ charts/promforecast/values.yaml | 16 + .../promforecast-incident-heatmap.json | 149 ++++ .../grafana/promforecast-is-it-working.json | 55 ++ docs/README.md | 4 + docs/operations/cardinality.md | 2 + docs/outputs/deploy-safety.md | 100 +++ docs/outputs/incident-patterns.md | 90 ++ docs/outputs/what-if-fleet.md | 110 +++ docs/predictive-autoscaling.md | 146 ++++ examples/alerts/promforecast-rules.yaml | 43 + examples/configs/deploy-and-incidents.yaml | 113 +++ examples/k8s/predictive-hpa.yaml | 71 ++ forecaster/src/promforecast/adapter.py | 580 +++++++++++++ forecaster/src/promforecast/config.py | 203 ++++- forecaster/src/promforecast/deploy_risk.py | 373 +++++++++ forecaster/src/promforecast/exporter.py | 252 +++++- .../src/promforecast/incident_pattern.py | 184 +++++ forecaster/src/promforecast/main.py | 130 ++- forecaster/src/promforecast/point_factory.py | 6 +- .../src/promforecast/runner/_scheduler.py | 477 ++++++++++- forecaster/src/promforecast/validate.py | 30 +- forecaster/src/promforecast/what_if.py | 534 ++++++++++++ .../tests/perf/test_query_id_lookup_bench.py | 2 +- forecaster/tests/test_adapter.py | 435 ++++++++++ .../tests/test_chart_metrics_adapter.py | 242 ++++++ forecaster/tests/test_deploy_risk.py | 777 ++++++++++++++++++ .../tests/test_example_alerts_labelset.py | 25 + forecaster/tests/test_incident_pattern.py | 457 ++++++++++ .../test_runner_conditional_decompose.py | 26 +- .../tests/test_runner_inheritance_registry.py | 10 +- forecaster/tests/test_validate_cli.py | 6 +- forecaster/tests/test_what_if.py | 658 +++++++++++++++ 45 files changed, 6915 insertions(+), 49 deletions(-) create mode 100644 charts/promforecast-metrics-adapter/Chart.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/_helpers.tpl create mode 100644 charts/promforecast-metrics-adapter/templates/apiservice.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/configmap.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/deployment.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/rbac.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/service.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/serviceaccount.yaml create mode 100644 charts/promforecast-metrics-adapter/templates/tls-secret.yaml create mode 100644 charts/promforecast-metrics-adapter/values.schema.json create mode 100644 charts/promforecast-metrics-adapter/values.yaml create mode 100644 dashboards/grafana/promforecast-incident-heatmap.json create mode 100644 docs/outputs/deploy-safety.md create mode 100644 docs/outputs/incident-patterns.md create mode 100644 docs/outputs/what-if-fleet.md create mode 100644 docs/predictive-autoscaling.md create mode 100644 examples/configs/deploy-and-incidents.yaml create mode 100644 examples/k8s/predictive-hpa.yaml create mode 100644 forecaster/src/promforecast/adapter.py create mode 100644 forecaster/src/promforecast/deploy_risk.py create mode 100644 forecaster/src/promforecast/incident_pattern.py create mode 100644 forecaster/src/promforecast/what_if.py create mode 100644 forecaster/tests/test_adapter.py create mode 100644 forecaster/tests/test_chart_metrics_adapter.py create mode 100644 forecaster/tests/test_deploy_risk.py create mode 100644 forecaster/tests/test_incident_pattern.py create mode 100644 forecaster/tests/test_what_if.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a29a42..13f2a3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: # --target-branch defaults to ``main``; explicit for clarity. run: | ct lint \ - --charts charts/promforecast,charts/promforecast-stack \ + --charts charts/promforecast,charts/promforecast-stack,charts/promforecast-metrics-adapter \ --validate-maintainers=false \ --check-version-increment=false \ --target-branch ${{ github.event.repository.default_branch }} @@ -176,6 +176,7 @@ jobs: run: | helm lint --strict charts/promforecast helm lint --strict charts/promforecast-stack + helm lint --strict charts/promforecast-metrics-adapter - name: Install kubeconform run: | @@ -186,8 +187,9 @@ jobs: - name: kubeconform run: | - helm template charts/promforecast | kubeconform -strict -summary -ignore-missing-schemas - helm template charts/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promforecast | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template charts/promforecast-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas # Also validate the bundled dashboards + PrometheusRule path so CI # catches regressions in the templated assets, not only the default # off path. diff --git a/Makefile b/Makefile index 721990e..ba73518 100644 --- a/Makefile +++ b/Makefile @@ -61,8 +61,10 @@ chart-deps: ## Resolve umbrella subchart dependencies. chart-lint: chart-deps ## helm lint + kubeconform on rendered manifests. helm lint $(CHART_DIR)/promforecast helm lint $(CHART_DIR)/promforecast-stack - helm template $(CHART_DIR)/promforecast | kubeconform -strict -summary -ignore-missing-schemas - helm template $(CHART_DIR)/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas + helm lint $(CHART_DIR)/promforecast-metrics-adapter + helm template $(CHART_DIR)/promforecast | kubeconform -strict -summary -ignore-missing-schemas + helm template $(CHART_DIR)/promforecast-stack | kubeconform -strict -summary -ignore-missing-schemas + helm template $(CHART_DIR)/promforecast-metrics-adapter | kubeconform -strict -summary -ignore-missing-schemas helm template $(CHART_DIR)/promforecast-stack \ --set dashboards.enabled=true \ --set prometheusRules.enabled=true \ diff --git a/charts/promforecast-metrics-adapter/Chart.yaml b/charts/promforecast-metrics-adapter/Chart.yaml new file mode 100644 index 0000000..615f5ce --- /dev/null +++ b/charts/promforecast-metrics-adapter/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: promforecast-metrics-adapter +description: Kubernetes custom-metrics adapter that exposes promforecast forecast metrics for HorizontalPodAutoscaler. +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.27.0-0" +home: https://github.com/esops-dev/promforecast +sources: + - https://github.com/esops-dev/promforecast +keywords: + - prometheus + - forecasting + - kubernetes + - autoscaling + - hpa + - custom-metrics +maintainers: + - name: esops-dev + url: https://github.com/esops-dev +icon: https://raw.githubusercontent.com/esops-dev/promforecast/main/docs/assets/logo.png +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/category: monitoring diff --git a/charts/promforecast-metrics-adapter/templates/_helpers.tpl b/charts/promforecast-metrics-adapter/templates/_helpers.tpl new file mode 100644 index 0000000..0a67390 --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/_helpers.tpl @@ -0,0 +1,131 @@ +{{/* +Common helpers for the promforecast-metrics-adapter chart. +*/}} + +{{- define "promforecast-metrics-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "promforecast-metrics-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "promforecast-metrics-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "promforecast-metrics-adapter.labels" -}} +helm.sh/chart: {{ include "promforecast-metrics-adapter.chart" . }} +{{ include "promforecast-metrics-adapter.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: promforecast +app.kubernetes.io/component: metrics-adapter +{{- end -}} + +{{- define "promforecast-metrics-adapter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "promforecast-metrics-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "promforecast-metrics-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "promforecast-metrics-adapter.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{- define "promforecast-metrics-adapter.image" -}} +{{- $tag := default .Chart.AppVersion .Values.image.tag -}} +{{- printf "%s:%s" .Values.image.repository $tag -}} +{{- end -}} + +{{/* +TLS certificate (self-signed when tls.generate=true). + +Stored in a ``hiddenSecret`` flavour so re-rendering on each +``helm upgrade`` regenerates the cert. Stable across upgrades within +the same release because the chart pins the CN to the service DNS +name; only the validity period rolls. +*/}} +{{- define "promforecast-metrics-adapter.tlsSecretName" -}} +{{- if .Values.tls.existingSecret -}} +{{- .Values.tls.existingSecret -}} +{{- else -}} +{{- printf "%s-tls" (include "promforecast-metrics-adapter.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +TLS material generation. + +``genCA`` / ``genSignedCert`` are non-deterministic across calls, +so the cert is generated *once* per Helm render and stashed on +``$.Values`` for the duration of that render. Subsequent calls +(e.g. one for the Secret, one for the APIService) pull the +already-generated material instead of producing a fresh CA. + +On ``helm upgrade`` against a live cluster, ``lookup`` finds the +chart-managed Secret from the previous install and the helper +returns *that* material — so the cert + CA bundle stays stable +across upgrades and the APIService caBundle doesn't churn (which +would otherwise force a rolling restart on every upgrade via the +``checksum/tls`` annotation and break in-flight HPA polls). The +fresh-generate path only fires when ``lookup`` returns nothing: +first install, or offline ``helm template`` (which is fine — the +golden-file render is deterministic per process but irrelevant to +a real cluster). + +Returns a dict with ``ca``, ``cert``, ``key`` (PEM strings). For +``tls.existingSecret`` installs the helper uses ``lookup`` against +the operator-named Secret and returns its data instead. +*/}} +{{- define "promforecast-metrics-adapter.tlsMaterial" -}} +{{- $fullname := include "promforecast-metrics-adapter.fullname" . -}} +{{- if .Values.tls.existingSecret -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.tls.existingSecret -}} +{{- if $secret -}} +{{- dict "ca" (index $secret.data "ca.crt" | b64dec) "cert" (index $secret.data "tls.crt" | b64dec) "key" (index $secret.data "tls.key" | b64dec) | toYaml -}} +{{- else -}} +{{- /* +The operator set ``tls.existingSecret`` but the Secret isn't in the +cluster (or we're rendering offline via ``helm template`` which +returns nothing from ``lookup``). Failing loud here is the right +default for a production install — the APIService would otherwise +fall back to ``insecureSkipTLSVerify: true`` and silently accept any +upstream cert, which defeats the existingSecret contract. + +For offline rendering (CI's ``helm template`` golden-file path), +either pass ``--set tls.generate=true`` to exercise the self-signed +path, or pre-create the Secret in a kind cluster and let the lookup +populate. +*/}} +{{- fail (printf "tls.existingSecret=%q is set but Secret %q was not found in namespace %q. Either create the Secret first (it must carry ca.crt/tls.crt/tls.key), or unset existingSecret and use tls.generate=true. Note: ``helm template`` cannot resolve ``lookup`` against a live cluster; pass --set tls.generate=true to render offline." .Values.tls.existingSecret .Values.tls.existingSecret .Release.Namespace) -}} +{{- end -}} +{{- else -}} +{{- if not (hasKey .Values "_tlsMaterial") -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace (printf "%s-tls" $fullname) -}} +{{- if and $existing (index $existing.data "ca.crt") (index $existing.data "tls.crt") (index $existing.data "tls.key") -}} +{{- $_ := set .Values "_tlsMaterial" (dict "ca" (index $existing.data "ca.crt" | b64dec) "cert" (index $existing.data "tls.crt" | b64dec) "key" (index $existing.data "tls.key" | b64dec)) -}} +{{- else -}} +{{- $svcDNS := printf "%s.%s.svc" $fullname .Release.Namespace -}} +{{- $svcDNSFull := printf "%s.%s.svc.cluster.local" $fullname .Release.Namespace -}} +{{- $altNames := list $svcDNS $svcDNSFull $fullname -}} +{{- $ca := genCA (printf "%s-ca" $fullname) (int .Values.tls.validityDays) -}} +{{- $cert := genSignedCert $svcDNS nil $altNames (int .Values.tls.validityDays) $ca -}} +{{- $_ := set .Values "_tlsMaterial" (dict "ca" $ca.Cert "cert" $cert.Cert "key" $cert.Key) -}} +{{- end -}} +{{- end -}} +{{- .Values._tlsMaterial | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/promforecast-metrics-adapter/templates/apiservice.yaml b/charts/promforecast-metrics-adapter/templates/apiservice.yaml new file mode 100644 index 0000000..957d868 --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/apiservice.yaml @@ -0,0 +1,45 @@ +{{- if .Values.apiService.enabled -}} +{{- $fullname := include "promforecast-metrics-adapter.fullname" . -}} +{{- $tls := include "promforecast-metrics-adapter.tlsMaterial" . | fromYaml -}} +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.external.metrics.k8s.io + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +spec: + service: + name: {{ $fullname }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.service.port }} + group: external.metrics.k8s.io + version: v1beta1 + versionPriority: 100 + groupPriorityMinimum: 100 + {{- if $tls.ca }} + caBundle: {{ $tls.ca | b64enc }} + {{- else }} + insecureSkipTLSVerify: true + {{- end }} +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +spec: + service: + name: {{ $fullname }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.service.port }} + group: custom.metrics.k8s.io + version: v1beta1 + versionPriority: 100 + groupPriorityMinimum: 100 + {{- if $tls.ca }} + caBundle: {{ $tls.ca | b64enc }} + {{- else }} + insecureSkipTLSVerify: true + {{- end }} +{{- end }} diff --git a/charts/promforecast-metrics-adapter/templates/configmap.yaml b/charts/promforecast-metrics-adapter/templates/configmap.yaml new file mode 100644 index 0000000..75df965 --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "promforecast-metrics-adapter.fullname" . }} + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +data: + adapter.yaml: | + # promforecast-metrics-adapter configuration. + # Schema: docs/predictive-autoscaling.md + datasource_url: {{ .Values.datasourceUrl | quote }} + listen: ":6443" + query_timeout_seconds: 5.0 + metrics: +{{- range .Values.metrics }} + - external_metric_name: {{ .externalMetricName | quote }} + promql: {{ .promql | quote }} + namespaced: {{ default true .namespaced }} +{{- end }} diff --git a/charts/promforecast-metrics-adapter/templates/deployment.yaml b/charts/promforecast-metrics-adapter/templates/deployment.yaml new file mode 100644 index 0000000..1abd618 --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "promforecast-metrics-adapter.fullname" . }} + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 8 }} + annotations: + # Re-trigger pod rollout when the config or the cert + # changes — Helm's checksum/config + checksum/secret + # annotations are the standard pattern. The TLS material + # helper looks up the chart-managed Secret on upgrade so the + # cert is *stable* across `helm upgrade` and this checksum + # only flips on a real rotation (initial install, the Secret + # being deleted out-of-band, or ``existingSecret`` swapped to + # different material) — uvicorn loads ssl_certfile once at + # boot and never reloads, so the rolling restart is the only + # way to pick up a new cert. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/tls: {{ include "promforecast-metrics-adapter.tlsMaterial" . | sha256sum }} + spec: + serviceAccountName: {{ include "promforecast-metrics-adapter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + containers: + - name: adapter + image: {{ include "promforecast-metrics-adapter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - adapter + - --config=/etc/promforecast-adapter/adapter.yaml + - --tls-cert=/var/run/promforecast-adapter/tls.crt + - --tls-key=/var/run/promforecast-adapter/tls.key + ports: + - name: https + containerPort: 6443 + protocol: TCP + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/promforecast-adapter + readOnly: true + - name: tls + mountPath: /var/run/promforecast-adapter + readOnly: true + volumes: + - name: config + configMap: + name: {{ include "promforecast-metrics-adapter.fullname" . }} + - name: tls + secret: + secretName: {{ include "promforecast-metrics-adapter.tlsSecretName" . }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/promforecast-metrics-adapter/templates/rbac.yaml b/charts/promforecast-metrics-adapter/templates/rbac.yaml new file mode 100644 index 0000000..e7f61cc --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/rbac.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rbac.create }} +# Two bindings ship by default; nothing else. The kube-controller-manager +# (which runs the HPA control loop) already holds permission on the +# metrics aggregate APIs via the cluster's built-in +# ``system:kube-controller-manager`` ClusterRole, and the API aggregator +# forwards its identity through to this adapter after the +# ``system:auth-delegator`` handshake below, so no extra ``metrics-reader`` +# ClusterRole / binding is needed for HPA to function. Workloads outside +# the controller manager that need direct access to +# ``external.metrics.k8s.io`` / ``custom.metrics.k8s.io`` (custom +# controllers, Argo Rollouts AnalysisRuns) should bind their own +# ServiceAccount to a ClusterRole they manage — keeping that policy +# choice in the operator's hands rather than burying a non-obvious +# default in this chart. +# +# The adapter is fronted by the Kubernetes API aggregator, which +# forwards already-authenticated requests through. To validate the +# forwarded tokens we bind the standard ``system:auth-delegator`` +# ClusterRole — the convention every custom-metrics adapter follows. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "promforecast-metrics-adapter.fullname" . }}-auth-delegator + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +# RoleBinding to extension-apiserver-authentication-reader so the +# adapter can read the front-proxy CA from kube-system — required +# for validating the aggregator's request signatures. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "promforecast-metrics-adapter.fullname" . }}-auth-reader + namespace: kube-system + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/promforecast-metrics-adapter/templates/service.yaml b/charts/promforecast-metrics-adapter/templates/service.yaml new file mode 100644 index 0000000..a5088ac --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "promforecast-metrics-adapter.fullname" . }} + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: https + selector: + {{- include "promforecast-metrics-adapter.selectorLabels" . | nindent 4 }} diff --git a/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml b/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000..b8d614c --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "promforecast-metrics-adapter.serviceAccountName" . }} + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/promforecast-metrics-adapter/templates/tls-secret.yaml b/charts/promforecast-metrics-adapter/templates/tls-secret.yaml new file mode 100644 index 0000000..15fad18 --- /dev/null +++ b/charts/promforecast-metrics-adapter/templates/tls-secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.tls.generate (not .Values.tls.existingSecret) -}} +{{- $tls := include "promforecast-metrics-adapter.tlsMaterial" . | fromYaml -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "promforecast-metrics-adapter.tlsSecretName" . }} + labels: + {{- include "promforecast-metrics-adapter.labels" . | nindent 4 }} +type: kubernetes.io/tls +data: + tls.crt: {{ $tls.cert | b64enc }} + tls.key: {{ $tls.key | b64enc }} + ca.crt: {{ $tls.ca | b64enc }} +{{- end }} diff --git a/charts/promforecast-metrics-adapter/values.schema.json b/charts/promforecast-metrics-adapter/values.schema.json new file mode 100644 index 0000000..a8607ee --- /dev/null +++ b/charts/promforecast-metrics-adapter/values.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "$id": "https://promforecast.io/schemas/metrics-adapter.values.v1.schema.json", + "title": "promforecast-metrics-adapter Helm values (v1)", + "description": "Schema for the Kubernetes custom-metrics adapter that exposes promforecast forecast metrics to HorizontalPodAutoscaler.", + "type": "object", + "additionalProperties": true, + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { "type": "string", "minLength": 1 }, + "tag": { "type": "string" }, + "pullPolicy": { + "type": "string", + "enum": ["Always", "IfNotPresent", "Never"] + } + }, + "required": ["repository"] + }, + "replicaCount": { "type": "integer", "minimum": 1 }, + "datasourceUrl": { "type": "string", "minLength": 1 }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "externalMetricName": { + "type": "string", + "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" + }, + "promql": { "type": "string", "minLength": 1 }, + "namespaced": { "type": "boolean" } + }, + "required": ["externalMetricName", "promql"] + } + }, + "tls": { + "type": "object", + "additionalProperties": false, + "properties": { + "generate": { "type": "boolean" }, + "existingSecret": { "type": "string" }, + "validityDays": { "type": "integer", "minimum": 1 } + } + }, + "apiService": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" } + } + }, + "resources": { "type": "object" }, + "probes": { "type": "object" }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "type": "string", "enum": ["ClusterIP", "NodePort", "LoadBalancer"] }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "targetPort": { "type": "integer", "minimum": 1, "maximum": 65535 } + } + }, + "serviceAccount": { + "type": "object", + "additionalProperties": false, + "properties": { + "create": { "type": "boolean" }, + "name": { "type": "string" }, + "annotations": { "type": "object" } + } + }, + "securityContext": { "type": "object" }, + "rbac": { + "type": "object", + "additionalProperties": false, + "properties": { + "create": { "type": "boolean" } + } + }, + "nameOverride": { "type": "string" }, + "fullnameOverride": { "type": "string" }, + "nodeSelector": { "type": "object" }, + "tolerations": { "type": "array" }, + "affinity": { "type": "object" } + }, + "required": ["image", "datasourceUrl"] +} diff --git a/charts/promforecast-metrics-adapter/values.yaml b/charts/promforecast-metrics-adapter/values.yaml new file mode 100644 index 0000000..febe338 --- /dev/null +++ b/charts/promforecast-metrics-adapter/values.yaml @@ -0,0 +1,146 @@ +# promforecast-metrics-adapter — Helm values. +# +# The adapter implements the Kubernetes custom-metrics API contracts +# (external.metrics.k8s.io/v1beta1 and custom.metrics.k8s.io/v1beta1) so +# HorizontalPodAutoscaler can scale on promforecast forecast metrics. +# Forecasts are still produced by the main `promforecast` chart and +# stored in VictoriaMetrics via the sink; this adapter is a thin +# read-side translator between HPA and PromQL. + +image: + repository: ghcr.io/esops-dev/promforecast + # tag: "" # Defaults to the chart's appVersion when empty. + pullPolicy: IfNotPresent + +# Single replica is the default. The adapter is stateless and a +# restart is invisible to HPA (which retries on 503), so HA-style +# multiple replicas are valid but rarely necessary. Use `replicaCount: +# >1` only for shops with strict per-pod availability requirements. +replicaCount: 1 + +# Promforecast datasource URL the adapter reads from. +# +# IMPORTANT — OVERRIDE THIS unless you installed the umbrella chart +# (``promforecast-stack``) into the ``monitoring`` namespace under the +# release name ``promforecast-stack``. The default hardcodes both +# pieces because Helm ``values.yaml`` cannot reference +# ``.Release.Namespace``; if either differs in your install (a +# ``monitoring-system`` namespace, a ``pf`` release name, an +# externally-managed VictoriaMetrics) the adapter will resolve the +# wrong service and every HPA poll returns 503. +# +# Conventional overrides: +# --set datasourceUrl=http://vm.observability.svc.cluster.local:8428 +# --set datasourceUrl=http://victoria-metrics-cluster-vmselect.vm:8481/select/0/prometheus +datasourceUrl: "http://promforecast-stack-victoria-metrics-single-server.monitoring.svc.cluster.local:8428" + +# Metric mappings — the meat of the chart. Each entry exposes one +# forecast metric to HPA. Use the `external.metrics.k8s.io` API for +# service-level forecasts (the common case); pair the configured +# PromQL with `$namespace` / `$pod` placeholders if you want HPA's +# namespace / labelSelector arguments to filter the query. +# +# Conventional pattern: define one external metric per service + +# signal pair (e.g. payments-rps-forecast, +# payments-p95-latency-forecast). HPA references the metric by name. +metrics: [] + # - externalMetricName: rps-forecast + # promql: 'sum by (namespace)(http_requests_rate_forecast{namespace="$namespace"})' + # namespaced: true + +# K8s API aggregator requires the adapter to serve TLS — the chart +# generates a self-signed cert at install time and registers its CA +# bundle with the APIService. cert-manager is not required. +tls: + # When false the chart does not generate a cert; the operator + # provides one via the existingSecret reference instead. Useful + # in shops that already run cert-manager and want a managed cert. + generate: true + existingSecret: "" + # Validity of the self-signed cert in days. The chart re-generates + # the cert on each `helm upgrade` if `generate: true`, so a long + # validity is appropriate. + validityDays: 3650 + +apiService: + # Whether to register an APIService for the external and custom + # metrics groups. Disable only if the cluster already has another + # metrics adapter registered (e.g. prometheus-adapter) and you're + # running this one alongside under a different group. + enabled: true + +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + +# Liveness / readiness probes against the adapter's /healthz +# endpoint. Tight readiness keeps HPA from polling a not-yet-ready +# replica during a rollout. +probes: + readiness: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + liveness: + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + +service: + type: ClusterIP + port: 443 + # The adapter listens on 6443 by convention (custom-metrics + # adapters share that port across implementations). The chart's + # service maps 443 -> 6443 so APIService consumers reach a + # standard HTTPS port. + targetPort: 6443 + +serviceAccount: + create: true + name: "" + annotations: {} + +# Pod security context. The adapter doesn't need anything special +# — runs as a non-root user, no privileged operations. +securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + +# NetworkPolicy — by default we don't ship one because the adapter +# needs the K8s API aggregator (kube-apiserver) to reach it on its +# TLS port, and that traffic is from arbitrary cluster IPs depending +# on how kube-apiserver is deployed. Operators who want a +# NetworkPolicy should pin it to their cluster's apiserver pod +# selector or CIDR. +# +# IMPORTANT — authentication model. The adapter trusts the K8s API +# aggregator to authenticate clients before forwarding requests; it +# does NOT independently verify the forwarded identity via +# TokenReview / SubjectAccessReview. Direct connections to the +# Service:port that bypass the aggregator (in-cluster `curl` from +# another pod, `kubectl port-forward`, a misconfigured ingress) read +# the configured metrics without authentication. In multi-tenant +# clusters pair this chart with a NetworkPolicy that restricts +# ingress to the kube-apiserver only (cluster-specific selector; +# omitted from defaults because the right selector is per-cluster). + +# RBAC scope — needed to authenticate clients via TokenReview/ +# SubjectAccessReview when the API aggregator forwards an +# authenticated request through to us. The chart binds the +# system:auth-delegator ClusterRole (the standard custom-metrics +# adapter requirement). +rbac: + create: true + +nameOverride: "" +fullnameOverride: "" + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/charts/promforecast/values.yaml b/charts/promforecast/values.yaml index 708138c..5d1b0ea 100644 --- a/charts/promforecast/values.yaml +++ b/charts/promforecast/values.yaml @@ -60,6 +60,15 @@ config: # exec`. The endpoint is unauthenticated like /metrics, so only # enable it when the metrics port is on a trusted network. expose_config_endpoint: false + # Off by default. Flip to true to expose `POST /forecast/what-if/fleet` + # which runs the preview machinery across every query whose + # `thresholds:` define a capacity ceiling and returns a ranked list + # of services projected to breach soonest under a hypothetical + # traffic multiplier. The endpoint is throttled by + # `safety.fleet_what_if_min_interval` so polling dashboards can't + # starve scheduled fits. Same network-level access-control model as + # /metrics and /-/reload. + expose_what_if_endpoint: false safety: max_series_per_query: 500 @@ -67,6 +76,13 @@ config: series_overflow: drop_lowest_priority fit_timeout: 60s query_timeout: 30s + # Minimum interval between successive fleet what-if runs. The + # endpoint refits every threshold-bearing query against the + # requested traffic multiplier; a single invocation is O(minutes) + # on a medium config so unthrottled access would starve scheduled + # fits. The default of 5m matches a typical `refresh_interval`. + # Has no effect unless `server.expose_what_if_endpoint` is true. + fleet_what_if_min_interval: 5m defaults: lookback: 14d diff --git a/dashboards/grafana/promforecast-incident-heatmap.json b/dashboards/grafana/promforecast-incident-heatmap.json new file mode 100644 index 0000000..c793815 --- /dev/null +++ b/dashboards/grafana/promforecast-incident-heatmap.json @@ -0,0 +1,149 @@ +{ + "title": "promforecast — Incident recurrence heatmap", + "uid": "promforecast-incident-heatmap", + "description": "Per-(day_of_week, hour_of_day) intensity of recurring incidents for queries opted into `data_profile: events`. Score normalised against the average non-empty bucket; values > 1.0 indicate above-average concentration in that bucket. Pair the heatmap with the interpretation guide in docs/outputs/incident-patterns.md — high score + low variance = preemptive action warranted; high score + high variance = pattern is trending up.", + "schemaVersion": 39, + "version": 1, + "editable": true, + "tags": ["promforecast", "incidents", "patterns"], + "timezone": "", + "time": { "from": "now-30d", "to": "now" }, + "refresh": "1h", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { "text": "VictoriaMetrics", "value": "victoriametrics" } + }, + { + "name": "group", + "label": "Group", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(forecast_incident_recurrence_score, group)", + "refresh": 2, + "includeAll": false, + "multi": false + }, + { + "name": "query", + "label": "Query", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(forecast_incident_recurrence_score{group=\"$group\"}, id)", + "refresh": 2, + "includeAll": false, + "multi": false + } + ] + }, + "panels": [ + { + "type": "heatmap", + "id": 1, + "title": "Incident recurrence intensity — $query ($group)", + "description": "Each cell is one (day_of_week, hour_of_day) bucket. Score = bucket event count / mean non-empty bucket count. > 1.0 (warm colour) means this bucket sees more incidents than average. Read columns to find time-of-day patterns; read rows to find day-of-week patterns.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 14, "w": 24, "x": 0, "y": 0 }, + "targets": [ + { + "refId": "A", + "expr": "forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"}", + "instant": true, + "format": "heatmap" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { "type": "linear" } + }, + "color": { "mode": "continuous-RdYlGr" }, + "min": 0, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "calculate": false, + "yAxis": { "axisLabel": "day_of_week (UTC; 0=Mon, 6=Sun)" }, + "color": { "mode": "scheme", "scheme": "RdYlGn", "reverse": true, "steps": 64 }, + "exemplars": { "color": "rgba(255,0,0,0.7)" }, + "cellGap": 1, + "showValue": "never", + "tooltip": { "show": true, "yHistogram": false } + } + }, + { + "type": "stat", + "id": 2, + "title": "Top bucket — $query", + "description": "The single most-concentrated (day_of_week, hour_of_day) bucket. Pair with the heatmap to focus on the highest-risk window.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 14 }, + "targets": [ + { + "refId": "A", + "expr": "topk(1, forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"})", + "instant": true, + "legendFormat": "Day {{day_of_week}} Hour {{hour_of_day}}" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1.0 }, + { "color": "orange", "value": 2.0 }, + { "color": "red", "value": 4.0 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "title": "Active buckets", + "description": "How many of the 168 (day, hour) buckets had at least one observed incident. A small count means activity is concentrated (worth preemptive action); a large count means activity is diffuse.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 8, "x": 8, "y": 14 }, + "targets": [ + { + "refId": "A", + "expr": "count(forecast_incident_recurrence_score{group=\"$group\", id=\"$query\"} > 0)", + "instant": true, + "legendFormat": "non-zero buckets" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [] + } + } + ] +} diff --git a/dashboards/grafana/promforecast-is-it-working.json b/dashboards/grafana/promforecast-is-it-working.json index 250fa9a..3210dd3 100644 --- a/dashboards/grafana/promforecast-is-it-working.json +++ b/dashboards/grafana/promforecast-is-it-working.json @@ -272,6 +272,61 @@ "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] } ] } + }, + { + "type": "table", + "id": 30, + "title": "Fleet capacity stress test — ranked by projected breach", + "description": "Ranks every threshold-bearing series by the projected time-to-threshold under *current* load. Pair with `POST /forecast/what-if/fleet` (opt-in via server.expose_what_if_endpoint) to compute the same ranking under a hypothetical traffic multiplier (`{\"traffic_multiplier\": 1.5, \"horizon\": \"90d\"}`) — the endpoint returns the same shape, but with the multiplier applied. The throttle is set by safety.fleet_what_if_min_interval (5m default).", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 27 }, + "targets": [ + { + "refId": "A", + "expr": "sort({__name__=~\".+_forecast_time_to_threshold_seconds\"})", + "format": "table", + "instant": true, + "legendFormat": "{{__name__}} {{name}}" + } + ], + "transformations": [ + { "id": "organize", + "options": { + "excludeByName": { "Time": true, "job": true, "instance": true, "cluster": true, "replica": true }, + "renameByName": { + "Value": "time_to_threshold_seconds", + "__name__": "metric", + "name": "threshold", + "group": "group", + "id": "query", + "model": "model", + "severity": "severity" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { "align": "left" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "green", "value": 86400 } + ] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "time_to_threshold_seconds" }, + "properties": [ + { "id": "unit", "value": "s" }, + { "id": "custom.cellOptions", "value": { "type": "color-text" } } + ] + } + ] + } } ] } diff --git a/docs/README.md b/docs/README.md index 99a5b1a..4ed020d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,7 +37,11 @@ metrics, calibration, explainability, and ad-hoc what-if endpoints. - [Explainability (decomposition)](outputs/explainability.md) — trend / seasonal / level / residual split of the forecast curve - [Forecast drift](outputs/drift.md) — prediction-vs-prediction shift telemetry - [Forecast preview](outputs/preview.md) — opt-in ad-hoc what-if endpoint +- [Fleet capacity what-if](outputs/what-if-fleet.md) — multi-service stress-test endpoint that ranks projected breaches under a hypothetical traffic multiplier +- [Deploy risk score](outputs/deploy-safety.md) — deploy-anchored composite of deviation across golden signals; pair with Argo Rollouts / Flagger as a rollback gate +- [Recurring incident patterns](outputs/incident-patterns.md) — 7 x 24 heatmap of when incidents tend to land; for queries with `data_profile: events` - [Canary forecasts](outputs/canary.md) +- [Predictive autoscaling](predictive-autoscaling.md) — Kubernetes custom-metrics adapter that exposes forecast metrics to HorizontalPodAutoscaler for scaling on the projected load curve ## Operations diff --git a/docs/operations/cardinality.md b/docs/operations/cardinality.md index db34cfc..7acaebd 100644 --- a/docs/operations/cardinality.md +++ b/docs/operations/cardinality.md @@ -48,6 +48,8 @@ Notation: | `forecast_conditional_band_calibration_offset` | `S × M × L` | `emission.conditional: true` *and* `accuracy.conformal.enabled: true` | | SLO error budget | `S × M × N_windows × (3 + L)` | `slo:` configured (3 sibling families × N_windows, plus level fan-out on exhausted_in) | | Recommended maintenance windows | `S × M × count` | `recommended_windows.enabled: true` | +| `forecast_deploy_risk_score` | `max_active_deploys` per service | One row per active deploy while inside `active_window`; bounded per service by `deploy_signals[].max_active_deploys`. Charged against the *global* budget, not the per-query series cap. | +| `forecast_incident_recurrence_score` | `168` per opted-in query | Fixed 7 days × 24 hours grid per query with `data_profile: events`. Independent of `S` — the heatmap is the aggregate across whatever the counter PromQL fans out into. | | Operational + preprocessing families | ~7 per group + ~6 per query | Mostly always on | ## Working example diff --git a/docs/outputs/deploy-safety.md b/docs/outputs/deploy-safety.md new file mode 100644 index 0000000..7ff3d03 --- /dev/null +++ b/docs/outputs/deploy-safety.md @@ -0,0 +1,100 @@ +# Deploy risk score + +`forecast_deploy_risk_score` turns multiple golden-signal deviation ratios into a single deploy-anchored risk number. +Instead of manually correlating latency, errors, saturation, and traffic during a rollout, you get one gauge per active deploy that SREs can alert on. + +The metric only exists while a deploy is inside its **active window** — background noise never pollutes dashboards. + +## Configuration + +```yaml +deploy_signals: + - service: payments-api + anchor: + type: regressor + query: payments_request_rate + regressor_id: deploys + threshold: 0 # transition from <=0 to >0 + signals: + - query: payments_request_rate + weight: 0.5 + - query: payments_p95_latency + weight: 1.0 + - query: payments_error_rate + weight: 2.0 + active_window: 30m + max_active_deploys: 8 +``` + +| Field | Default | Meaning | +|------------------|---------|---------| +| `service` | — | Human-readable service name | +| `anchor` | — | Detects deploy start (via regressor transition) | +| `signals` | — | Weighted golden signals (`errors ≥ latency ≥ saturation ≥ traffic` is conventional) | +| `active_window` | — | How long the score stays emitted after detection | +| `max_active_deploys` | `8` | LRU cap per service to bound cardinality | + +## Emitted metric + +``` +forecast_deploy_risk_score{ + service="payments-api", + deployment_id="payments-api-1715000000", + contributing_signals="3" +} 0.42 +``` + +- **Value** — weighted mean of absolute `forecast_deviation_ratio` across contributing signals (higher = more risk). +- `deployment_id` — stable identifier for the current deploy. +- `contributing_signals` — number of signals that produced a finite deviation this cycle. + +## Anchor detection + +The anchor watches the configured regressor on the anchor query. +A deploy is registered when the leading value crosses `threshold` from ≤ threshold to > threshold. +A persistently high regressor counts as **one** deploy, not repeated triggers. + +Typical anchor regressor: + +```yaml +regressors: + - id: deploys + promql: changes(kube_deployment_status_observed_generation{deployment="payments-api"}[5m]) +``` + +## Worked examples + +**Argo Rollouts** +Expose rollout phase changes as a counter and reference it as the anchor. +Argo’s analysis backend can read `forecast_deploy_risk_score` directly and mark the rollout `Failed` if the score exceeds your threshold. + +**Flagger** +Define a `MetricTemplate` that queries the risk score and attach it to your Canary resource. +A sustained high score triggers automatic rollback. + +## Cardinality + +At most `max_active_deploys` rows per service at any time. +With 10 services the worst-case total is ~80 rows — safely bounded by LRU eviction. + +## Example alerts + +```promql +# High risk during rollout +forecast_deploy_risk_score{contributing_signals!~"0|1"} > 0.5 + +# Critical risk +forecast_deploy_risk_score{contributing_signals!~"0|1"} > 1.0 +``` + +## Operational notes + +- With HA enabled, only the leader computes scores (followers serve the cached snapshot). +- Config reloads reset the tracker; active deploys disappear until re-detected. +- Score is computed from the current snapshot’s deviation rows — a deploy registered before the first signal fit will temporarily show `contributing_signals: 0`. + +## See also + +- [Deviation metrics](deviation.md) — the signals the score is built from +- [Drift telemetry](drift.md) — distinct from deploy-anchored risk +- [Canary / weighted groups](canary.md) — single-metric alternative when you have only one golden signal diff --git a/docs/outputs/incident-patterns.md b/docs/outputs/incident-patterns.md new file mode 100644 index 0000000..87dc5fb --- /dev/null +++ b/docs/outputs/incident-patterns.md @@ -0,0 +1,90 @@ +# Recurring incident pattern detection + +`forecast_incident_recurrence_score` turns a stream of discrete incident events into a **7×24 heatmap** of recurrence intensity. +It answers “when do incidents tend to happen?” so you can proactively staff on-call or harden systems around predictable hot spots. + +This is discrete-event forecasting (not continuous metrics) and sits at the boundary with the sibling `promanomaly` project. + +## Configuration + +Mark the query with `data_profile: events`: + +```yaml +groups: + - name: incidents + queries: + - id: pagerduty_incidents_total + promql: sum(pagerduty_incidents_total) + data_profile: events +``` + +The PromQL must return a counter that increments on each incident. The runner aggregates all returned series into a single event stream before bucketing. + +Typical sources: +- `pagerduty_incidents_total` +- `alertmanager_notifications_total{severity="critical"}` +- A custom webhook counter + +## Emitted metric + +A full 168-cell grid is always published: + +``` +forecast_incident_recurrence_score{ + group="incidents", + id="pagerduty_incidents_total", + day_of_week="0", # 0 = Monday … 6 = Sunday + hour_of_day="14" +} 2.5 +``` + +- Value = events in this bucket ÷ mean events across all non-empty buckets. + `> 1.0` = above-average concentration; `0` = never observed. + +## Interpretation guide + +| Score | Meaning | +|-----------|--------------------------------------| +| `0` | No events observed | +| `< 1.0` | Below-average activity | +| `1.0` | Exactly average | +| `1.5–3.0` | 50–200 % above average | +| `> 3.0` | Strong, reliable hot spot | + +Look at both the score **and** the overall distribution shape: +- Few high-score buckets → predictable pattern (act on it). +- Many low-score buckets → random noise (descriptive only). + +## Worked examples + +**PagerDuty** +```yaml +- id: pagerduty_incidents_total + promql: sum(pagerduty_incidents_total{service=~"$service_pattern"}) + data_profile: events +``` + +**Alertmanager** +```yaml +- id: alertmanager_critical_notifications + promql: sum(alertmanager_notifications_total{severity="critical"}) + data_profile: events +``` + +**Custom webhook** +Increment a Prometheus `Counter("incidents_total")` in your webhook handler and scrape it normally. + +## Operational notes + +- No forecast curve, bands, or deviation metrics are emitted — only the heatmap. +- Fixed cardinality: exactly **168 rows** per opted-in query (counted by `promforecast validate`). +- Time-of-day is always UTC. +- Counter resets (negative deltas) are treated as zero events. + +The bundled `promforecast-incident-heatmap.json` dashboard renders the heatmap with group/query selectors and includes “Active buckets” + “Top bucket” stat panels for quick triage. + +## Cross-tool overlay (roadmap) + +A future dashboard will overlay this recurrence score with `promanomaly` outputs to answer “did the pattern hold this week, and did the anomaly detector agree?”. + +Until then, the heatmap stands alone as a preemptive-staffing signal. diff --git a/docs/outputs/what-if-fleet.md b/docs/outputs/what-if-fleet.md new file mode 100644 index 0000000..fb525f2 --- /dev/null +++ b/docs/outputs/what-if-fleet.md @@ -0,0 +1,110 @@ +# Fleet capacity what-if endpoint + +`POST /forecast/what-if/fleet` answers the **strategic** question: +“If traffic grows by X% over the next N days, which services will hit their capacity ceilings first?” + +It re-uses the single-query preview machinery internally, looping over every query that defines a capacity threshold (`gt` or `lt` comparators only) and ranking the projected ETAs. + +## Enabling + +```yaml +server: + expose_what_if_endpoint: true # default: false + +safety: + fleet_what_if_min_interval: 5m # default: 5m +``` + +The endpoint follows the same network-level access model as `/forecast/preview` and `/-/reload`. + +A rate limit prevents the endpoint from starving scheduled fits. Requests that hit the throttle return `429 Too Many Requests` with a `Retry-After` header and this JSON body: + +```json +{ + "error": "throttled", + "message": "fleet what-if throttled; retry after 287s", + "retry_after_seconds": 287.4 +} +``` + +## Request body + +```json +{ + "traffic_multiplier": 1.5, + "horizon": "90d", + "groups": ["payments", "search"] +} +``` + +| Field | Required | Notes | +|----------------------|----------|-------| +| `traffic_multiplier` | Yes | Positive number (`1.0` = baseline ranking) | +| `horizon` | No | Prometheus duration or seconds; defaults to `defaults.horizon` | +| `groups` | No | Restrict scan to listed groups (unknown groups → 400) | + +The multiplier is applied by: +1. **Regressor scaling** (preferred) — if the query has a traffic regressor (`traffic`, `rps`, `qps`, `request_rate`, etc.), the value is overridden. +2. **Baseline fallback** — otherwise the preview runs unchanged and the response row flags `traffic_multiplier_applied: "baseline"`. + +## Response + +```json +{ + "horizon_seconds": 7776000, + "traffic_multiplier": 1.5, + "served_by": "promforecast-0", + "evaluated_query_count": 24, + "services": [ + { + "query_id": "payments_db_storage_used_bytes", + "group": "infra", + "threshold_name": "warning", + "threshold_value": 858993459200, + "comparator": "gt", + "time_to_threshold_seconds": 86400.0, + "model": "AutoARIMA", + "traffic_multiplier_applied": "regressor" + } + ] +} +``` + +- Rows are ranked by ascending `time_to_threshold_seconds` (sooner breaches first). +- Finite ETAs sort ahead of non-finite (`NaN` = no breach within horizon). +- Failed previews appear with an `"error"` field but do not break the rest of the ranking. + +## Worked examples + +**Q4 capacity planning** +```bash +curl -s -X POST http://promforecast:9091/forecast/what-if/fleet \ + -H 'content-type: application/json' \ + -d '{"traffic_multiplier": 1.4, "horizon": "90d"}' \ + | jq '.services[] | select(.time_to_threshold_seconds < 7776000)' +``` + +**Post-acquisition absorption** +```bash +curl -s -X POST ... \ + -d '{"traffic_multiplier": 1.3, "horizon": "90d", "groups": ["api", "ingest"]}' +``` + +**Black Friday peak simulation** +```bash +curl -s -X POST ... \ + -d '{"traffic_multiplier": 3.0, "horizon": "7d"}' \ + | jq '.services[] | select(.time_to_threshold_seconds < 86400)' +``` + +## Operational notes + +- Throttle is per-replica (in-memory). In HA mode each replica enforces its own limit. +- The endpoint is stateless and emits no metrics. +- The bundled “Is it working?” dashboard includes a *Fleet capacity stress test* table that shows the same shape — pair it with a notebook curl for baseline vs. overlay comparisons. + +## See also + +- [`preview.md`](preview.md) — single-query what-if this endpoint composes over +- [`capacity.md`](capacity.md) — the underlying `time_to_threshold_seconds` metric +- [`maintenance-windows.md`](maintenance-windows.md) — ranked quiet windows instead of ranked services diff --git a/docs/predictive-autoscaling.md b/docs/predictive-autoscaling.md new file mode 100644 index 0000000..fe4dbfb --- /dev/null +++ b/docs/predictive-autoscaling.md @@ -0,0 +1,146 @@ +# Predictive autoscaling with Kubernetes HPA + +The `promforecast-metrics-adapter` sub-component exposes promforecast forecasts through the Kubernetes custom-metrics API. +HorizontalPodAutoscaler can then scale **on the forecast** of load 15–30 minutes ahead instead of reacting to current load — capacity arrives *before* the spike. + +The adapter is a separate deployable (own Helm chart and container entrypoint). The main forecaster is unaffected. + +## Architecture + +```mermaid +flowchart TD + F["promforecast forecaster"] + VM[("VictoriaMetrics
long-term TSDB
(forecasts via sink)")] + A["promforecast-metrics-adapter
(this sub-chart)
• external.metrics.k8s.io
• custom.metrics.k8s.io
• serves TLS"] + K["kube-apiserver
aggregator front"] + H["HorizontalPodAutoscaler"] + + F -->|remote_write forecasts| VM + VM -->|PromQL| A + A -->|TLS, via APIService| K + K -->|polls| H +``` + +Stateless: the adapter only reads forecasts that the main forecaster has already written to VictoriaMetrics via the sink. + +## Install + +```bash +helm install promforecast-adapter ./charts/promforecast-metrics-adapter \ + --namespace monitoring \ + --set datasourceUrl=http://victoriametrics:8428 \ + --values - <<'EOF' +metrics: + - externalMetricName: rps-forecast + promql: 'sum by (service)(http_requests_rate_forecast{service="$service"})' + namespaced: true + - externalMetricName: cpu-forecast + promql: 'avg(node_cpu_busy_pct_forecast)' + namespaced: false +EOF +``` + +The chart creates a self-signed cert, registers the two required `APIService` objects, and sets up RBAC. No cert-manager needed. + +## Config reference + +```yaml +# values.yaml +datasourceUrl: http://victoriametrics:8428 + +metrics: + - externalMetricName: rps-forecast + promql: | + sum by (service)( + http_requests_rate_forecast{service="$service"} + ) + namespaced: true + +tls: + generate: true # self-signed cert + validityDays: 3650 + # existingSecret: my-adapter-tls # optional +``` + +**Placeholders** in `promql` are substituted from the HPA’s label selector. Namespaced metrics automatically get `namespace=$namespace`. + +## HPA example + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: payments-api-predictive +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: payments-api + minReplicas: 3 + maxReplicas: 30 + metrics: + - type: External + external: + metric: + name: rps-forecast + selector: + matchLabels: + service: payments-api + target: + type: AverageValue + averageValue: "50" + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + scaleDown: + stabilizationWindowSeconds: 600 +``` + +A full example ships in `examples/k8s/predictive-hpa.yaml`. + +## Quality gate + +Forecast quality is a hard prerequisite for safe predictive scaling. +Common pattern: run **two HPAs** on the same target: + +- **Primary** — predictive HPA (the one above). +- **Fallback** — classic reactive HPA on CPU / request rate with a higher target. + +Kubernetes `selectPolicy: Max` ensures the more aggressive recommendation wins. The predictive HPA usually leads; the reactive one is the safety net. + +For tighter control you can gate the predictive HPA on `forecast_quality_score > 0.6` (small operator or custom metric). + +## Failure modes + +| State | Adapter response | HPA sees | +|--------------------------------|---------------------------|---------------------------| +| VM unreachable | 503 ServiceUnavailable | Metric unavailable | +| Metric not found | 404 NotFound | `failedGetMetric` | +| PromQL returns no series | 200 with value `0` | Treated as zero | +| Forecast is NaN / Inf | 200 with value `0` | Treated as zero | + +Use `clamp_min(..., 0.001)` in the PromQL if you need to distinguish “no data” from “real zero”. + +## Verification + +```bash +# Check pod and APIServices +kubectl -n monitoring get pods -l app.kubernetes.io/name=promforecast-metrics-adapter +kubectl get apiservice v1beta1.external.metrics.k8s.io + +# Test the metric directly +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/payments/rps-forecast" | jq . +``` + +## Operational notes + +- **TLS cert rotation** — the chart reuses the existing Secret on `helm upgrade`. To force rotation, delete the `-tls` Secret. +- **No hot-reload** — metric mappings are part of the Kubernetes API surface, so changes require a Deployment rollout. +- **Compatibility** — if you already run prometheus-adapter, set `apiService.enabled: false` on this chart. +- **Single replica is sufficient** — the adapter is stateless and HPA retries on 503. +- **Security** — the adapter trusts the kube-apiserver aggregator. Use a NetworkPolicy pinned to the apiserver pods in multi-tenant clusters. + +See also: +- [`preview.md`](preview.md) — single-query what-if +- [`capacity.md`](capacity.md) — the underlying forecast metrics +- [`maintenance-windows.md`](maintenance-windows.md) — ranked quiet windows diff --git a/examples/alerts/promforecast-rules.yaml b/examples/alerts/promforecast-rules.yaml index bcfd15d..8139eff 100644 --- a/examples/alerts/promforecast-rules.yaml +++ b/examples/alerts/promforecast-rules.yaml @@ -404,3 +404,46 @@ spec: above 4h. The queue is being worked off, but the backlog is too large to clear before downstream SLAs are at risk. Add consumer capacity or shed load. + + # ---------------------------------------------------------------- + # Deploy-anchored risk score alerts. + # ---------------------------------------------------------------- + # These rules assume the ``deploy_signals:`` top-level config + # block is populated with at least one service. The forecaster + # emits ``forecast_deploy_risk_score{deployment_id, service, + # contributing_signals}`` per active deploy; an empty + # ``deploy_signals`` produces no series and the alerts stay + # quiet. ``contributing_signals >= 2`` keeps the alert from + # firing on single-signal blips. + - name: promforecast.deploy.examples + interval: 1m + rules: + - alert: DeployHighRisk + expr: | + forecast_deploy_risk_score{contributing_signals!~"0|1"} > 0.5 + for: 5m + labels: + severity: warning + annotations: + summary: "Deploy {{ $labels.deployment_id }} on {{ $labels.service }} shows elevated risk" + description: | + The composite of |forecast_deviation_ratio| across the + configured signals for {{ $labels.service }} exceeded + 0.5 for 5m after a deploy. Consider rolling back if + the deviation persists or worsens — see DeployCriticalRisk. + + - alert: DeployCriticalRisk + expr: | + forecast_deploy_risk_score{contributing_signals!~"0|1"} > 1.0 + for: 2m + labels: + severity: critical + annotations: + summary: "Deploy {{ $labels.deployment_id }} on {{ $labels.service }} is materially degrading signals" + description: | + The composite of |forecast_deviation_ratio| across the + configured signals for {{ $labels.service }} exceeded + 1.0 for 2m — at least one signal is a full band-width + outside its forecast. Roll back unless the deploy is + expected to shift the metric (in which case retrain + the model with the new regime). diff --git a/examples/configs/deploy-and-incidents.yaml b/examples/configs/deploy-and-incidents.yaml new file mode 100644 index 0000000..fdf562c --- /dev/null +++ b/examples/configs/deploy-and-incidents.yaml @@ -0,0 +1,113 @@ +# Example forecaster config exercising the two predictive-ops emission +# families: +# +# 1. ``deploy_signals:`` — per-service composite risk score anchored at +# each rollout. Pair with the ``DeployHighRisk`` / ``DeployCriticalRisk`` +# example alerts in examples/alerts/promforecast-rules.yaml. +# +# 2. ``data_profile: events`` — incident-pattern detection. The runner +# routes the query to the 7×24 recurrence heatmap path instead of +# fitting a forecast curve. Pair with the +# ``promforecast-incident-heatmap`` Grafana dashboard. +# +# Both families are off unless explicitly configured. Adapt the metric +# names and label selectors to your environment. + +apiVersion: promforecast.io/v1 + +datasource: + url: http://victoriametrics:8428/ + +# All queries below aggregate to ~1 series per service via ``sum by``. +# Lower the per-query cap accordingly so ``promforecast validate``'s +# cardinality estimate reflects reality rather than the worst case +# ``safety.max_series_per_query`` default (500). Operators with +# higher fan-out (per-pod / per-instance breakdowns) should raise this. +safety: + max_series_per_query: 10 + +defaults: + lookback: 14d + step: 5m + horizon: 6h + models: [AutoARIMA, SeasonalNaive] + season_length: 288 + accuracy: + evaluate: false + auto_select: true + fallback_model: SeasonalNaive + +groups: + - name: deploy_safety + priority: 1 + queries: + # Latency (p95). Used by the deploy-risk composite below. + - id: payments_p95_latency_seconds + promql: | + histogram_quantile( + 0.95, + sum by (service, le) (rate(http_request_duration_seconds_bucket{service="payments"}[5m])) + ) + regressors: + # The ``deploys`` regressor below is what the deploy-risk + # block anchors against. ``changes(...)`` ticks once per + # rollout that bumps the observed generation. + - id: deploys + promql: | + changes(kube_deployment_status_observed_generation{deployment="payments"}[5m]) + + # Error rate. Second contributor to the composite. + - id: payments_5xx_per_second + promql: | + sum by (service) (rate(http_requests_total{service="payments", status=~"5.."}[5m])) + + # Saturation proxy (CPU). Third contributor. + - id: payments_cpu_seconds_rate + promql: | + sum by (service) (rate(container_cpu_usage_seconds_total{pod=~"payments-.*"}[5m])) + + - name: incidents + priority: 2 + queries: + # PagerDuty incidents bucketed into a 7×24 recurrence heatmap. + # Counter resets (target restart) are dropped to zero — see + # promforecast.incident_pattern.counter_to_event_counts. + - id: pagerduty_incidents_total + promql: pagerduty_incidents_total{service=~"payments|checkout|search"} + data_profile: events + + # Alertmanager critical notifications, same treatment. Pairs + # well with the heatmap dashboard's ``query`` template selector. + - id: alertmanager_critical_notifications + promql: alertmanager_notifications_total{severity="critical"} + data_profile: events + +# ---------------------------------------------------------------------- +# Per-service deploy-anchored risk composite. +# +# Cardinality: ``max_active_deploys`` rows per service while at least +# one deploy is inside its ``active_window``. Outside the window the +# rows disappear from /metrics — the gauge is silent at steady state. +# +# Weights are conventionally latency >= traffic >= errors >= saturation +# (latency moves first under a bad deploy); operators tune per service. +# ---------------------------------------------------------------------- +deploy_signals: + - service: payments + anchor: + type: regressor + query: payments_p95_latency_seconds + regressor_id: deploys + threshold: 0 + signals: + - query: payments_p95_latency_seconds + weight: 2.0 + - query: payments_5xx_per_second + weight: 1.5 + - query: payments_cpu_seconds_rate + weight: 1.0 + # How long after a detected rollout the gauge stays live. + # 30 minutes is the conventional window for Argo Rollouts / Flagger + # analysis runs; tune to your rollout duration. + active_window: 30m + max_active_deploys: 4 diff --git a/examples/k8s/predictive-hpa.yaml b/examples/k8s/predictive-hpa.yaml new file mode 100644 index 0000000..53dc190 --- /dev/null +++ b/examples/k8s/predictive-hpa.yaml @@ -0,0 +1,71 @@ +# Example: predictive HPA scaling on a promforecast forecast metric. +# +# The HPA references the external metric ``rps-forecast`` — the +# metric name as registered with the promforecast-metrics-adapter +# (see the adapter chart's ``metrics:`` array). The forecast value +# is delivered to HPA via the Kubernetes custom-metrics API +# aggregator; the adapter resolves it by querying VictoriaMetrics. +# +# **Quality gate.** A bad forecast can produce a runaway scale-up +# or scale-down recommendation. Pair the predictive metric with a +# ``forecast_quality_score`` gate so the HPA only acts when the +# underlying forecast is trustworthy. The conventional pattern is a +# second HPA scaling on a different signal as a fallback (e.g. raw +# CPU) so the predictive HPA is the primary and the reactive HPA is +# the safety net. +# +# **Stabilization window.** The ``behavior.scaleDown.stabilizationWindowSeconds`` +# is set deliberately wide (10 minutes) so a transient bad forecast +# doesn't cause a flap. Predictive scaling is a "scale up early, +# scale down slowly" workflow — the standard scaling-policy defaults +# bias the wrong way for a predictive signal. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: payments-api-predictive + namespace: payments +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: payments-api + minReplicas: 3 + maxReplicas: 30 + metrics: + # Primary: predictive forecast metric. The HPA target value is + # in the same units as the underlying forecast — e.g. if the + # forecast is requests-per-second per pod, set the target to + # the desired RPS-per-pod. + - type: External + external: + metric: + name: rps-forecast + selector: + matchLabels: + service: payments-api + target: + type: AverageValue + averageValue: "50" + behavior: + scaleUp: + # Scale up promptly — the whole point of predictive scaling is + # to provision capacity before the spike arrives. + stabilizationWindowSeconds: 0 + policies: + - type: Pods + value: 4 + periodSeconds: 30 + - type: Percent + value: 50 + periodSeconds: 30 + selectPolicy: Max + scaleDown: + # Scale down slowly. A transient bad forecast that + # under-projects load briefly should not cause the HPA to + # drop pods. + stabilizationWindowSeconds: 600 + policies: + - type: Pods + value: 1 + periodSeconds: 60 + selectPolicy: Min diff --git a/forecaster/src/promforecast/adapter.py b/forecaster/src/promforecast/adapter.py new file mode 100644 index 0000000..8572d85 --- /dev/null +++ b/forecaster/src/promforecast/adapter.py @@ -0,0 +1,580 @@ +"""Kubernetes custom-metrics adapter for predictive autoscaling. + +A separate sub-component that implements the Kubernetes custom-metrics API +contracts: + +* ``external.metrics.k8s.io/v1beta1`` — flat (cluster-scoped or + namespace-scoped) metrics referenced by HPA's ``external`` metric + type. The HPA's ``targetValue`` is compared against the value + returned here. +* ``custom.metrics.k8s.io/v1beta1`` — pod / object-scoped metrics + with selectors. Less common in practice; the predictive + autoscaling use-case usually fits the external API better + because the forecast is per-service rather than per-pod. The + adapter implements a minimal stub for the custom-metrics surface + so HPAs that prefer that flavour still get a sensible response. + +The adapter reads from VM via PromQL — it does **not** run the +forecast pipeline. Forecasts are produced by the main forecaster +and stored in VM via the sink; the adapter just exposes them under +HPA-compatible names. Stateless, side-effect-free. + +Mounted behind the Kubernetes API aggregator via an +``APIService`` resource. The aggregator requires TLS; the adapter's +Helm chart generates a self-signed cert at install time so the +operator doesn't need cert-manager just to stand this up. + +**Why a separate process?** The HPA control loop polls the adapter +every 15 seconds. Embedding the adapter into the forecaster would +mean the forecaster's HTTP surface had to be cluster-API-shaped +(TLS, APIService-fronted), which complicates the simple +``/metrics`` scrape model. A small separate adapter, talking to the +same VM, keeps each component's contract tight. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import re +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import structlog +import yaml +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field + +logger = structlog.get_logger(__name__) + + +class _MetricMapping(BaseModel): + """One promforecast metric exposed to HPA. + + ``external_metric_name`` is the name HPA references in its + ``external.metric.name`` field. Must conform to the Kubernetes + metric-name regex ([a-z0-9-]+) so the API server accepts it. + + ``promql`` is the PromQL query the adapter runs against VM to + resolve the metric. The query should return a scalar / single + value — multi-series responses are summed across labels (the + HPA only consumes a scalar). For per-pod or per-namespace + autoscaling, parametrise the PromQL with the conventional + ``namespace`` / ``pod`` labels and let the adapter substitute + them at query time. + + ``namespaced: true`` (default) routes the metric under the + namespaced HPA path. Set false for cluster-scoped metrics that + HPA can reference regardless of namespace. + """ + + model_config = ConfigDict(extra="forbid") + + external_metric_name: str = Field(pattern=r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") + promql: str + namespaced: bool = True + + +class _AdapterConfig(BaseModel): + """Top-level adapter config. + + Mounted via a ConfigMap in the standard install. The schema is + deliberately minimal — the adapter's job is to translate + HPA-shaped queries to PromQL; richer behaviour belongs on the + forecaster. + """ + + model_config = ConfigDict(extra="forbid") + + datasource_url: str + metrics: list[_MetricMapping] = Field(default_factory=list) + # HTTP listen address. The chart's APIService points to the + # service's port; the deployment's pod listens on this port. + listen: str = ":6443" + # Per-PromQL request timeout. The HPA's poll cadence is 15s; + # tighter than that and we'd risk consecutive misses cascading + # into HPA marking the metric "unavailable". + query_timeout_seconds: float = 5.0 + + +# --------------------------------------------------------------------------- +# K8s API JSON shapes. These mirror the upstream Go types; the adapter +# emits the exact JSON shape the API aggregator expects. +# --------------------------------------------------------------------------- + + +def _api_group_list() -> dict[str, Any]: + """Body for ``GET /apis/`` (group discovery).""" + return { + "kind": "APIGroupList", + "apiVersion": "v1", + "groups": [ + { + "name": "external.metrics.k8s.io", + "versions": [ + { + "groupVersion": "external.metrics.k8s.io/v1beta1", + "version": "v1beta1", + } + ], + "preferredVersion": { + "groupVersion": "external.metrics.k8s.io/v1beta1", + "version": "v1beta1", + }, + }, + { + "name": "custom.metrics.k8s.io", + "versions": [ + { + "groupVersion": "custom.metrics.k8s.io/v1beta1", + "version": "v1beta1", + } + ], + "preferredVersion": { + "groupVersion": "custom.metrics.k8s.io/v1beta1", + "version": "v1beta1", + }, + }, + ], + } + + +def _external_resource_list(metrics: list[_MetricMapping]) -> dict[str, Any]: + """Body for ``GET /apis/external.metrics.k8s.io/v1beta1`` (resource listing).""" + return { + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "external.metrics.k8s.io/v1beta1", + "resources": [ + { + "name": m.external_metric_name, + "singularName": "", + "namespaced": m.namespaced, + "kind": "ExternalMetricValueList", + "verbs": ["get"], + } + for m in metrics + ], + } + + +def _custom_resource_list() -> dict[str, Any]: + """Body for ``GET /apis/custom.metrics.k8s.io/v1beta1`` (resource listing). + + Minimal placeholder — the adapter doesn't ship per-object + custom metrics today. The empty list is a valid response + that tells the API aggregator "this group exists, no + metrics under it yet" rather than 404, which would make HPA + list-time errors noisier than needed. + """ + return { + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "custom.metrics.k8s.io/v1beta1", + "resources": [], + } + + +def _external_metric_value_list( + *, + metric_name: str, + value: float, + metric_labels: dict[str, str] | None = None, +) -> dict[str, Any]: + """Body for ``GET /apis/external.metrics.k8s.io/v1beta1/.../``. + + The Kubernetes API expects ``value`` as a Quantity string — + integers without a unit when the value is whole, decimals when + fractional. Render as a plain string and let the API parse it. + """ + return { + "kind": "ExternalMetricValueList", + "apiVersion": "external.metrics.k8s.io/v1beta1", + "metadata": {}, + "items": [ + { + "metricName": metric_name, + "metricLabels": metric_labels or {}, + "timestamp": datetime.now(tz=UTC) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "value": _format_quantity(value), + } + ], + } + + +def _format_quantity(value: float) -> str: + """Render a float as a K8s Quantity string. + + Integers come out without a decimal so HPA's parser accepts them + cleanly; fractional values get six digits of precision (matches + Prometheus's ``%g``-flavoured exposition). + + **NaN / Inf -> "0" trade-off.** HPA's ``failedGetMetric`` event + (the alternative shape: return a non-200) marks the metric + unavailable, which keeps the autoscaler on its last good + recommendation indefinitely — *fail open* in HPA's terms. For a + forecast that hasn't produced a sample yet (cold-start, ramp-up, + transient query failure) the operator usually wants the HPA to + *fail closed* — i.e. fall back to whatever ``minReplicas`` says + rather than freeze in place. Returning ``"0"`` achieves that: + a target like ``averageValue: 10`` against a current value of + ``0`` triggers a scale-down toward minReplicas. Operators who + want the opposite shape (freeze on NaN) should wrap their + PromQL with ``clamp_min(..., 0.001)`` and a sentinel value + upstream — that's a downstream config decision the adapter + can't make safely on its own. + """ + import math # noqa: PLC0415 + + if not math.isfinite(value): + return "0" + if value.is_integer(): + return str(int(value)) + return f"{value:.6f}".rstrip("0").rstrip(".") + + +# --------------------------------------------------------------------------- +# PromQL fetch shim. The adapter uses httpx directly rather than the +# forecaster's ``PromSource`` because the adapter's contract is +# substantively narrower — single-value queries with a short timeout +# — and pulling in the full source would drag scheduling / retry +# infrastructure the adapter doesn't need. +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _PromValue: + """One scalar result from PromQL.""" + + value: float + timestamp: datetime + labels: dict[str, str] + + +async def _query_promql( + *, + client: httpx.AsyncClient, + base_url: str, + promql: str, + timeout_seconds: float, +) -> list[_PromValue]: + """Run a PromQL instant query and return the matching values. + + The adapter uses ``/api/v1/query`` (instant) rather than + ``/api/v1/query_range`` (range) — HPA wants the most recent + value, not a time series. VM accepts the same endpoint shape + as Prometheus. + + Returns an empty list when the query has no matching series. + Raises ``httpx.HTTPError`` on transport failure; the caller + translates that into a 500 / 503 response. + """ + url = base_url.rstrip("/") + "/api/v1/query" + response = await client.get(url, params={"query": promql}, timeout=timeout_seconds) + response.raise_for_status() + body = response.json() + if body.get("status") != "success": + return [] + result = body.get("data", {}).get("result", []) + values: list[_PromValue] = [] + for entry in result: + metric_labels = {str(k): str(v) for k, v in entry.get("metric", {}).items()} + value_pair = entry.get("value") + if not value_pair or len(value_pair) != 2: # noqa: PLR2004 + continue + try: + ts = datetime.fromtimestamp(float(value_pair[0]), tz=UTC) + val = float(value_pair[1]) + except (TypeError, ValueError): + continue + values.append(_PromValue(value=val, timestamp=ts, labels=metric_labels)) + return values + + +# --------------------------------------------------------------------------- +# FastAPI app construction. +# --------------------------------------------------------------------------- + + +def _build_app(config: _AdapterConfig) -> FastAPI: + """Wire up the adapter HTTP surface against ``config``. + + The app is built once per process — the metric list is static + over the process lifetime. Reloading happens via process + restart (Helm ``kubectl rollout restart``) so the contract + stays as simple as a stateless adapter can be. + """ + metrics_by_name = {m.external_metric_name: m for m in config.metrics} + + @asynccontextmanager + async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]: + # If a test pre-seeded a client (or a previous lifespan left + # one behind), close it before installing the one we own — + # otherwise we leak a connection pool on every restart. + prior = getattr(fastapi_app.state, "http", None) + if prior is not None: + with contextlib.suppress(Exception): + await prior.aclose() + async with httpx.AsyncClient() as client: + fastapi_app.state.http = client + yield + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + # Lazily build the http client on first request so tests that + # bypass ``lifespan`` (bare ``TestClient(app)`` without a ``with`` + # block) get a usable client without us leaking a pool that the + # lifespan would otherwise replace and never close. Production + # boots through ``lifespan`` and never hits the lazy path. + app.state.http = None + + @app.get("/healthz") + async def healthz() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/apis") + async def apis() -> Response: + return JSONResponse(_api_group_list()) + + @app.get("/apis/") + async def apis_slash() -> Response: + # The API aggregator may probe either form; both serve the + # same body so a trailing-slash mismatch doesn't break + # discovery. + return JSONResponse(_api_group_list()) + + @app.get("/apis/external.metrics.k8s.io/v1beta1") + async def external_metrics_list() -> Response: + return JSONResponse(_external_resource_list(config.metrics)) + + @app.get("/apis/external.metrics.k8s.io/v1beta1/") + async def external_metrics_list_slash() -> Response: + return JSONResponse(_external_resource_list(config.metrics)) + + @app.get("/apis/custom.metrics.k8s.io/v1beta1") + async def custom_metrics_list() -> Response: + return JSONResponse(_custom_resource_list()) + + async def _fetch_external_value(metric_name: str, label_selector: str | None) -> Response: + mapping = metrics_by_name.get(metric_name) + if mapping is None: + return JSONResponse( + { + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": f"metric {metric_name!r} is not configured", + "reason": "NotFound", + "code": 404, + }, + status_code=404, + ) + promql = _apply_label_selector(mapping.promql, label_selector) + # ``lifespan`` installs the client on app startup; the lazy + # path here only fires for tests that skip the lifespan + # (bare ``TestClient(app)`` without a ``with`` block). The + # lazy client lives for the rest of the app's lifetime; tests + # that care about cleanup should still use the context-manager + # form of TestClient. + if app.state.http is None: + app.state.http = httpx.AsyncClient() + client = app.state.http + try: + values = await _query_promql( + client=client, + base_url=config.datasource_url, + promql=promql, + timeout_seconds=config.query_timeout_seconds, + ) + except httpx.HTTPError as exc: + logger.warning( + "adapter_query_failed", + metric=metric_name, + error=str(exc), + ) + return JSONResponse( + { + "kind": "Status", + "apiVersion": "v1", + "status": "Failure", + "message": f"upstream query failed: {exc}", + "reason": "ServiceUnavailable", + "code": 503, + }, + status_code=503, + ) + # Sum across returned series. HPA wants a scalar; multi- + # series responses are nearly always per-pod or per-replica + # fan-outs that the operator wants aggregated for + # autoscaling purposes. Operators who want a different + # reduction wrap their PromQL with ``avg()`` / ``max()`` / + # ``topk()`` at config time. + aggregate = sum(v.value for v in values) if values else 0.0 + return JSONResponse( + _external_metric_value_list( + metric_name=metric_name, + value=aggregate, + ) + ) + + @app.get("/apis/external.metrics.k8s.io/v1beta1/{metric_name}") + async def external_metric_cluster(metric_name: str, request: Request) -> Response: + return await _fetch_external_value(metric_name, request.query_params.get("labelSelector")) + + @app.get("/apis/external.metrics.k8s.io/v1beta1/namespaces/{namespace}/{metric_name}") + async def external_metric_namespaced( + namespace: str, metric_name: str, request: Request + ) -> Response: + selector = request.query_params.get("labelSelector") + # The URL path's namespace is authoritative — the K8s API + # server has already enforced RBAC against it. Strip any + # caller-supplied ``namespace=`` pair from the labelSelector + # before appending our own so a caller can't bypass scoping by + # passing ``labelSelector=namespace=other``. Other pairs pass + # through. + cleaned = _strip_namespace_pair(selector) + ns_selector = f"namespace={namespace}" + merged = ns_selector if not cleaned else f"{cleaned},{ns_selector}" + return await _fetch_external_value(metric_name, merged) + + return app + + +def _strip_namespace_pair(selector: str | None) -> str | None: + """Drop any ``namespace=...`` pair from a labelSelector. + + The namespaced endpoint reads the namespace from the URL path + (already authenticated and authorised by the K8s API server) and + refuses to let a caller-supplied ``namespace=`` pair override it + — otherwise a workload in namespace A could read metrics scoped + to namespace B. Other pairs are preserved verbatim. + """ + if not selector: + return selector + kept = [p for p in selector.split(",") if p.partition("=")[0].strip() != "namespace"] + return ",".join(kept) if kept else None + + +def _apply_label_selector(promql: str, selector: str | None) -> str: + """Substitute ``$