Operator dept governance

README.md

@@ -53,9 +53,19 @@ path. See [`docs/reference/cli.md`](docs/reference/cli.md) for `promforecast val
 which CI users can run against a config file before merging.
 
 See [`docs/`](docs/) for configuration, model selection, and operational
-guidance. For a real-world deployment — multi-group config, HA values, and
-ready-to-apply ArgoCD/Flux manifests with a reference architecture diagram
-— see [`examples/production/`](examples/production/).
+guidance. Before flipping production traffic, walk
+[`docs/production-checklist.md`](docs/production-checklist.md) — sizing
+tables, reference architectures, and the boxes worth ticking off
+before going live. For a real-world deployment — multi-group config,
+HA values, and ready-to-apply ArgoCD/Flux manifests with a reference
+architecture diagram — see [`examples/production/`](examples/production/).
+
+New to forecasting Prometheus metrics? The
+[`docs/cookbook/`](docs/cookbook/) collection covers the common
+metric families (disk-fill, request rate, Kafka lag, JVM GC, AWS
+spend, ...) with copy-pasteable configs sized for a small fleet and
+companion alert snippets — the fastest path from "I want to forecast
+X" to "X is on a dashboard with an alert".
 
 ## Status
 

charts/promforecast-stack/templates/prometheusrule.yaml

@@ -25,6 +25,22 @@ spec:
               {{`{{ $labels.group }}`}} in over 2 hours. Check pod logs and
               datasource availability.
 
+        - alert: ForecastSeriesStale
+          expr: forecast_data_staleness_seconds > 14400
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Forecast {{`{{ $labels.id }}`}} model {{`{{ $labels.model }}`}} on {{`{{ $labels.instance }}`}} hasn't been refit for >4h"
+            description: |
+              The (series, model) tuple for {{`{{ $labels.id }}`}} on group
+              {{`{{ $labels.group }}`}} hasn't produced a successful forecast
+              for over four hours. The group-level ForecastStale alert stays
+              quiet because other series in the group are still being fit
+              successfully; this rule catches the silent per-series failure
+              (model crash, persistent fit timeout, data gap that drops the
+              series under min_points).
+
         - alert: ForecastModelDegraded
           expr: forecast_accuracy_mape > 30
           for: 24h
@@ -49,7 +65,7 @@ spec:
               the forecast is doing no better than a seasonal naive baseline.
 
         - alert: ForecastFallbackPersistent
-          expr: count by (group) (forecast_quality_score{model_fallback="true"}) > 0
+          expr: count by (group) (forecast_quality_score{model_fallback="true", horizon=""}) > 0
           for: 6h
           labels:
             severity: info
@@ -61,7 +77,7 @@ spec:
 
         - alert: ForecastQualityLow
           expr: |
-            avg by (group) (forecast_quality_score{cold_start!="true"}) < 0.4
+            avg by (group) (forecast_quality_score{cold_start!="true", horizon=""}) < 0.4
           for: 6h
           labels:
             severity: info
@@ -73,7 +89,7 @@ spec:
               downstream forecast-based alerts with caution.
 
         - alert: ForecastQualityZero
-          expr: forecast_quality_score{cold_start!="true"} == 0
+          expr: forecast_quality_score{cold_start!="true", horizon=""} == 0
           for: 1h
           labels:
             severity: info
@@ -103,7 +119,7 @@ spec:
         - alert: ForecastDeviation
           expr: |
             forecast_deviation_outside_band == 1
-            and on (id, group, model) forecast_quality_score > 0.6
+            and on (id, group, model) forecast_quality_score{horizon=""} > 0.6
           for: 15m
           labels:
             severity: warning
@@ -117,7 +133,7 @@ spec:
         - alert: ForecastDeviationSevere
           expr: |
             abs(forecast_deviation_ratio) > 2
-            and on (id, group, model) forecast_quality_score > 0.6
+            and on (id, group, model) forecast_quality_score{horizon=""} > 0.6
           for: 5m
           labels:
             severity: critical

charts/promforecast-tuner/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v2
+name: promforecast-tuner
+description: Weekly CronJob that surfaces hyperparameter recommendations for promforecast configs.
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+kubeVersion: ">=1.27.0-0"
+home: https://github.com/esops-dev/promforecast
+sources:
+  - https://github.com/esops-dev/promforecast
+keywords:
+  - prometheus
+  - forecasting
+  - hyperparameter-tuning
+  - cronjob
+maintainers:
+  - name: esops-dev
+    url: https://github.com/esops-dev
+icon: https://raw.githubusercontent.com/esops-dev/promforecast/main/docs/assets/logo.png
+annotations:
+  artifacthub.io/license: Apache-2.0
+  artifacthub.io/category: monitoring

charts/promforecast-tuner/README.md

@@ -0,0 +1,31 @@
+# promforecast-tuner
+
+Weekly Kubernetes `CronJob` that surfaces hyperparameter recommendations for the configured promforecast groups. Pushes one `forecast_recommended_param` sample per candidate that beats the configured baseline by `--min-improvement-pct` (default 5%).
+
+The tuner reuses the live forecaster's container image (`promforecast tune ...` is a one-shot subcommand) and reads the same YAML config the live runner uses, so the baseline `(model, season_length)` values match exactly.
+
+## Install
+
+```bash
+helm install promforecast-tuner \
+  oci://ghcr.io/esops-dev/promforecast/charts/promforecast-tuner \
+  --namespace promforecast \
+  --set configMap.name=promforecast-config
+```
+
+## Values
+
+See `values.yaml` for the full set of knobs and their defaults. The common ones:
+
+| Key | Default | Description |
+|---|---|---|
+| `schedule` | `0 2 * * 0` | Cron expression. Weekly Sunday 02:00 UTC by default. |
+| `configMap.name` | `promforecast` | Existing ConfigMap carrying the forecaster YAML. Defaults to the ConfigMap emitted by `helm install promforecast charts/promforecast`; override for other install shapes (umbrella: `<release>-promforecast`). |
+| `sinkUrlOverride` | `""` | Override `sink.remote_write.url`. Inherits from the config when empty. |
+| `minImprovementPct` | `5.0` | Minimum MAPE improvement to surface a candidate. |
+| `lookback` | `null` | Prometheus duration override (e.g. `14d`). |
+| `groups` | `[]` | Subset of groups to tune. Empty = all. |
+
+## How to read the recommendations
+
+See [`docs/operations/auto-tuning.md`](../../docs/operations/auto-tuning.md) for the "should I apply this recommendation?" decision guide.

charts/promforecast-tuner/templates/_helpers.tpl

@@ -0,0 +1,51 @@
+{{/*
+Common helpers for the promforecast-tuner chart.
+*/}}
+
+{{- define "promforecast-tuner.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "promforecast-tuner.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "promforecast-tuner.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "promforecast-tuner.labels" -}}
+helm.sh/chart: {{ include "promforecast-tuner.chart" . }}
+{{ include "promforecast-tuner.selectorLabels" . }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: promforecast
+app.kubernetes.io/component: tuner
+{{- end -}}
+
+{{- define "promforecast-tuner.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "promforecast-tuner.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "promforecast-tuner.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+{{- default (include "promforecast-tuner.fullname" .) .Values.serviceAccount.name -}}
+{{- else -}}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "promforecast-tuner.image" -}}
+{{- $tag := default .Chart.AppVersion .Values.image.tag -}}
+{{- printf "%s:%s" .Values.image.repository $tag -}}
+{{- end -}}

charts/promforecast-tuner/templates/cronjob.yaml

@@ -0,0 +1,85 @@
+{{- if .Values.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "promforecast-tuner.fullname" . }}
+  labels:
+    {{- include "promforecast-tuner.labels" . | nindent 4 }}
+spec:
+  schedule: {{ .Values.schedule | quote }}
+  concurrencyPolicy: {{ .Values.concurrencyPolicy }}
+  successfulJobsHistoryLimit: {{ .Values.successfulJobsHistoryLimit }}
+  failedJobsHistoryLimit: {{ .Values.failedJobsHistoryLimit }}
+  {{- if .Values.startingDeadlineSeconds }}
+  startingDeadlineSeconds: {{ .Values.startingDeadlineSeconds }}
+  {{- end }}
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: {{ .Values.activeDeadlineSeconds }}
+      backoffLimit: {{ .Values.backoffLimit }}
+      template:
+        metadata:
+          labels:
+            {{- include "promforecast-tuner.selectorLabels" . | nindent 12 }}
+        spec:
+          serviceAccountName: {{ include "promforecast-tuner.serviceAccountName" . }}
+          # The tuner needs no K8s API access — it reads from VM via PromQL
+          # and pushes to the configured sink. Disabling the API token mount
+          # keeps the principle of least privilege in shops where the default
+          # ServiceAccount has cluster-wide read by accident.
+          automountServiceAccountToken: false
+          restartPolicy: {{ .Values.restartPolicy }}
+          {{- with .Values.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.tolerations }}
+          tolerations:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.affinity }}
+          affinity:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          containers:
+            - name: tuner
+              image: {{ include "promforecast-tuner.image" . | quote }}
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              args:
+                - "tune"
+                - "--config=/etc/promforecast/{{ .Values.configMap.key }}"
+                - "--min-improvement-pct={{ .Values.minImprovementPct }}"
+                {{- if .Values.sinkUrlOverride }}
+                - "--sink-url={{ .Values.sinkUrlOverride }}"
+                {{- end }}
+                {{- if .Values.lookback }}
+                - "--lookback={{ .Values.lookback }}"
+                {{- end }}
+                {{- with .Values.groups }}
+                - "--groups"
+                {{- range . }}
+                - {{ . | quote }}
+                {{- end }}
+                {{- end }}
+              {{- with .Values.extraEnv }}
+              env:
+                {{- toYaml . | nindent 16 }}
+              {{- end }}
+              volumeMounts:
+                - name: config
+                  mountPath: /etc/promforecast
+                  readOnly: true
+              resources:
+                {{- toYaml .Values.resources | nindent 16 }}
+          volumes:
+            - name: config
+              configMap:
+                name: {{ .Values.configMap.name | quote }}
+                items:
+                  - key: {{ .Values.configMap.key | quote }}
+                    path: {{ .Values.configMap.key | quote }}
+{{- end }}

charts/promforecast-tuner/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "promforecast-tuner.serviceAccountName" . }}
+  labels:
+    {{- include "promforecast-tuner.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/promforecast-tuner/values.schema.json

@@ -0,0 +1,84 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/esops-dev/promforecast/charts/promforecast-tuner/v1",
+  "title": "promforecast-tuner Helm values",
+  "description": "Schema for the promforecast-tuner sub-chart values. Surfaces the knobs that drive the weekly hyperparameter-recommendation CronJob; the tuner reuses the live forecaster's container image so image / resource shapes mirror the main chart.",
+  "type": "object",
+  "additionalProperties": true,
+  "properties": {
+    "image": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "repository": { "type": "string", "minLength": 1 },
+        "tag": { "type": ["string", "null"] },
+        "pullPolicy": { "type": "string", "enum": ["Always", "IfNotPresent", "Never"] }
+      },
+      "required": ["repository"]
+    },
+    "enabled": { "type": "boolean" },
+    "schedule": {
+      "type": "string",
+      "description": "Cron expression for the CronJob. Defaults to weekly (Sunday 02:00 UTC)."
+    },
+    "configMap": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "key": { "type": "string", "minLength": 1 }
+      },
+      "required": ["name", "key"]
+    },
+    "sinkUrlOverride": { "type": "string" },
+    "minImprovementPct": { "type": "number", "minimum": 0 },
+    "lookback": {
+      "type": ["string", "null"],
+      "description": "Prometheus duration override for the lookback (e.g. ``14d``)."
+    },
+    "groups": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "resources": { "type": "object" },
+    "concurrencyPolicy": {
+      "type": "string",
+      "enum": ["Allow", "Forbid", "Replace"]
+    },
+    "startingDeadlineSeconds": { "type": "integer", "minimum": 0 },
+    "successfulJobsHistoryLimit": { "type": "integer", "minimum": 0 },
+    "failedJobsHistoryLimit": { "type": "integer", "minimum": 0 },
+    "activeDeadlineSeconds": { "type": "integer", "minimum": 1 },
+    "backoffLimit": { "type": "integer", "minimum": 0 },
+    "restartPolicy": {
+      "type": "string",
+      "enum": ["OnFailure", "Never"]
+    },
+    "serviceAccount": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "create": { "type": "boolean" },
+        "name": { "type": "string" },
+        "annotations": { "type": "object" }
+      },
+      "required": ["create"]
+    },
+    "securityContext": { "type": "object" },
+    "nameOverride": { "type": "string" },
+    "fullnameOverride": { "type": "string" },
+    "nodeSelector": { "type": "object" },
+    "tolerations": { "type": "array" },
+    "affinity": { "type": "object" },
+    "extraEnv": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string", "minLength": 1 }
+        },
+        "required": ["name"]
+      }
+    }
+  }
+}