diff --git a/Makefile b/Makefile index 8a6d409..721990e 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,16 @@ exit('!! forecast_query_cache_events_total{kind=\"miss\"} is zero -- the cache i r=json.loads(u.urlopen('http://localhost:9090/api/v1/query?query=forecast_accuracy_mape').read())['data']['result']; \ print(f' forecast_accuracy_mape: {len(r)} series'); \ exit('!! Prometheus has 0 series - check the prometheus scrape config and forecaster reachability') if not r else None" + @echo ">> no label-divergent dual-source forecast labelsets in VM (regression guard for the scrape-drop)" + @cd $(FORECASTER) && .venv/bin/promforecast diagnose duplicates --datasource http://localhost:8428 --since 10m \ + > /tmp/promforecast-diagnose.txt 2>&1 \ + && echo ">> diagnose duplicates: clean" \ + || (echo "!! diagnose duplicates flagged a label-divergent duplicate state; see /tmp/promforecast-diagnose.txt"; tail -30 /tmp/promforecast-diagnose.txt; exit 1) + @echo ">> no same-labelset dual-source forecast ingest in VM (regression guard for sample-rate contamination)" + @cd $(FORECASTER) && .venv/bin/promforecast diagnose sample-rate --datasource http://localhost:8428 --window 10m --max-samples 5 \ + > /tmp/promforecast-sample-rate.txt 2>&1 \ + && echo ">> diagnose sample-rate: clean" \ + || (echo "!! diagnose sample-rate flagged at least one series above the per-window cap; see /tmp/promforecast-sample-rate.txt"; tail -30 /tmp/promforecast-sample-rate.txt; exit 1) @if docker compose -f docker-compose.dev.yml ps --status running --services 2>/dev/null | grep -qx grafana; then \ echo ">> dashboards reachable in Grafana at http://localhost:3000 (admin / admin)"; \ else \ diff --git a/README.md b/README.md index 397521f..cc5068e 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,18 @@ helm install promforecast promforecast/promforecast-stack This installs VictoriaMetrics, the forecaster, and example dashboards. For an existing TSDB, install only `promforecast/promforecast`. +The umbrella chart ships **sink + VictoriaMetrics** as the canonical +forecast write path: the forecaster pushes the full forecast curve into +VM with native future timestamps, and the bundled ServiceMonitor drops +the redundant `*_forecast` / `*_forecast_lower` / `*_forecast_upper` +snapshot families from the scrape so dashboards plot a smooth forward +curve rather than averaging the sink push against a 5-minute snapshot +staircase. See [`docs/architecture/emission-paths.md`](docs/architecture/emission-paths.md) +for what each path emits and the dual-source trap that motivates the +defaults. Existing snapshot-only installs upgrading to the new defaults +should follow [`docs/how-to/migrate-to-sink-first.md`](docs/how-to/migrate-to-sink-first.md) +and use `promforecast diagnose duplicates` to confirm the cutover. + Editing the YAML config does **not** require a pod restart: the forecaster watches the mounted ConfigMap, validates the new config in-place, and rolls back on validation failure (`kubectl apply` to the diff --git a/charts/promforecast-stack/README.md b/charts/promforecast-stack/README.md index 9a1eabb..00d764a 100644 --- a/charts/promforecast-stack/README.md +++ b/charts/promforecast-stack/README.md @@ -22,11 +22,49 @@ accepted (VM enforces a 2-day minimum). **Tune this higher than your longest configured forecast horizon or your longest forecasts will be silently dropped.** +## Sink-first defaults + +The chart pre-configures the **sink + VM** emission path: the forecaster +pushes the full forecast curve into the bundled VictoriaMetrics with +native future timestamps, and the forecaster's ServiceMonitor drops the +`*_forecast` / `*_forecast_lower` / `*_forecast_upper` snapshot families +from the scrape so the long-term TSDB stores one canonical labelset per +forecast metric. Bundled dashboards default to VictoriaMetrics as their +datasource. + +A `helm test` hook +(`-promforecast-stack-test-labelset`) queries VM after install +and asserts that no `*_forecast` series carry the forecaster's scrape +`job` label — the canonical signature of the dual-source state. Run it +on demand: + +```bash +helm test +``` + +See [`docs/architecture/emission-paths.md`](../../docs/architecture/emission-paths.md) +for what the two paths emit, why running both unguarded is a silent +trap, and the `metric_relabel_configs` snippet to paste into an +externally-managed Prometheus. + +**ServiceMonitor CRD required.** `promforecast.serviceMonitor.enabled` +defaults to `true` so the chart-managed snapshot drop is applied. On +clusters without prometheus-operator (no `monitoring.coreos.com/v1` +ServiceMonitor CRD), `helm install` will fail; flip +`promforecast.serviceMonitor.enabled=false` and apply the scrape-drop +from the docs above to your plain-Prometheus config instead. + ## Subchart toggles - `victoria-metrics-single.enabled`: bundle a VM single-node install. -- `promforecast.enabled`: bundle the forecaster. -- `dashboards.enabled`: install Grafana dashboards as ConfigMaps. +- `promforecast.enabled`: bundle the forecaster (sink to VM by default). +- `dashboards.enabled`: install Grafana dashboards as ConfigMaps tagged + for the Grafana sidecar. +- `datasources.enabled`: ship a Grafana datasource provisioning + ConfigMap (also sidecar-tagged) that registers VM as the default + datasource the bundled dashboards target. - `prometheusRules.enabled`: install example `PrometheusRule` manifests. +- `tests.labelsetUniqueness.enabled`: include the `helm test` probe + described above. See [`values.yaml`](values.yaml) for the full schema. diff --git a/charts/promforecast-stack/templates/datasources.yaml b/charts/promforecast-stack/templates/datasources.yaml new file mode 100644 index 0000000..4921d5b --- /dev/null +++ b/charts/promforecast-stack/templates/datasources.yaml @@ -0,0 +1,47 @@ +{{- if .Values.datasources.enabled }} +{{- /* + Grafana datasource provisioning shipped as ConfigMaps with the sidecar + label so a Grafana running the kiwigrid/k8s-sidecar pattern picks them + up automatically. VictoriaMetrics is registered as the default so the + bundled dashboards land on the sink-fed copy of the forecast metrics + (smooth curves) rather than the scraped snapshot (5-min staircase). + See ``docs/architecture/emission-paths.md`` for why the default matters. +*/}} +{{- $vm := .Values.datasources.victoriaMetrics }} +{{- $prom := .Values.datasources.prometheus }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-datasources" (include "promforecast-stack.fullname" .) | trunc 63 | trimSuffix "-" }} + labels: +{{- include "promforecast-stack.labels" . | nindent 4 }} +{{- range $k, $v := .Values.datasources.labels }} + {{ $k }}: {{ $v | quote }} +{{- end }} +data: + promforecast-datasources.yaml: |- + apiVersion: 1 + datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + uid: {{ $vm.uid | quote }} + url: {{ $vm.url | quote }} + isDefault: {{ $vm.isDefault }} + editable: true + jsonData: + httpMethod: POST + timeInterval: 15s +{{- if $prom.url }} + - name: Prometheus + type: prometheus + access: proxy + uid: {{ $prom.uid | quote }} + url: {{ $prom.url | quote }} + isDefault: {{ $prom.isDefault }} + editable: true + jsonData: + httpMethod: POST + timeInterval: 15s +{{- end }} +{{- end }} diff --git a/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml b/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml new file mode 100644 index 0000000..b975b4f --- /dev/null +++ b/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml @@ -0,0 +1,136 @@ +{{- /* + Post-install test: assert that every ``*_forecast`` / ``*_forecast_lower`` + / ``*_forecast_upper`` series in VictoriaMetrics exists under exactly + one labelset family — specifically, no copy carries the forecaster's + own scrape labels (``job=-promforecast`` / a forecaster-pod + ``instance``) alongside the sink-emitted copy that lacks them. The + dual-source state silently rendered the forecast curve as an average + of the smooth sink copy with the 5-minute snapshot staircase before + the sink-first realignment. Wired as a ``helm test`` hook (not part + of the normal install) so an operator can run it on demand. CI does + not execute it — no kind cluster — but ``helm template`` still + renders it and ``kubeconform`` validates the Pod schema. Set + ``tests.labelsetUniqueness.enabled=false`` to suppress the asset for + managed clusters that forbid hook resources. + + The probe uses VictoriaMetrics' ``/api/v1/series`` endpoint with a + URL-encoded ``match[]`` selector that filters in-VM on the + forecaster-scrape labels. The expected response under the sink-first + defaults is an empty result set; any series returned signals dual + source. We deliberately do NOT scan for the *presence* of an + ``instance`` label on forecast series — the sink legitimately + propagates the source ``instance`` (e.g. ``node-exporter:9100``), + so a presence-only check would false-fire on a clean install. +*/}} +{{- if .Values.tests.labelsetUniqueness.enabled }} +{{- /* + The standalone forecaster ServiceMonitor sets + ``jobLabel: app.kubernetes.io/name``, so the Prometheus scrape job + label is the stable value ``promforecast`` regardless of Helm + release name. The probe's selector targets that exact job string. + Override ``tests.scrapeJobPattern`` only when an external scrape + (i.e. not the umbrella-managed ServiceMonitor) re-labels the job. +*/}} +{{- $jobPattern := default "promforecast" .Values.tests.scrapeJobPattern }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ printf "%s-test-labelset" (include "promforecast-stack.fullname" .) | trunc 63 | trimSuffix "-" }} + labels: +{{- include "promforecast-stack.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: assert + image: {{ .Values.tests.image | quote }} + env: + - name: VM_URL + # Pinned to the VM subchart's Service name, which the umbrella + # locks with ``victoria-metrics-single.fullnameOverride`` so the + # URL is release-name-agnostic. + value: {{ .Values.tests.victoriaMetricsUrl | default "http://promforecast-stack-victoria-metrics-single-server:8428" | quote }} + - name: SCRAPE_JOB_PATTERN + value: {{ $jobPattern | quote }} + - name: FIRST_REFRESH_TIMEOUT_SECONDS + value: {{ .Values.tests.firstRefreshTimeoutSeconds | default 300 | quote }} + command: + - sh + - -c + - | + set -eu + echo "Querying ${VM_URL} for the labelset-uniqueness assertion..." + # URL-encode the selectors by hand because busybox lacks ``jq`` + # and a plain interpolation of the literal ``{__name__=~"..."}`` + # characters into a URL would not survive HTTP parsing. The + # first selector asks VM for forecast-family series carrying + # the forecaster's scrape ``job`` label — under the sink-first + # defaults the snapshot drop strips them, so the count must be + # zero. The second selector asks VM for forecast-family series + # without that ``job`` label — those are the sink-pushed copies + # and the count must be non-zero (a broken sink + applied drop + # would silently leave VM with no forecast data at all, which + # is also a regression we want to surface). + dup_selector='%7B__name__%3D~%22.%2B_forecast%28_lower%7C_upper%29%3F%22%2Cjob%3D~%22'"$SCRAPE_JOB_PATTERN"'%22%7D' + sink_selector='%7B__name__%3D~%22.%2B_forecast%28_lower%7C_upper%29%3F%22%2Cjob%3D%22%22%7D' + # The first scheduled fit completes some seconds-to-minutes after + # ``helm install`` finishes; the sink push lands in VM only after + # that. Poll for up to ``FIRST_REFRESH_TIMEOUT_SECONDS`` (5min + # default) before declaring the sink-side empty. The dup check + # is meaningful at any point — even before the first fit, the + # scraped snapshot family should never reach VM under the + # sink-first defaults — but we re-evaluate it on each iteration + # so a single ``helm test`` invocation reports both failure modes. + deadline=$(( $(date +%s) + FIRST_REFRESH_TIMEOUT_SECONDS )) + while :; do + dup_out=$(wget -q -O - "${VM_URL}/api/v1/series?match%5B%5D=${dup_selector}") || { + echo "ERROR: could not query VictoriaMetrics at $VM_URL" >&2 + exit 2 + } + sink_out=$(wget -q -O - "${VM_URL}/api/v1/series?match%5B%5D=${sink_selector}") || { + echo "ERROR: could not query VictoriaMetrics at $VM_URL" >&2 + exit 2 + } + # ``grep -c`` over the ``__name__`` key is good enough — we only + # need zero-vs-non-zero. A clean install returns + # ``{"status":"success","data":[]}`` for the dup query and a + # non-empty data array for the sink query. + dup_n=$(echo "$dup_out" | grep -oE '"__name__":"[^"]+"' | wc -l | tr -d ' ') + sink_n=$(echo "$sink_out" | grep -oE '"__name__":"[^"]+"' | wc -l | tr -d ' ') + # Dup detection is fail-fast: the snapshot drop is a static + # ServiceMonitor relabel that does not depend on the first fit, + # so any series carrying the scrape job at any point is a + # config bug we should report immediately rather than waiting + # for the timeout to elapse. + if [ "$dup_n" -gt 0 ]; then + echo "FAIL: $dup_n forecast series carry the forecaster scrape job ($SCRAPE_JOB_PATTERN)." >&2 + echo " The snapshot scrape and the sink push are both being stored." >&2 + echo "Remediation: promforecast.serviceMonitor.enabled=true (the umbrella default)" >&2 + echo " so the standalone chart drops the snapshot families. For an" >&2 + echo " externally-managed Prometheus, add the metric_relabel_configs" >&2 + echo " snippet from docs/architecture/emission-paths.md." >&2 + echo "$dup_out" >&2 + exit 1 + fi + if [ "$sink_n" -gt 0 ]; then + echo "OK: $sink_n sink-pushed forecast series in VM, zero copies carrying the forecaster scrape job." + exit 0 + fi + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + echo "FAIL: no *_forecast series found without the forecaster scrape job." >&2 + echo " Waited ${FIRST_REFRESH_TIMEOUT_SECONDS}s for the first scheduled" >&2 + echo " refresh to push curves to VM. Either the forecaster's" >&2 + echo " ``refresh_interval`` is longer than this timeout, the sink" >&2 + echo " configuration is broken, or the forecaster has not become" >&2 + echo " ready yet. Check ``forecast_failures_total`` and" >&2 + echo " ``forecast_remote_write_failures_total`` on the forecaster" >&2 + echo " ``/metrics`` endpoint, or raise ``tests.firstRefreshTimeoutSeconds``." >&2 + exit 1 + fi + echo "waiting for first sink push (${sink_n} sink series so far, $((deadline - now))s left)..." + sleep 5 + done +{{- end }} diff --git a/charts/promforecast-stack/values.schema.json b/charts/promforecast-stack/values.schema.json index ef6ddd5..6cfb42d 100644 --- a/charts/promforecast-stack/values.schema.json +++ b/charts/promforecast-stack/values.schema.json @@ -23,6 +23,49 @@ "enabled": { "type": "boolean" }, "labels": { "type": "object" } } + }, + "datasources": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "labels": { "type": "object" }, + "victoriaMetrics": { + "type": "object", + "additionalProperties": false, + "properties": { + "uid": { "type": "string", "minLength": 1 }, + "url": { "type": "string" }, + "isDefault": { "type": "boolean" } + } + }, + "prometheus": { + "type": "object", + "additionalProperties": false, + "properties": { + "uid": { "type": "string", "minLength": 1 }, + "url": { "type": "string" }, + "isDefault": { "type": "boolean" } + } + } + } + }, + "tests": { + "type": "object", + "additionalProperties": false, + "properties": { + "image": { "type": "string", "minLength": 1 }, + "victoriaMetricsUrl": { "type": "string" }, + "scrapeJobPattern": { "type": "string" }, + "firstRefreshTimeoutSeconds": { "type": "integer", "minimum": 0 }, + "labelsetUniqueness": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" } + } + } + } } } } diff --git a/charts/promforecast-stack/values.yaml b/charts/promforecast-stack/values.yaml index 542e2e6..1a3bc59 100644 --- a/charts/promforecast-stack/values.yaml +++ b/charts/promforecast-stack/values.yaml @@ -3,6 +3,14 @@ victoria-metrics-single: enabled: true + # Pin the VM Service name regardless of Helm release name so the + # forecaster's hardcoded ``datasource.url`` / ``sink.remote_write.url`` + # below resolve in any installation. Without this, ``helm install foo + # ./promforecast-stack`` would create a Service called + # ``foo-victoria-metrics-single-server`` while the forecaster still + # tries to reach ``promforecast-stack-victoria-metrics-single-server``. + # Override only when integrating into an existing VM deployment. + fullnameOverride: promforecast-stack-victoria-metrics-single server: # IMPORTANT: forecasts are written with future timestamps when the # forecaster's `sink.remote_write` is enabled. Without this flag @@ -30,14 +38,37 @@ promforecast: auth: type: none # Push the full forecast curve back into VictoriaMetrics so historical - # forecast lines are queryable in Grafana. Disabled by default — the - # /metrics snapshot endpoint is enough for many users; flip this on - # when you want forecasts as first-class historical data. + # forecast lines are queryable in Grafana as a smooth projection rather + # than a 5-minute snapshot staircase. Default ON in the umbrella — the + # sink+VM pair is the canonical write path for this stack. The + # forecaster-chart ServiceMonitor below pairs with this by dropping the + # ``*_forecast`` / ``*_forecast_lower`` / ``*_forecast_upper`` snapshot + # families from the scrape, so the long-term TSDB stores one canonical + # labelset per forecast metric and dashboards do not silently average + # the scraped staircase across the sink-pushed curve. + # VictoriaMetrics accepts the future-timestamped samples because the + # subchart above sets ``futureRetention: "2d"``. Increase that knob if + # you increase ``defaults.horizon`` above 2d. sink: remote_write: - enabled: false + enabled: true url: http://promforecast-stack-victoria-metrics-single-server:8428/api/v1/import/prometheus timeout: 30s + # Turn the forecaster's ServiceMonitor on by default and let it derive + # the snapshot-drop from the sink toggle above (``dropForecastSnapshots: + # null`` resolves to true when the sink is on). + # + # **Requires the prometheus-operator ServiceMonitor CRD** to be + # installed in the cluster (kube-prometheus-stack, the stand-alone + # prometheus-operator chart, or any equivalent). ``helm install`` will + # fail with ``no matches for kind ServiceMonitor`` on a cluster without + # the CRD. For installs that scrape with a plain Prometheus (no + # operator), flip this off and paste the ``metric_relabel_configs`` + # snippet from ``docs/architecture/emission-paths.md`` into your scrape + # config — the snapshot families must be dropped somewhere or the + # dual-source trap reappears. + serviceMonitor: + enabled: true # Bundled dashboards / alerts toggles. Off by default; enable explicitly so # users on shops with their own asset pipelines aren't surprised. @@ -46,6 +77,72 @@ dashboards: labels: grafana_dashboard: "1" +# Optional Grafana datasource provisioning shipped as ConfigMaps with the +# sidecar label (``grafana_datasource: "1"``) so a Grafana running the +# kiwigrid/k8s-sidecar pattern (kube-prometheus-stack, official Grafana +# chart) picks them up. Off by default — flip on alongside +# ``dashboards.enabled`` if you want this chart to wire up the datasources +# for you. VictoriaMetrics is registered as the default datasource so the +# bundled dashboards land on the sink-fed copy of the forecast metrics +# rather than the snapshot. +datasources: + enabled: false + labels: + grafana_datasource: "1" + # Pre-set so the dashboards' ``${datasource}`` template variables resolve + # to it. Override only when integrating into an existing Grafana that + # already provisions VM under a different UID. + # + # ``isDefault`` flips VM to Grafana's default datasource. Set to + # ``false`` when Grafana already has a default (e.g. kube-prometheus-stack + # registering Prometheus as default) — otherwise the sidecar reload + # collides and one of the entries is silently dropped depending on + # Grafana version. The bundled dashboards always carry an explicit + # ``uid: victoriametrics`` reference so they keep working even when VM + # is not the global default. + victoriaMetrics: + uid: victoriametrics + url: http://promforecast-stack-victoria-metrics-single-server:8428 + isDefault: true + # Prometheus is registered alongside VM but not as the default; the + # dashboards still let you flip to it from the dropdown for scrape-pipeline + # debugging. Leave ``url`` empty to skip Prometheus provisioning entirely. + prometheus: + uid: prometheus + url: "" + isDefault: false + prometheusRules: enabled: false labels: {} + +# ``helm test`` hooks shipped with this chart. The labelset-uniqueness +# probe queries the bundled VM for ``*_forecast`` series that carry the +# forecaster's scrape ``job`` label (i.e., the snapshot copy) and +# asserts the count is zero. Under the sink-first default install only +# the sink copy lives in VM, so the probe passes. Suppress with +# ``tests.labelsetUniqueness.enabled=false`` in managed clusters that +# forbid hook resources. +tests: + image: busybox:1.36 + # Override only when VM lives outside the umbrella (e.g. an existing + # cluster-shared VM). Empty resolves to the umbrella's bundled + # ``-victoria-metrics-single-server``. + victoriaMetricsUrl: "" + # Regex matched against forecast-series ``job`` labels to identify the + # snapshot copy. Defaults to ``promforecast`` — the value the + # forecaster's ServiceMonitor pins via + # ``jobLabel: app.kubernetes.io/name`` so the probe is release-name + # agnostic. Override only when an external (non-operator) scrape job + # re-labels the job to something else. + scrapeJobPattern: "" + # How long the labelset-uniqueness probe waits for the first + # scheduled refresh to push curves into VM before declaring the + # sink-side empty. Defaults to 5 minutes — enough for a default + # ``refresh_interval: 1h`` install where the first run fires + # immediately at boot. Raise when the forecaster takes longer to + # become ready (large model registry, slow datasource), or lower for + # tight CI loops that already pre-fit before running the probe. + firstRefreshTimeoutSeconds: 300 + labelsetUniqueness: + enabled: true diff --git a/charts/promforecast/README.md b/charts/promforecast/README.md index 471d8f5..de8bbb0 100644 --- a/charts/promforecast/README.md +++ b/charts/promforecast/README.md @@ -28,6 +28,56 @@ coordination would race on the snapshot cache. Set `highAvailability.enabled: true` and configure Redis to opt into multi-replica serving with leader election. +## Sink + ServiceMonitor snapshot drop + +Set `config.sink.remote_write.enabled: true` to push the full forecast +curve into a long-term TSDB. When `serviceMonitor.enabled: true` the +generated ServiceMonitor automatically drops the `*_forecast` / +`*_forecast_lower` / `*_forecast_upper` snapshot families from the +scrape so the long-term TSDB doesn't end up with both copies under +different labelsets. The derivation key is +`serviceMonitor.dropForecastSnapshots`: + +| sink | dropForecastSnapshots | snapshot drop in ServiceMonitor | +| --- | --- | --- | +| on | `null` (default) | applied | +| off | `null` (default) | not applied | +| on | `false` (explicit override) | not applied (both copies stored) | +| off | `true` (explicit override) | applied (no snapshot, no sink either) | + +When using `existingConfigMap` the chart cannot inspect the sink setting +inside the external ConfigMap. Rather than silently defaulting to the +off side (which would land the GitOps user in the dual-source state), +the template **fails the render** until `serviceMonitor.dropForecastSnapshots` +is pinned to a literal `true` or `false`. + +### Scrape labelset pins + +The ServiceMonitor pins two scrape-time labels so the bundled example +alerts in `examples/alerts/promforecast-rules.yaml` and the +labelset-uniqueness probe in `charts/promforecast-stack` work +release-name-agnostically: + +- `jobLabel: app.kubernetes.io/name` — the scrape `job` label is the + stable value `promforecast` regardless of Helm release name. + Without this prometheus-operator generates `//` + as the job, which makes generic PromQL examples impossible to write. +- `honorLabels: true` — the forecaster's `/metrics` exposes operational + and derived `*_forecast_*` families with the underlying signal's + source labels (e.g. `instance="node-exporter:9100"`). Honoring those + prevents Prometheus from clobbering `instance` with the forecaster + pod's own address. + +> **Upgrade note.** Both pins changed scraped labelsets compared to +> earlier releases. Alerts and dashboards that filtered on the +> auto-generated `job` (e.g. `job="-promforecast"`) or the +> forecaster pod's `instance` need to be updated to the new stable +> values before upgrading. + +See [`docs/architecture/emission-paths.md`](../../docs/architecture/emission-paths.md) +for what each emission path emits and the dual-source trap the drop +prevents. + ## Values See [`values.yaml`](values.yaml) for the full schema. diff --git a/charts/promforecast/templates/servicemonitor.yaml b/charts/promforecast/templates/servicemonitor.yaml index 24623fb..720d9d6 100644 --- a/charts/promforecast/templates/servicemonitor.yaml +++ b/charts/promforecast/templates/servicemonitor.yaml @@ -1,4 +1,30 @@ {{- if .Values.serviceMonitor.enabled }} +{{- /* + Derive the forecast-snapshot drop block from the sink toggle when the + operator has not pinned ``dropForecastSnapshots`` to a literal value. + ``null`` (the values.yaml default) means "follow sink.remote_write.enabled"; + a literal true/false overrides the derivation. We pull the sink toggle + out of the nested config map with ``dig`` so unset intermediate keys + don't crash the render. + + When the operator manages the forecaster config out-of-band + (``existingConfigMap``), the chart cannot see the sink toggle and the + derivation is ambiguous. Defaulting silently to off would land the + GitOps user in the dual-source state; we fail the template instead and + force an explicit ``dropForecastSnapshots`` decision. +*/}} +{{- $explicit := .Values.serviceMonitor.dropForecastSnapshots }} +{{- if and .Values.existingConfigMap (not (kindIs "bool" $explicit)) }} +{{- fail "serviceMonitor.dropForecastSnapshots must be set explicitly (true/false) when existingConfigMap is set — the chart cannot read the sink toggle from an externally-managed ConfigMap. See docs/architecture/emission-paths.md for the dual-source labelset trap." }} +{{- end }} +{{- $sinkOn := dig "sink" "remote_write" "enabled" false .Values.config }} +{{- $drop := ternary $explicit $sinkOn (kindIs "bool" $explicit) }} +{{- $extraRelabelings := default (list) .Values.serviceMonitor.metricRelabelings }} +{{- $relabelings := $extraRelabelings }} +{{- if $drop }} +{{- $dropRule := dict "sourceLabels" (list "__name__") "regex" ".+_forecast(_lower|_upper)?" "action" "drop" }} +{{- $relabelings = prepend $extraRelabelings $dropRule }} +{{- end }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -10,17 +36,34 @@ metadata: {{- include "promforecast.labels" . | nindent 4 }} {{- with .Values.serviceMonitor.labels }}{{- toYaml . | nindent 4 }}{{- end }} spec: + # Pin the scrape ``job`` label to the Service's ``app.kubernetes.io/name`` + # (``promforecast``) so it's a stable, release-name-agnostic value. + # Without this prometheus-operator auto-generates + # ``//`` as the job, which makes the + # umbrella's ``tests.labelsetUniqueness`` probe and the example PromQL in + # ``examples/alerts/promforecast-rules.yaml`` impossible to write + # generically. + jobLabel: app.kubernetes.io/name selector: matchLabels: {{- include "promforecast.selectorLabels" . | nindent 6 }} endpoints: - port: http path: /metrics + # ``honorLabels: true`` keeps the source ``instance`` (and any other + # source-side labels) that the forecaster exposes on its operational + # and derived ``*_forecast_*`` families — without this, Prometheus + # would clobber ``instance`` with the forecaster pod's own address, + # making the example alerts in ``examples/alerts/promforecast-rules.yaml`` + # (which interpolate ``{{`{{ $labels.instance }}`}}`` to identify the + # node-exporter target) report the wrong host. Matches the + # ``honor_labels: true`` snippet in ``docs/architecture/emission-paths.md``. + honorLabels: true interval: {{ .Values.serviceMonitor.interval }} scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} {{- with .Values.serviceMonitor.relabelings }} relabelings: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.serviceMonitor.metricRelabelings }} + {{- with $relabelings }} metricRelabelings: {{- toYaml . | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/promforecast/values.schema.json b/charts/promforecast/values.schema.json index 13cd9a9..336b3ac 100644 --- a/charts/promforecast/values.schema.json +++ b/charts/promforecast/values.schema.json @@ -71,7 +71,11 @@ "scrapeTimeout": { "type": "string" }, "labels": { "type": "object" }, "relabelings": { "type": "array" }, - "metricRelabelings": { "type": "array" } + "metricRelabelings": { "type": "array" }, + "dropForecastSnapshots": { + "description": "When true, the generated ServiceMonitor drops *_forecast / *_forecast_lower / *_forecast_upper metrics so the sink+TSDB copy is the only one stored. When null (the default) the value is derived from config.sink.remote_write.enabled — true if the sink is on, false otherwise. Set explicitly to false to keep snapshot gauges alongside the sink-pushed curves.", + "type": ["boolean", "null"] + } } }, "resources": { "type": "object" }, diff --git a/charts/promforecast/values.yaml b/charts/promforecast/values.yaml index 75673a1..f7a3f39 100644 --- a/charts/promforecast/values.yaml +++ b/charts/promforecast/values.yaml @@ -196,6 +196,30 @@ serviceMonitor: labels: {} relabelings: [] metricRelabelings: [] + # Drop the forecast snapshot gauges (``*_forecast``, ``*_forecast_lower``, + # ``*_forecast_upper``) from the scrape when the remote_write sink is on. + # The sink pushes the full forecast curve into the TSDB directly with + # native future timestamps; the scraped snapshot is a duplicate carrying a + # different labelset (``instance=":9091"``, + # ``job=""``) that dashboards either average across or have to + # pre-filter. Default ``null`` resolves to ``true`` when + # ``sink.remote_write.enabled`` is true, ``false`` otherwise — so the + # default install never ends up in the dual-source state. Set explicitly + # to ``false`` to override and keep both copies (rare; useful only when + # the snapshot end-of-horizon gauges are wanted alongside the curves). + # Operational ``forecast_*`` metrics (``forecast_quality_score``, + # ``forecast_failures_total``, etc.) are unaffected — only the + # ``*_forecast`` / ``*_forecast_lower`` / ``*_forecast_upper`` families + # are dropped. + # + # **Edge case for ``existingConfigMap``**: the chart cannot inspect the + # sink setting inside an out-of-band ConfigMap, so the ``null`` + # derivation falls through to ``false``. If you use ``existingConfigMap`` + # AND the external config has ``sink.remote_write.enabled: true``, pin + # ``dropForecastSnapshots: true`` here explicitly — otherwise the + # snapshot copy stays scraped alongside the sink push and you end up in + # the dual-source state. + dropForecastSnapshots: null resources: requests: diff --git a/dashboards/grafana/promforecast-accuracy.json b/dashboards/grafana/promforecast-accuracy.json index 7a6fc95..c3e95ae 100644 --- a/dashboards/grafana/promforecast-accuracy.json +++ b/dashboards/grafana/promforecast-accuracy.json @@ -4,9 +4,16 @@ "schemaVersion": 39, "version": 1, "editable": true, - "tags": ["promforecast", "forecasting", "accuracy"], + "tags": [ + "promforecast", + "forecasting", + "accuracy" + ], "timezone": "", - "time": { "from": "now-7d", "to": "now" }, + "time": { + "from": "now-7d", + "to": "now" + }, "refresh": "5m", "templating": { "list": [ @@ -14,12 +21,18 @@ "name": "datasource", "type": "datasource", "query": "prometheus", - "current": {} + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } }, { "name": "group", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_accuracy_mape, group)", "refresh": 2, "includeAll": true, @@ -28,7 +41,10 @@ { "name": "id", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_accuracy_mape{group=~\"$group\"}, id)", "refresh": 2, "includeAll": true, @@ -37,7 +53,10 @@ { "name": "model", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_accuracy_mape{group=~\"$group\",id=~\"$id\"}, model)", "refresh": 2, "includeAll": true, @@ -50,8 +69,16 @@ "type": "stat", "id": 1, "title": "Median MAPE (24h horizon)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, "targets": [ { "expr": "quantile(0.5, forecast_accuracy_mape{group=~\"$group\",id=~\"$id\",model=~\"$model\",horizon=\"24h\"})", @@ -64,9 +91,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 15 }, - { "color": "red", "value": 30 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 15 + }, + { + "color": "red", + "value": 30 + } ] } }, @@ -77,8 +113,16 @@ "type": "stat", "id": 2, "title": "p90 MAPE (24h horizon)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, "targets": [ { "expr": "quantile(0.9, forecast_accuracy_mape{group=~\"$group\",id=~\"$id\",model=~\"$model\",horizon=\"24h\"})", @@ -91,9 +135,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 25 }, - { "color": "red", "value": 50 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "red", + "value": 50 + } ] } }, @@ -104,8 +157,16 @@ "type": "stat", "id": 3, "title": "Series tracked", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, "targets": [ { "expr": "sum(forecast_series_count{group=~\"$group\"})", @@ -113,7 +174,9 @@ } ], "fieldConfig": { - "defaults": { "unit": "none" }, + "defaults": { + "unit": "none" + }, "overrides": [] } }, @@ -121,8 +184,16 @@ "type": "timeseries", "id": 4, "title": "MAPE by horizon", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 6 + }, "targets": [ { "expr": "avg by (horizon) (forecast_accuracy_mape{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -130,7 +201,9 @@ } ], "fieldConfig": { - "defaults": { "unit": "percent" }, + "defaults": { + "unit": "percent" + }, "overrides": [] } }, @@ -138,8 +211,16 @@ "type": "timeseries", "id": 5, "title": "MASE by model", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 6 + }, "targets": [ { "expr": "avg by (model) (forecast_accuracy_mase{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -147,7 +228,9 @@ } ], "fieldConfig": { - "defaults": { "unit": "none" }, + "defaults": { + "unit": "none" + }, "overrides": [] } }, @@ -155,8 +238,16 @@ "type": "table", "id": 6, "title": "Worst-performing series (24h MAPE)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 15 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 15 + }, "targets": [ { "expr": "topk(20, forecast_accuracy_mape{group=~\"$group\",id=~\"$id\",model=~\"$model\",horizon=\"24h\"})", @@ -165,11 +256,21 @@ } ], "fieldConfig": { - "defaults": { "unit": "percent" }, + "defaults": { + "unit": "percent" + }, "overrides": [] }, "transformations": [ - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true } } } + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + } + } + } ] } ] diff --git a/dashboards/grafana/promforecast-capacity.json b/dashboards/grafana/promforecast-capacity.json index 908163b..980a342 100644 --- a/dashboards/grafana/promforecast-capacity.json +++ b/dashboards/grafana/promforecast-capacity.json @@ -4,9 +4,16 @@ "schemaVersion": 39, "version": 1, "editable": true, - "tags": ["promforecast", "forecasting", "capacity"], + "tags": [ + "promforecast", + "forecasting", + "capacity" + ], "timezone": "", - "time": { "from": "now-24h", "to": "now" }, + "time": { + "from": "now-24h", + "to": "now" + }, "refresh": "1m", "templating": { "list": [ @@ -14,13 +21,19 @@ "name": "datasource", "type": "datasource", "query": "prometheus", - "current": {} + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } }, { "name": "group", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "query": "label_values({__name__=~\".*_forecast_time_to_threshold_seconds\"}, group)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values(forecast_quality_score, group)", "refresh": 2, "includeAll": true, "multi": true @@ -28,8 +41,11 @@ { "name": "id", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "query": "label_values({__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\"}, id)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values(forecast_quality_score{group=~\"$group\"}, id)", "refresh": 2, "includeAll": true, "multi": true @@ -37,32 +53,50 @@ { "name": "level", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "query": "label_values({__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\"}, level)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values(forecast_deviation_outside_band{group=~\"$group\"}, level)", "refresh": 2, "includeAll": false, "multi": false, - "current": { "text": "95", "value": "95" } + "current": { + "text": "95", + "value": "95" + } }, { "name": "threshold", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "query": "label_values({__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\"}, name)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values({__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\"}, name)", "refresh": 2, "includeAll": false, "multi": false, - "current": { "text": "critical", "value": "critical" } + "current": { + "text": "critical", + "value": "critical" + } }, { "name": "growth_window", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "query": "label_values({__name__=~\".*_forecast_growth_rate\",group=~\"$group\"}, window)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values({__name__=~\".+_forecast_growth_rate\",group=~\"$group\"}, window)", "refresh": 2, "includeAll": false, "multi": false, - "current": { "text": "1d", "value": "1d" } + "current": { + "text": "1d", + "value": "1d" + } } ] }, @@ -71,11 +105,19 @@ "type": "stat", "id": 1, "title": "Series with ETA < 6h ($threshold @ $level)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, "targets": [ { - "expr": "count({__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"} < 6 * 60 * 60)", + "expr": "count({__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"} < 6 * 60 * 60)", "legendFormat": "imminent" } ], @@ -85,9 +127,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] } }, @@ -98,39 +149,73 @@ "type": "stat", "id": 2, "title": "Median ETA ($threshold @ $level)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, "targets": [ { - "expr": "quantile(0.5, {__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", + "expr": "quantile(0.5, {__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", "legendFormat": "median" } ], - "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } }, { "type": "stat", "id": 3, "title": "Worst ETA ($threshold @ $level)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, "targets": [ { - "expr": "min({__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", + "expr": "min({__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", "legendFormat": "min" } ], - "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } }, { "type": "table", "id": 4, "title": "Fleet ETAs ($threshold @ $level — soonest first)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 6 + }, "targets": [ { - "expr": "topk(50, -{__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", + "expr": "topk(50, -{__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"})", "instant": true, "format": "table" } @@ -139,14 +224,21 @@ { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true } + "excludeByName": { + "Time": true, + "__name__": true + } } }, { "id": "calculateField", "options": { "mode": "binary", - "binary": { "left": "Value", "operator": "*", "right": "-1" }, + "binary": { + "left": "Value", + "operator": "*", + "right": "-1" + }, "alias": "seconds_to_threshold", "replaceFields": true } @@ -157,16 +249,26 @@ "type": "bargauge", "id": 5, "title": "Top-N fastest growing (window=$growth_window)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 12, "w": 12, "x": 0, "y": 18 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 18 + }, "targets": [ { - "expr": "topk(10, {__name__=~\".*_forecast_growth_rate\",group=~\"$group\",id=~\"$id\",window=\"$growth_window\"})", + "expr": "topk(10, {__name__=~\".+_forecast_growth_rate\",group=~\"$group\",id=~\"$id\",window=\"$growth_window\"})", "legendFormat": "{{id}} {{instance}}" } ], "fieldConfig": { - "defaults": { "unit": "short" }, + "defaults": { + "unit": "short" + }, "overrides": [] } }, @@ -174,16 +276,26 @@ "type": "bargauge", "id": 6, "title": "Top-N fastest shrinking (window=$growth_window)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 12, "w": 12, "x": 12, "y": 18 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 18 + }, "targets": [ { - "expr": "bottomk(10, {__name__=~\".*_forecast_growth_rate\",group=~\"$group\",id=~\"$id\",window=\"$growth_window\"})", + "expr": "bottomk(10, {__name__=~\".+_forecast_growth_rate\",group=~\"$group\",id=~\"$id\",window=\"$growth_window\"})", "legendFormat": "{{id}} {{instance}}" } ], "fieldConfig": { - "defaults": { "unit": "short" }, + "defaults": { + "unit": "short" + }, "overrides": [] } }, @@ -191,15 +303,28 @@ "type": "timeseries", "id": 7, "title": "ETA evolution ($threshold @ $level)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 30 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 30 + }, "targets": [ { - "expr": "{__name__=~\".*_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"}", + "expr": "{__name__=~\".+_forecast_time_to_threshold_seconds\",group=~\"$group\",id=~\"$id\",name=\"$threshold\",level=\"$level\"}", "legendFormat": "{{id}} {{instance}}" } ], - "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } } ] } diff --git a/dashboards/grafana/promforecast-deviation.json b/dashboards/grafana/promforecast-deviation.json index 07acb6a..766005f 100644 --- a/dashboards/grafana/promforecast-deviation.json +++ b/dashboards/grafana/promforecast-deviation.json @@ -4,9 +4,16 @@ "schemaVersion": 39, "version": 1, "editable": true, - "tags": ["promforecast", "forecasting", "anomaly"], + "tags": [ + "promforecast", + "forecasting", + "anomaly" + ], "timezone": "", - "time": { "from": "now-24h", "to": "now" }, + "time": { + "from": "now-24h", + "to": "now" + }, "refresh": "1m", "templating": { "list": [ @@ -14,12 +21,18 @@ "name": "datasource", "type": "datasource", "query": "prometheus", - "current": {} + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } }, { "name": "group", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_deviation_outside_band, group)", "refresh": 2, "includeAll": true, @@ -28,7 +41,10 @@ { "name": "id", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_deviation_outside_band{group=~\"$group\"}, id)", "refresh": 2, "includeAll": true, @@ -37,17 +53,26 @@ { "name": "level", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_deviation_outside_band{group=~\"$group\",id=~\"$id\"}, level)", "refresh": 2, "includeAll": false, "multi": false, - "current": { "text": "95", "value": "95" } + "current": { + "text": "95", + "value": "95" + } }, { "name": "quality_floor", "type": "textbox", - "current": { "text": "0.6", "value": "0.6" } + "current": { + "text": "0.6", + "value": "0.6" + } } ] }, @@ -56,8 +81,16 @@ "type": "stat", "id": 1, "title": "Series outside band right now (quality > $quality_floor)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, "targets": [ { "expr": "sum(forecast_deviation_outside_band{group=~\"$group\",id=~\"$id\",level=\"$level\"} == 1 and on (id, group, model) forecast_quality_score > $quality_floor)", @@ -70,9 +103,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] } }, @@ -83,64 +125,124 @@ "type": "stat", "id": 2, "title": "Median |deviation ratio|", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, "targets": [ { "expr": "quantile(0.5, abs(forecast_deviation_ratio{group=~\"$group\",id=~\"$id\",level=\"$level\"}))", "legendFormat": "median" } ], - "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } }, { "type": "stat", "id": 3, "title": "p99 |deviation ratio|", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, "targets": [ { "expr": "quantile(0.99, abs(forecast_deviation_ratio{group=~\"$group\",id=~\"$id\",level=\"$level\"}))", "legendFormat": "p99" } ], - "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } }, { "type": "timeseries", "id": 4, "title": "Outside-band count over time", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 6 + }, "targets": [ { "expr": "sum by (id) (forecast_deviation_outside_band{group=~\"$group\",id=~\"$id\",level=\"$level\"})", "legendFormat": "{{id}}" } ], - "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } }, { "type": "timeseries", "id": 5, "title": "Deviation ratio (signed)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 6 + }, "targets": [ { "expr": "forecast_deviation_ratio{group=~\"$group\",id=~\"$id\",level=\"$level\"}", "legendFormat": "{{id}} {{model}}" } ], - "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } }, { "type": "table", "id": 6, "title": "Series currently outside band (gated by quality > $quality_floor)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 15 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 15 + }, "targets": [ { "expr": "forecast_deviation_outside_band{group=~\"$group\",id=~\"$id\",level=\"$level\"} == 1 and on (id, group, model) forecast_quality_score > $quality_floor", @@ -149,7 +251,15 @@ } ], "transformations": [ - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true } } } + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + } + } + } ] } ] diff --git a/dashboards/grafana/promforecast-node-overlay-example.json b/dashboards/grafana/promforecast-node-overlay-example.json index 8771c70..ff928ca 100644 --- a/dashboards/grafana/promforecast-node-overlay-example.json +++ b/dashboards/grafana/promforecast-node-overlay-example.json @@ -1,13 +1,21 @@ { "title": "promforecast — Node overlay (example)", "uid": "promforecast-node-overlay", - "description": "Example: how a typical node CPU/memory dashboard looks once forecasts are layered on. With only the snapshot exporter (no remote_write yet), the trick is to plot the forecast metric with `offset ` so 'predicted-30m-ago' aligns on the X axis with 'actual now'. Where they track each other, the model is calibrated; where they diverge, that's prediction error. v0.3 remote_write will add real forecast curves projected into the future.", + "description": "Example: how a typical node CPU/memory dashboard looks once forecasts are layered on. Assumes the sink+VM emission path (the umbrella chart default). Each overlay plots the live signal next to its forecast curve, which the sink pushes into VictoriaMetrics with native future timestamps — the dashed line extends past `now` rather than being snapshot-aligned with offset trickery. The shaded band is the 95% confidence interval; the annotation row marks moments when an actual fell outside it.", "schemaVersion": 39, - "version": 1, + "version": 2, "editable": true, - "tags": ["promforecast", "forecasting", "node", "example"], + "tags": [ + "promforecast", + "forecasting", + "node", + "example" + ], "timezone": "", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-2h", + "to": "now+30m" + }, "refresh": "30s", "templating": { "list": [ @@ -15,17 +23,26 @@ "name": "datasource", "type": "datasource", "query": "prometheus", - "current": {} + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } }, { "name": "instance", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(node_cpu_busy_pct_forecast, instance)", "refresh": 2, "includeAll": true, "multi": true, - "current": { "text": "All", "value": "$__all" } + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -33,12 +50,15 @@ "list": [ { "name": "Outside band", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "enable": true, "iconColor": "red", - "expr": "forecast_deviation_outside_band{instance=~\"$instance\",level=\"95\"} == 1 and on(instance,id,model) forecast_quality_score > 0", + "expr": "forecast_deviation_outside_band{instance=~\"$instance\",level=\"95\"} == 1 and on(id,group,instance,model) forecast_quality_score > 0.6", "titleFormat": "Outside 95% band", - "tagKeys": "id,instance,model" + "tagKeys": "id,group,model" } ] }, @@ -47,7 +67,12 @@ "type": "row", "id": 100, "title": "Now vs. predicted (+30m)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false }, { @@ -55,8 +80,16 @@ "id": 1, "title": "CPU now", "description": "Live CPU busy %, computed from node_cpu_seconds_total exactly as a normal node-exporter dashboard would.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, "targets": [ { "expr": "100 - (avg by (instance)(rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"$instance\"}[2m])) * 100)", @@ -69,9 +102,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "red", "value": 85 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 85 + } ] } }, @@ -81,13 +123,21 @@ { "type": "stat", "id": 2, - "title": "CPU in +30m (predicted)", - "description": "What the AutoARIMA model thinks CPU will be 30 minutes from now. Single-point snapshot — every refresh_interval the forecaster re-fits and overwrites this value.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "title": "CPU at +30m (predicted)", + "description": "What the AutoARIMA model thinks CPU will be 30 minutes from now. Reads the end-of-horizon forecast carried by the sink-pushed curve.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, "targets": [ { - "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\",model=\"AutoARIMA\"}", + "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\",model=\"AutoARIMA\",horizon=\"next\"}", "legendFormat": "{{instance}}" } ], @@ -97,9 +147,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "red", "value": 85 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 85 + } ] } }, @@ -111,8 +170,16 @@ "id": 3, "title": "Memory now", "description": "Live memory in use.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, "targets": [ { "expr": "node_memory_MemTotal_bytes{instance=~\"$instance\"} - node_memory_MemAvailable_bytes{instance=~\"$instance\"}", @@ -120,34 +187,50 @@ } ], "fieldConfig": { - "defaults": { "unit": "bytes" }, + "defaults": { + "unit": "bytes" + }, "overrides": [] } }, { "type": "stat", "id": 4, - "title": "Memory in +30m (predicted)", - "description": "Predicted memory usage 30 minutes ahead.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, + "title": "Memory at +30m (predicted)", + "description": "Predicted memory usage at the end of the configured horizon.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, "targets": [ { - "expr": "node_memory_used_bytes_forecast{instance=~\"$instance\",model=\"AutoARIMA\"}", + "expr": "node_memory_used_bytes_forecast{instance=~\"$instance\",model=\"AutoARIMA\",horizon=\"next\"}", "legendFormat": "{{instance}}" } ], "fieldConfig": { - "defaults": { "unit": "bytes" }, + "defaults": { + "unit": "bytes" + }, "overrides": [] } }, - { "type": "row", "id": 200, "title": "Forecast trustworthiness", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, "collapsed": false }, { @@ -155,12 +238,20 @@ "id": 5, "title": "Quality score — CPU", "description": "0..1 composite of MAPE, band width, and data coverage. Anything < 0.4 means: do not trust this forecast. Anything > 0.7 is solid.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 7 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, "targets": [ { "expr": "forecast_quality_score{id=\"node_cpu_busy_pct\",instance=~\"$instance\"}", - "legendFormat": "{{instance}}" + "legendFormat": "{{instance}} / {{model}}" } ], "fieldConfig": { @@ -171,9 +262,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.4 }, - { "color": "green", "value": 0.7 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.4 + }, + { + "color": "green", + "value": 0.7 + } ] } }, @@ -184,12 +284,20 @@ "type": "gauge", "id": 6, "title": "Quality score — Memory", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 7 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, "targets": [ { "expr": "forecast_quality_score{id=\"node_memory_used_bytes\",instance=~\"$instance\"}", - "legendFormat": "{{instance}}" + "legendFormat": "{{instance}} / {{model}}" } ], "fieldConfig": { @@ -200,9 +308,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.4 }, - { "color": "green", "value": 0.7 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.4 + }, + { + "color": "green", + "value": 0.7 + } ] } }, @@ -214,37 +331,62 @@ "id": 7, "title": "Auto-select winners", "description": "Which model the auto-selector picked per series. ``model_fallback=true`` means the configured fallback model is being used because the configured models all backtested poorly.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 7 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 7 + }, "targets": [ { - "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\"}", + "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\",horizon=\"next\"}", "instant": true, "format": "table", "legendFormat": "{{id}} {{instance}} → {{model}} (fallback={{model_fallback}})" } ], "fieldConfig": { - "defaults": { "unit": "none" }, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "options": { "textMode": "name", "graphMode": "none" } + "options": { + "textMode": "name", + "graphMode": "none" + } }, - { "type": "row", "id": 300, - "title": "CPU: actual vs predicted (offset overlay)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "title": "CPU: actual + forward forecast", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "collapsed": false }, { "type": "timeseries", "id": 8, - "title": "CPU — actual now vs. what was predicted 30m ago for now", - "description": "Solid line: actual CPU at time T. Dashed line: the forecast MADE at T-30m, i.e. 'what the model thought CPU would be at T'. The two should track each other when the model is calibrated. The shaded area is the 95% confidence band, also offset by 30m. Vertical red lines (annotations) mark times when the actual fell outside the 95% band — gated by quality > 0 to suppress noise from poor fits.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 14 }, + "title": "CPU — actual vs. forward forecast (95% band)", + "description": "Solid line: actual CPU. Dashed line: the sink-pushed forecast curve, extending from now into the future. The shaded area is the 95% confidence band. Vertical red lines (annotations) mark times when the actual fell outside the 95% band — gated by quality > 0 to suppress noise from poor fits.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 14 + }, "targets": [ { "expr": "100 - (avg by (instance)(rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"$instance\"}[2m])) * 100)", @@ -252,64 +394,119 @@ "refId": "A" }, { - "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\",model=\"AutoARIMA\"} offset 30m", - "legendFormat": "predicted-30m-ago {{instance}}", + "expr": "node_cpu_busy_pct_forecast{instance=~\"$instance\",model=\"AutoARIMA\"}", + "legendFormat": "forecast {{instance}}", "refId": "B" }, { - "expr": "node_cpu_busy_pct_forecast_lower{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"} offset 30m", + "expr": "node_cpu_busy_pct_forecast_lower{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"}", "legendFormat": "lower 95% {{instance}}", "refId": "C" }, { - "expr": "node_cpu_busy_pct_forecast_upper{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"} offset 30m", + "expr": "node_cpu_busy_pct_forecast_upper{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"}", "legendFormat": "upper 95% {{instance}}", "refId": "D" } ], "fieldConfig": { - "defaults": { "unit": "percent" }, + "defaults": { + "unit": "percent" + }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*predicted-30m-ago.*" }, + "matcher": { + "id": "byRegexp", + "options": "^forecast .*$" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "custom.lineWidth", "value": 2 } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*lower 95%.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*lower 95%.*" + }, "properties": [ - { "id": "custom.lineWidth", "value": 0 }, - { "id": "custom.fillOpacity", "value": 0 } + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*upper 95%.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*upper 95%.*" + }, "properties": [ - { "id": "custom.lineWidth", "value": 0 }, - { "id": "custom.fillOpacity", "value": 15 }, - { "id": "custom.fillBelowTo", "value": "lower 95% " } + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 15 + }, + { + "id": "custom.fillBelowTo", + "value": "lower 95% " + } ] } ] }, - "options": { "legend": { "displayMode": "table", "placement": "bottom" } } + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + } + } }, - { "type": "row", "id": 400, - "title": "Memory: actual vs predicted (offset overlay)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "title": "Memory: actual + forward forecast", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "collapsed": false }, { "type": "timeseries", "id": 9, - "title": "Memory — actual now vs. what was predicted 30m ago for now", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 25 }, + "title": "Memory — actual vs. forward forecast (95% band)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, "targets": [ { "expr": "node_memory_MemTotal_bytes{instance=~\"$instance\"} - node_memory_MemAvailable_bytes{instance=~\"$instance\"}", @@ -317,56 +514,103 @@ "refId": "A" }, { - "expr": "node_memory_used_bytes_forecast{instance=~\"$instance\",model=\"AutoARIMA\"} offset 30m", - "legendFormat": "predicted-30m-ago {{instance}}", + "expr": "node_memory_used_bytes_forecast{instance=~\"$instance\",model=\"AutoARIMA\"}", + "legendFormat": "forecast {{instance}}", "refId": "B" }, { - "expr": "node_memory_used_bytes_forecast_lower{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"} offset 30m", + "expr": "node_memory_used_bytes_forecast_lower{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"}", "legendFormat": "lower 95% {{instance}}", "refId": "C" }, { - "expr": "node_memory_used_bytes_forecast_upper{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"} offset 30m", + "expr": "node_memory_used_bytes_forecast_upper{instance=~\"$instance\",model=\"AutoARIMA\",level=\"95\"}", "legendFormat": "upper 95% {{instance}}", "refId": "D" } ], "fieldConfig": { - "defaults": { "unit": "bytes" }, + "defaults": { + "unit": "bytes" + }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*predicted-30m-ago.*" }, + "matcher": { + "id": "byRegexp", + "options": "^forecast .*$" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "custom.lineWidth", "value": 2 } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*lower 95%.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*lower 95%.*" + }, "properties": [ - { "id": "custom.lineWidth", "value": 0 }, - { "id": "custom.fillOpacity", "value": 0 } + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*upper 95%.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*upper 95%.*" + }, "properties": [ - { "id": "custom.lineWidth", "value": 0 }, - { "id": "custom.fillOpacity", "value": 15 }, - { "id": "custom.fillBelowTo", "value": "lower 95% " } + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 15 + }, + { + "id": "custom.fillBelowTo", + "value": "lower 95% " + } ] } ] }, - "options": { "legend": { "displayMode": "table", "placement": "bottom" } } + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + } + } }, - { "type": "row", "id": 500, "title": "Forecast diagnostics over time", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, "collapsed": false }, { @@ -374,23 +618,44 @@ "id": 10, "title": "MAPE by horizon (lower is better)", "description": "Backtest MAPE per (model, horizon). A spike here means the model lost calibration — investigate before trusting the forward forecast.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 36 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, "targets": [ { "expr": "forecast_accuracy_mape{instance=~\"$instance\",model=\"AutoARIMA\"}", "legendFormat": "{{id}} @{{horizon}} {{instance}}" } ], - "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + } }, { "type": "timeseries", "id": 11, "title": "Deviation ratio (signed)", "description": "How far the most recent actual sits from the forecast band centre, normalised by band width. ±0.5 = at the edge of the band; |x| > 0.5 = outside; 0 = perfectly on the forecast.", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 36 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 36 + }, "targets": [ { "expr": "forecast_deviation_ratio{instance=~\"$instance\",level=\"95\",model=\"AutoARIMA\"}", @@ -400,13 +665,26 @@ "fieldConfig": { "defaults": { "unit": "none", - "custom": { "thresholdsStyle": { "mode": "line" } }, + "custom": { + "thresholdsStyle": { + "mode": "line" + } + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.5 }, - { "color": "red", "value": 2 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } ] } }, diff --git a/dashboards/grafana/promforecast-quality.json b/dashboards/grafana/promforecast-quality.json index 8f00466..3f2d19b 100644 --- a/dashboards/grafana/promforecast-quality.json +++ b/dashboards/grafana/promforecast-quality.json @@ -4,9 +4,16 @@ "schemaVersion": 39, "version": 1, "editable": true, - "tags": ["promforecast", "forecasting", "quality"], + "tags": [ + "promforecast", + "forecasting", + "quality" + ], "timezone": "", - "time": { "from": "now-7d", "to": "now" }, + "time": { + "from": "now-7d", + "to": "now" + }, "refresh": "5m", "templating": { "list": [ @@ -14,12 +21,18 @@ "name": "datasource", "type": "datasource", "query": "prometheus", - "current": {} + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } }, { "name": "group", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_quality_score, group)", "refresh": 2, "includeAll": true, @@ -28,7 +41,10 @@ { "name": "id", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_quality_score{group=~\"$group\"}, id)", "refresh": 2, "includeAll": true, @@ -37,7 +53,10 @@ { "name": "model", "type": "query", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "query": "label_values(forecast_quality_score{group=~\"$group\",id=~\"$id\"}, model)", "refresh": 2, "includeAll": true, @@ -50,8 +69,16 @@ "type": "stat", "id": 1, "title": "Mean quality (0..1)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, "targets": [ { "expr": "avg(forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -66,9 +93,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.4 }, - { "color": "green", "value": 0.7 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.4 + }, + { + "color": "green", + "value": 0.7 + } ] } }, @@ -79,22 +115,43 @@ "type": "stat", "id": 2, "title": "Series above 0.7 quality", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, "targets": [ { "expr": "count(forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"} > 0.7)", "legendFormat": "high quality" } ], - "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + } }, { "type": "stat", "id": 3, "title": "Series below 0.3 quality", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, "targets": [ { "expr": "count(forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"} < 0.3)", @@ -107,9 +164,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] } }, @@ -120,8 +186,16 @@ "type": "timeseries", "id": 4, "title": "Quality distribution over time", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 6 + }, "targets": [ { "expr": "quantile(0.1, forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -137,7 +211,11 @@ } ], "fieldConfig": { - "defaults": { "unit": "none", "min": 0, "max": 1 }, + "defaults": { + "unit": "none", + "min": 0, + "max": 1 + }, "overrides": [] } }, @@ -145,8 +223,16 @@ "type": "timeseries", "id": 5, "title": "Quality by model", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 6 + }, "targets": [ { "expr": "avg by (model) (forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -154,7 +240,11 @@ } ], "fieldConfig": { - "defaults": { "unit": "none", "min": 0, "max": 1 }, + "defaults": { + "unit": "none", + "min": 0, + "max": 1 + }, "overrides": [] } }, @@ -162,8 +252,16 @@ "type": "table", "id": 6, "title": "Lowest-quality series (current)", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 15 }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 15 + }, "targets": [ { "expr": "bottomk(20, forecast_quality_score{group=~\"$group\",id=~\"$id\",model=~\"$model\"})", @@ -172,11 +270,23 @@ } ], "fieldConfig": { - "defaults": { "unit": "none", "min": 0, "max": 1 }, + "defaults": { + "unit": "none", + "min": 0, + "max": 1 + }, "overrides": [] }, "transformations": [ - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true } } } + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + } + } + } ] } ] diff --git a/docker/prometheus.yml b/docker/prometheus.yml index fe69f53..3d007ba 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -42,13 +42,28 @@ scrape_configs: static_configs: - targets: ["forecaster:9091"] # Drop forecast snapshot gauges (``*_forecast``, ``*_forecast_lower``, - # ``*_forecast_upper``) from the scrape. The forecaster's remote_write - # sink already pushes the full forecast curve into VM directly; the - # scraped snapshot is a duplicate with a different label set (carries - # ``instance="forecaster:9091"``) that confuses dashboards. Operational - # metrics (``forecast_failures_total``, ``forecast_quality_score``, + # ``*_forecast_upper``) from the scrape. + # + # *** Coupled to ``sink.remote_write.enabled`` in docker/dev-config.yaml. *** + # + # The dev-stack forecaster runs with ``sink.remote_write.enabled: true`` + # (see docker/dev-config.yaml) so the full forecast curve is pushed into + # VictoriaMetrics directly. The scraped snapshot would be a duplicate + # under a different label set (carries ``instance="forecaster:9091"``, + # ``job="promforecast"``) and any dashboard not pre-filtering on the + # labelset renders an average of the two — masking the smooth curve + # under the 5-min snapshot staircase. The Helm charts derive the same + # drop from the same toggle (see ``serviceMonitor.dropForecastSnapshots`` + # in charts/promforecast/values.yaml and the umbrella default in + # charts/promforecast-stack/values.yaml). If you turn the sink off in + # docker/dev-config.yaml, comment out the metric_relabel_configs block + # below so the snapshot becomes the canonical write path again. + # + # Operational metrics (``forecast_failures_total``, ``forecast_quality_score``, # ``forecast_run_duration_seconds``, etc.) stay scraped — they only - # exist on /metrics. + # exist on /metrics regardless of sink configuration. See + # docs/architecture/emission-paths.md for the dual-source trap and the + # cross-metric-join labelset asymmetry this drop preserves. metric_relabel_configs: - source_labels: [__name__] regex: '.+_forecast(_lower|_upper)?' diff --git a/docs/README.md b/docs/README.md index a81da29..efb72a5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,9 +9,12 @@ The docs follow the [Diataxis](https://diataxis.fr/) framework: ## Index +- [Emission paths (snapshot vs. sink + VM)](architecture/emission-paths.md) — what each path emits, the dual-source trap, and why the official charts default to sink + VM - [Config schema](config-schema.md) — `apiVersion`, stability guarantees, and the `v1alpha1` → `v1` migration - [Models](models.md) — built-in forecasters, auto-select, accuracy metrics -- [CLI](cli.md) — `promforecast run` and `promforecast validate` +- [CLI](cli.md) — `promforecast run`, `promforecast validate`, `promforecast diagnose` +- [Reload verification](operations/reload.md) — confirming a config reload actually picked up the expected queries +- [Duplicate-labelset diagnosis](operations/diagnose.md) — detecting and remediating the scrape+sink dual-source state - [High availability](ha.md) — Lease-based leader election, Redis snapshot cache - [Query cache](cache.md) — in-memory LRU + optional Redis dedup for overlapping PromQL - [Series discovery](discovery.md) — template-driven auto-fan-out over label values diff --git a/docs/architecture/emission-paths.md b/docs/architecture/emission-paths.md new file mode 100644 index 0000000..e128026 --- /dev/null +++ b/docs/architecture/emission-paths.md @@ -0,0 +1,252 @@ +# Emission paths: `/metrics` snapshot vs. sink + VM + +promforecast can publish each forecast through two different write paths. +Production deployments should pick one of them as the canonical store; the +official charts default to **sink + VM** for a fresh install. This page +explains what each path actually emits, what their trade-offs are, why +running both unguarded silently produces wrong-looking dashboards, and +how the chart defaults steer you onto the recommended path. + +## The two paths + +### 1. `/metrics` snapshot (pull) + +Every fit overwrites a single gauge per (series, model, horizon). When +Prometheus scrapes the forecaster's `/metrics` endpoint it sees the +**end-of-horizon** value as a single point at the scrape timestamp; the +intermediate forecast steps from `t+1` to `t+horizon` are not exposed. + +- **What gets stored.** Per scrape interval, one sample per emitted + forecast metric per series, carrying the forecaster's scrape labels + (`instance=":9091"`, `job=""`, plus whatever `cluster` / + `replica` external labels the scraper adds). +- **What it looks like on a Grafana time-series.** A 5-minute staircase + — each scrape redraws the gauge at the current value, so the line + steps every refresh interval rather than tracing a smooth curve. +- **When it's the right pick.** Single-replica low-cardinality installs + where the snapshot is enough to feed alerts on end-of-horizon + thresholds, and you don't need to render the forecast curve as a + smooth projection. + +### 2. `sink.remote_write` → VictoriaMetrics (push) + +After every fit the forecaster pushes the **full forecast curve** (every +step from `t+1` to `t+horizon`) into a long-term TSDB via +`remote_write` with native future timestamps. VictoriaMetrics accepts +future samples when started with `-futureRetention=2d` or higher. + +- **What gets stored.** Per fit, one sample *per future step* per + series — carrying only the source labels of the underlying signal + (no `instance=":9091"`, no `job="promforecast"`). +- **What it looks like on a Grafana time-series.** A smooth curve + extending from `now` into the future, with `_lower` / `_upper` bands + rendered as a shaded interval. No `offset ` trickery is + needed to align actual against predicted. +- **When it's the right pick.** Almost always. Smooth curves are what + capacity-planning and deviation-detection dashboards want; the + forecast becomes a first-class historical signal queryable with + normal PromQL. + +## The dual-source trap + +Running both paths against the same long-term TSDB without filtering +puts **two copies of every `*_forecast` / `*_forecast_lower` / +`*_forecast_upper` series** in the store under different labelsets: + +- The scraped snapshot copy carries `job="promforecast"` (set by the + ServiceMonitor's `jobLabel: app.kubernetes.io/name`), the + forecaster's source `instance` (preserved by `honorLabels: true`), + and any external labels Prometheus' `remote_write` adds at ingest + (`cluster`, `replica`). +- The sink-pushed copy carries the source signal's labels + (`instance="node-exporter:9100"`, `device`, etc.) and **no `job`** — + the forecaster pushes directly to VM, bypassing the Prometheus scrape + pipeline that would otherwise stamp one on. + +The labelset difference therefore reduces to the presence (or absence) +of the `job` label, plus any external labels the scrape pipeline adds +at remote_write time. Both the helm-test probe in +`charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml` +and the manual verification snippet below use exactly this asymmetry. + +A dashboard panel that does not pre-filter on the labelset renders an +**average** of the two — the smooth sink curve smeared against the +5-minute snapshot staircase. The graph looks plausibly noisy at a +glance; the failure mode is silent. + +This is the implicit default state of any install that turns the sink +on *without* dropping the snapshot families from the scrape. Both the +umbrella chart and the standalone chart fix this by default (see +"Recommended path" below). + +## Recommended path: sink + VM, snapshot dropped from the scrape + +Production deployments should: + +1. **Enable the sink.** Set `config.sink.remote_write.enabled: true` + and point it at the long-term TSDB. +2. **Drop the snapshot families from the scrape.** Apply a + `metric_relabel_configs` block (or `metricRelabelings` on the + ServiceMonitor) that drops `.+_forecast(_lower|_upper)?`. The + operational `forecast_*` metrics (`forecast_quality_score`, + `forecast_failures_total`, `forecast_accuracy_mape`, …) and the + derived snapshot-only families (`*_forecast_time_to_threshold_seconds`, + `*_forecast_growth_rate`, `*_forecast_contribution_ratio`) stay + scraped — they only exist on `/metrics`. +3. **Point dashboards at VM as the default datasource.** The bundled + dashboards under `dashboards/grafana/` declare VictoriaMetrics + (`uid: victoriametrics`) as their templating default. The umbrella + chart can optionally provision the matching datasource entries — + set `datasources.enabled: true` (off by default to avoid colliding + with installations that already manage Grafana datasources + out-of-band). When enabled, the chart writes a sidecar-loaded + ConfigMap pointing `uid: victoriametrics` at the bundled VM. If + Grafana already has a default datasource, set + `datasources.victoriaMetrics.isDefault: false` so the bundled + provisioning does not collide with it — the dashboards keep working + because they reference the explicit UID, not the global default. + +The official charts ship that combination on by default: + +- The **umbrella `promforecast-stack` chart** sets + `promforecast.config.sink.remote_write.enabled: true` and + `promforecast.serviceMonitor.enabled: true`. The forecaster's + ServiceMonitor (Story 2) then derives the snapshot drop from the + sink toggle automatically, so a fresh `helm install` lands on + sink+VM with the dual-source trap closed. +- The **standalone `promforecast` chart** exposes + `serviceMonitor.dropForecastSnapshots` (default `null`). When + `null` it resolves to the value of `config.sink.remote_write.enabled`: + the drop is on when the sink is on, off when the sink is off. Pin + to `true` / `false` to override the derivation. + +For an externally-managed Prometheus (not driven by prometheus-operator), +copy this snippet into the scrape job that targets the forecaster: + +```yaml +scrape_configs: + - job_name: promforecast + honor_labels: true + static_configs: + - targets: ["forecaster:9091"] + metric_relabel_configs: + - source_labels: [__name__] + regex: '.+_forecast(_lower|_upper)?' + action: drop +``` + +The two knobs do different things and both matter: + +- `metric_relabel_configs` drops the snapshot copy of the curve families + (`*_forecast`, `*_forecast_lower`, `*_forecast_upper`) so only the + sink-pushed copy lives in the TSDB. Without this the dual-source trap + reappears. +- `honor_labels: true` stops Prometheus from *overwriting* the + `instance` label that the forecaster exposes on its operational and + derived `*_forecast_*` families (`forecast_quality_score`, + `*_forecast_time_to_threshold_seconds`, `*_forecast_growth_rate`, + `*_forecast_contribution_ratio`). Those families inherit the source + signal's `instance` (e.g. `node-exporter:9100`) and the example + alerts in `examples/alerts/promforecast-rules.yaml` interpolate + `{{ $labels.instance }}` to identify the underlying host. Default + Prometheus scraping would clobber that with the forecaster pod's own + address; `honor_labels: true` preserves it. The umbrella chart's + ServiceMonitor sets the equivalent `honorLabels: true`. + +### Verifying single-labelset after install + +After `helm install` and at least one successful refresh (the forecaster's +`refresh_interval`, one hour by default, so the **first scheduled fit +must have completed** before the sink push lands any data in VM), query +VM directly to confirm the curve families exist under exactly one +labelset. PromQL matcher values must be double-quoted, so the snippets +below use single-quoted shell wrappers to pass them through verbatim: + +``` +# Expect ``data: []`` — no forecast series under the forecaster scrape job. +curl -s --data-urlencode 'match[]={__name__=~".+_forecast(_lower|_upper)?",job="promforecast"}' \ + "$VM/api/v1/series" + +# Expect a non-empty data array — sink-pushed copies (no scrape job). +curl -s --data-urlencode 'match[]={__name__=~".+_forecast(_lower|_upper)?",job=""}' \ + "$VM/api/v1/series" +``` + +The umbrella chart bundles this exact pair of probes as a `helm test` +hook (`charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml`) +so you can run `helm test ` instead of curling by hand. The +hook polls VM for up to `tests.firstRefreshTimeout` (default `5m`) so +it tolerates the post-install delay before the first scheduled refresh +lands the sink push. + +## Why operational metrics still carry source-side labels + +The forecaster's operational metrics — `forecast_quality_score`, +`forecast_accuracy_mape`, `forecast_deviation_outside_band`, +`forecast_deviation_ratio`, `forecast_failures_total`, and the +threshold / growth-rate derived families — are emitted on `/metrics` and +inherit the same `(id, group, model, instance, …)` source labels as +the underlying series. This is intentional and aligned with design +principle 8 (label alignment with the sibling `promanomaly` project): +keeping the source `instance` (and other join keys) on the operational +metrics lets you write `and on (id, group, instance) ` +PromQL joins without label-rewriting recording rules. The scrape labels +(`job="promforecast"`, `instance=":9091"`) sit alongside as +metadata; PromQL's `on (...)` joins are explicit about which labels +they care about, so the extra metadata does not break the join. + +### Cross-metric joins: the labelset asymmetry + +The sink-pushed curve families (`*_forecast`, `*_forecast_lower`, +`*_forecast_upper`) carry only source labels — **no `job`, no forecaster +`instance`**. The scraped operational families +(`forecast_quality_score`, `forecast_deviation_outside_band`, …) carry +source labels **plus** `job="promforecast"`. That asymmetry is by +design — it lets you join across them as long as you are explicit about +the join keys. + +**Wrong** (relies on the default "everything matches" join): + +```promql +# Returns zero results — RHS has ``job="promforecast"`` that LHS lacks, +# so the implicit full-labelset match never finds a peer. +forecast_deviation_outside_band == 1 + and forecast_quality_score > 0.6 +``` + +**Right** (explicit `on(...)` restricts the join to keys present on both +sides): + +```promql +forecast_deviation_outside_band{level="95"} == 1 + and on (id, group, model) forecast_quality_score > 0.6 +``` + +The bundled example alerts in `examples/alerts/promforecast-rules.yaml` +use exactly this form. If you copy them and drop the `on(...)` clause +the rule silently matches zero series. + +## Trade-off summary + +| Concern | Snapshot only | Sink + VM (recommended) | +| --- | --- | --- | +| Cardinality footprint | one sample per series per scrape | one sample per series per *future step* per fit | +| Visual quality on Grafana | 5-minute staircase | smooth forward curve | +| Forecast history queryable as PromQL | only the latest gauge | full curve back to install time | +| Query cost | cheap (single point per series) | proportional to horizon × refresh density | +| Storage cost | minimal | grows with horizon and refresh interval | +| TSDB requirements | any Prometheus-compatible store | must accept future-timestamped writes (VM `-futureRetention`, Mimir, Thanos receive) | +| Setup complexity | one config block (snapshot is on by default) | sink config + future retention + scrape-drop | +| Dual-source risk | n/a | mitigated by scrape-drop | + +The cardinality and storage figures cut both ways: the sink path is +strictly more expensive per emission, but it eliminates the need for +`offset ` Grafana tricks and gives you forecast history as a +queryable time-series instead of just a current-snapshot gauge. + +## Cross-references + +- The umbrella chart values that ship sink-first: `charts/promforecast-stack/values.yaml`. +- The standalone chart knob that derives the drop: `charts/promforecast/values.yaml` (`serviceMonitor.dropForecastSnapshots`). +- The example alert rules that assume sink labels: `examples/alerts/promforecast-rules.yaml`. +- The bundled dashboards that default to VM: `dashboards/grafana/`. diff --git a/docs/cli.md b/docs/cli.md index 200d128..dde691b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,6 +1,7 @@ # CLI reference -The `promforecast` binary provides two subcommands. +The `promforecast` binary provides three subcommands: `run`, `validate`, +and `diagnose` (with `duplicates` and `sample-rate` sub-subcommands). Running `promforecast --config foo.yaml` is identical to `promforecast run --config foo.yaml`, so existing container entrypoints continue to work unchanged. ## `promforecast run` @@ -81,3 +82,46 @@ Estimated cardinality (worst case at max_series_per_query): total 34621 WARN: total exceeds safety.max_total_series (5000); series_overflow will drop the excess ``` + +## `promforecast diagnose` + +Operator diagnostics that run against the long-term TSDB rather than the +config file. Two subcommands: + +### `promforecast diagnose duplicates` + +```bash +promforecast diagnose duplicates \ + [--datasource | --config ] \ + [--selector '{...}'] [--since ] [--json] \ + [--bearer-token ] [--basic-user --basic-password ] \ + [--tls-cert --tls-key ] [--tls-ca ] [--tls-insecure-skip-verify] +``` + +Scans the configured TSDB for forecast metrics +(`*_forecast` / `*_forecast_lower` / `*_forecast_upper`) that appear +under more than one labelset family — the dual-source state described +in [the architecture page](architecture/emission-paths.md). Exit codes +are CI-friendly: `0` for clean, `1` for duplicates detected, `2` for +datasource unreachable. + +### `promforecast diagnose sample-rate` + +```bash +promforecast diagnose sample-rate \ + [--datasource | --config ] \ + [--window ] [--max-samples ] \ + [--selector '{...}'] [--json] \ + [auth flags as above] +``` + +Probes the per-series sample rate via +`count_over_time([])` and flags any series whose count +over the window exceeds `--max-samples`. Covers the same-labelset case +`duplicates` cannot see from labelset shape alone — two ingest paths +writing to the **same** series produce a count higher than either path +alone would. + +See [operations/diagnose.md](operations/diagnose.md) for the full +reference, authentication recipes, in-cluster invocation +(`kubectl exec`), and CI-integration patterns. diff --git a/docs/operations/diagnose.md b/docs/operations/diagnose.md new file mode 100644 index 0000000..b70f4bf --- /dev/null +++ b/docs/operations/diagnose.md @@ -0,0 +1,142 @@ +# `promforecast diagnose` + +Operator diagnostics that query the long-term TSDB (not just the config file). +Two subcommands are currently available: `duplicates` (labelset-shape detection) and `sample-rate` (same-labelset blind spot). + +Both subcommands share common authentication and output flags. + +## `duplicates` + +```bash +promforecast diagnose duplicates \ + [--datasource | --config ] \ + [--selector '{...}'] \ + [--since ] \ + [auth flags] \ + [--json] +``` + +Detects forecast metrics (`*_forecast`, `*_forecast_lower`, `*_forecast_upper`) that appear under more than one distinct labelset family — the classic **dual-source state** caused by running both the `/metrics` scrape and `remote_write` sink into the same TSDB. + +### Exit codes + +| Code | Meaning | Typical use | +|------|------------------------------------------------------|-------------| +| `0` | No duplicate labelset families found | CI / pre-flight pass | +| `1` | At least one forecast metric has duplicate families | CI / pre-flight fail | +| `2` | Datasource unreachable or malformed response | Infra / transient error | + +### Flags + +| Flag | Purpose | +|-------------------|---------| +| `--datasource` | TSDB base URL (e.g. `http://victoriametrics:8428`). Required unless `--config` is used. | +| `--config` | Path to forecaster YAML (inherits `datasource.url`). | +| `--selector` | Optional PromQL label selector (e.g. `'{cluster="prod"}'`). | +| `--since` | Lookback window (e.g. `6h`, `2d`). | +| `--max-concurrent`| Max in-flight requests (default `8`). | +| `--json` | Machine-readable JSON output. | + +### What it detects + +It groups series by their **set of label names** (shape, not values). +Flags a metric when one family is scrape-shaped (contains `job`, `pod`, `replica`, etc.) and another is sink-shaped (lacks those labels). + +Intra-source variance (e.g. `calibrated="true"`) is **not** flagged. + +### Text output example + +```text +FAIL 1 of 27 forecast metric(s) appear under more than one labelset family. + + node_cpu_busy_pct_forecast: + [sink-shaped] count=4 keys=['group', 'instance', 'model', ...] + [scrape-shaped] count=4 keys=['cluster', 'group', 'instance', 'job', 'replica', ...] + +Remediation: + 1. Drop forecast metrics from the forecaster scrape job (recommended). + 2. Or enable the chart-managed drop. + 3. Clean historical scrape-shaped samples with a delete_series call. +``` + +### JSON output + +```json +{ + "has_duplicates": true, + "metrics_scanned": 27, + "findings": [ + { + "metric": "node_cpu_busy_pct_forecast", + "is_duplicate": true, + "families": [ ... ] + } + ] +} +``` + +### CI usage + +```yaml +- name: Check for dual-source state + run: | + promforecast diagnose duplicates --datasource "$VM_URL" --since 1h --json > result.json + test "$(jq -r '.has_duplicates' < result.json)" = "false" +``` + +**Note:** `duplicates` catches distinguishable labelset cases. Use `sample-rate` for the silent same-labelset case (most common with `honor_labels: true`). + +## `sample-rate` + +```bash +promforecast diagnose sample-rate \ + [--datasource | --config ] \ + [--window 10m] \ + [--max-samples 5] \ + [--selector '{...}'] \ + [auth flags] \ + [--json] +``` + +Detects series receiving samples from multiple ingest paths by counting samples over a recent window. + +### Key flags + +- `--window`: Lookback (default `10m`). +- `--max-samples`: Per-series cap (default `5`). Series exceeding this are flagged. + +### Output (text) + +```text +FAIL 4 of 312 series exceeded the configured sample-rate cap. + + node_cpu_busy_pct_forecast: + count=41 labels={...} +``` + +**Tuning tip:** Raise `--max-samples` for very fine `step` or during catch-up after long stalls. + +## Authentication + +Both commands support: + +- `--bearer-token` / `$PROMFORECAST_BEARER_TOKEN` +- `--basic-user` + `--basic-password` +- `--tls-cert` / `--tls-key`, `--tls-ca`, `--tls-insecure-skip-verify` + +## Running in-cluster + +```bash +# Preferred: exec into pod +kubectl exec deploy/promforecast -- promforecast diagnose duplicates --datasource http://victoriametrics:8428 + +# One-shot pod +kubectl run promforecast-diagnose --rm -it --image=ghcr.io/... \ + --env=PROMFORECAST_BEARER_TOKEN=... \ + -- promforecast diagnose duplicates --datasource http://victoriametrics:8428 +``` + +## See also + +- [Emission paths](../architecture/emission-paths.md) — why dual-source is a problem. +- [Migrate to sink-first](../how-to/migrate-to-sink-first.md) — recommended remediation. diff --git a/docs/operations/reload.md b/docs/operations/reload.md new file mode 100644 index 0000000..44e8768 --- /dev/null +++ b/docs/operations/reload.md @@ -0,0 +1,54 @@ +# Verifying a reload picked up the expected changes + +The forecaster reloads its config on three triggers: + +- `POST /-/reload` +- `SIGHUP` +- ConfigMap watcher (`server.reload.watch_configmap: true`, default) + +All three paths use the same validate-then-swap logic. A successful reload always emits: + +1. One `config_reload_succeeded` line. +2. One `group_queries_loaded` line **per group**, listing the active query IDs. + +```json +{"event":"config_reload_succeeded","added":[],"removed":[],"changed":["node_capacity"],"config_hash":"..."} +{"event":"group_queries_loaded","group":"node_capacity","queries":["node_cpu_busy_pct","node_memory_used_bytes","node_load1","node_load5"],"source":"reload"} +``` + +**Recommended verification method:** Check the `group_queries_loaded` lines after every reload. They show exactly what the forecaster is running. + +## Why this matters: the partial-config trap + +`config_reload_succeeded` only confirms the YAML parsed correctly. +On some platforms (notably macOS Docker Desktop bind-mounts), a fast edit can trigger a reload on a **truncated** file. The forecaster accepts the partial config and runs with missing queries. + +The `group_queries_loaded` lines are the only reliable way to detect this. A missing query ID is the smoking gun. + +## Log line fields + +| Field | Meaning | +|-----------|---------| +| `event` | Always `group_queries_loaded` | +| `group` | Group `name` (one line per group, in YAML order) | +| `queries` | List of query `id`s for that group (in YAML order) | +| `source` | `boot` or `reload` | + +Empty groups emit `queries: []` so missing queries are never ambiguous. + +## Boot-time emission + +The same lines appear at startup (with `source: boot`), right after `config_loaded`: + +```json +{"event":"config_loaded","path":"/etc/promforecast/config.yaml","groups":4} +{"event":"group_queries_loaded","group":"node_capacity","queries":["..."],"source":"boot"} +``` + +## Recovery if truncation occurred + +1. Confirm by inspecting the latest `group_queries_loaded` lines — they always reflect reality. +2. Force a clean reload: make a trivial change (e.g. add/remove a comment) to update the file hash, then trigger a reload. +3. If the problem persists, restart the pod to get a fresh mount. + +There is no separate “config status” endpoint by design — the `group_queries_loaded` logs are the source of truth. diff --git a/examples/alerts/promforecast-rules.yaml b/examples/alerts/promforecast-rules.yaml index 17e6c9c..cee7d6c 100644 --- a/examples/alerts/promforecast-rules.yaml +++ b/examples/alerts/promforecast-rules.yaml @@ -1,3 +1,14 @@ +# These example rules assume the sink+VM emission path: the forecaster's +# ``sink.remote_write`` is on and the ``*_forecast`` / ``*_forecast_lower`` +# / ``*_forecast_upper`` snapshot families have been dropped from the +# Prometheus scrape (the umbrella chart and the standalone chart's +# ``serviceMonitor.dropForecastSnapshots`` default this on whenever the +# sink is enabled). Under that path forecast curves carry only their +# source labels (``instance``, ``device``, etc.) — no +# ``instance=":9091"`` and no ``job="promforecast"`` — +# so the expressions below filter purely on source-side labels. See +# ``docs/architecture/emission-paths.md`` for the two-path picture and +# the scrape-drop snippet for an externally-managed Prometheus. apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: @@ -60,17 +71,24 @@ spec: # The exporter keeps falling back to the configured fallback model, # which means the configured `models:` list isn't a good fit for # this data. Symptom of mis-configuration, not an outage. + # Cold-start fits are excluded for the same reason as the quality + # rules below: cold-start routinely engages the fallback while the + # model adapts, so including them would mask configured-models + # failures and noisily flag healthy cold-start ramps. - alert: ForecastFallbackPersistent - expr: count by (group) (forecast_quality_score{model_fallback="true"}) > 0 + expr: | + count by (group) ( + forecast_quality_score{model_fallback="true", cold_start!="true"} + ) > 0 for: 6h labels: severity: info annotations: summary: "Auto-select keeps falling back for group {{ $labels.group }}" description: | - At least one series in {{ $labels.group }} has been falling - back to the configured fallback model for 6h. The configured - models are likely a poor fit for this data. + At least one steady-state series in {{ $labels.group }} has + been falling back to the configured fallback model for 6h. + The configured models are likely a poor fit for this data. # Persistently low quality across a group: forecasts shouldn't be # trusted for alerting. Tune the threshold; 0.4 is a useful default. @@ -139,38 +157,48 @@ spec: # band. Gated on quality so poor forecasts don't fire false # positives. Tune `for:` and the quality threshold to your noise # tolerance. + # Pinned to ``level="95"`` (the conservative band) so the rule + # fires once per series when the actual leaves *that* band, not + # twice — once for the 80% band and again for the 95%. The + # ``forecast_deviation_outside_band`` metric carries a ``level`` + # label that ``forecast_quality_score`` does not, so without + # pinning, the join's right-hand side would bind both 80% and + # 95% rows of the same series and Alertmanager would receive + # two distinct alerts for one incident. - alert: ForecastDeviation expr: | - forecast_deviation_outside_band == 1 + forecast_deviation_outside_band{level="95"} == 1 and on (id, group, model) forecast_quality_score > 0.6 for: 15m labels: severity: warning annotations: - summary: "Actual {{ $labels.id }} is outside its forecast band" + summary: "Actual {{ $labels.id }} is outside its 95% forecast band" description: | {{ $labels.id }} (group {{ $labels.group }}, model - {{ $labels.model }}, level {{ $labels.level }}) has been - outside its forecast confidence band for 15m. Quality score - for this series is above 0.6, so the deviation is likely - meaningful. + {{ $labels.model }}) has been outside its 95% forecast + confidence band for 15m. Quality score for this series is + above 0.6, so the deviation is likely meaningful. # Stronger signal: actual is far outside the band (|ratio| > 2 means # it's a full band-width past the edge). Good candidate for paging. + # Pinned to ``level="95"`` for the same reason as ``ForecastDeviation`` + # above: the metric carries ``level`` and a naive join would fire + # twice per series. - alert: ForecastDeviationSevere expr: | - abs(forecast_deviation_ratio) > 2 + abs(forecast_deviation_ratio{level="95"}) > 2 and on (id, group, model) forecast_quality_score > 0.6 for: 5m labels: severity: critical annotations: - summary: "Actual {{ $labels.id }} is far outside its forecast band" + summary: "Actual {{ $labels.id }} is far outside its 95% forecast band" description: | {{ $labels.id }} (group {{ $labels.group }}, model - {{ $labels.model }}, level {{ $labels.level }}) has a - deviation ratio with magnitude above 2 for 5m. The actual is - well past the edge of the high-confidence forecast band. + {{ $labels.model }}) has a deviation ratio with magnitude + above 2 for 5m. The actual is well past the edge of the + high-confidence forecast band. - name: promforecast.capacity.examples interval: 5m @@ -223,12 +251,7 @@ spec: expr: | node_cpu_busy_pct_forecast_time_to_threshold_seconds{ name="warn", comparator="gt" - } - and on (instance) ( - node_cpu_busy_pct_forecast_time_to_threshold_seconds{ - name="warn", comparator="gt" - } < 4 * 60 * 60 - ) + } < 4 * 60 * 60 for: 30m labels: severity: info diff --git a/forecaster/src/promforecast/diagnose.py b/forecaster/src/promforecast/diagnose.py new file mode 100644 index 0000000..30760bb --- /dev/null +++ b/forecaster/src/promforecast/diagnose.py @@ -0,0 +1,998 @@ +"""``promforecast diagnose`` — operator diagnostics against the long-term TSDB. + +Two subcommands today: + +* ``duplicates`` — scan for the dual-source forecast labelset state described + in ``docs/architecture/emission-paths.md``. Catches the case where the + scraped and sink-pushed copies are *distinguishable* by labelset shape + (different external_labels, ``honor_labels: false`` accidentally set, + source signals missing a ``job`` label). +* ``sample-rate`` — covers the same-labelset case that ``duplicates`` + cannot see from labelset shape alone. For each forecast metric, probe + the per-series sample rate via ``count_over_time``; any series whose + count over the window exceeds the configured cap is flagged as + likely-duplicate ingestion. + +Exit codes (uniform across subcommands): + +* ``0`` — nothing detected. +* ``1`` — duplicate / anomaly detected. +* ``2`` — datasource unreachable / malformed response. + +Designed to drop into CI / pre-flight checks: the JSON output is stable +and the exit codes match the standard "tests passed / failed / errored" +convention so a ``promforecast diagnose ...`` step in a GitHub Action +gates the next stage cleanly. + +The HTTP client is async and probes every forecast metric in parallel +under a small semaphore. Transient TSDB flakes (network blip, 5xx) are +retried with exponential backoff; 4xx (auth, bad query) fails fast so the +operator sees the real error instead of a retry storm masking it. +""" + +from __future__ import annotations + +import asyncio +import json +import re +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import IO, Any + +import httpx +from tenacity import ( + AsyncRetrying, + retry_if_exception, + stop_after_attempt, + wait_exponential, +) + +# HTTP status code for a successful TSDB response. Pulled out to a named +# constant so ruff's PLR2004 magic-number check isn't tripped on the call +# sites that compare against it. +_HTTP_OK = 200 +_HTTP_5XX_FLOOR = 500 +# Length of a Prometheus instant-query value tuple ``[timestamp, value]``. +_VALUE_PAIR_LEN = 2 + + +# Metrics affected by the dual-source trap. The forecaster's operational +# families (``forecast_quality_score``, ``forecast_failures_total``, the +# ``*_time_to_threshold_seconds`` derived ETA metrics) are *not* in scope — +# they live on /metrics only and intentionally carry source-side labels +# like ``job="node-exporter"`` per design principle 8. See the operational- +# metrics section of ``docs/architecture/emission-paths.md``. +FORECAST_METRIC_REGEX = r".+_forecast(_lower|_upper)?" + +# Scrape-only marker labels. A labelset family carrying *any* of these is +# almost certainly the scraped copy: ``job`` is the canonical marker (set +# uniformly to ``promforecast`` by the standalone chart's ServiceMonitor), +# the others are common Kubernetes / Prometheus external_labels that the +# sink push doesn't introduce. The list is deliberately conservative — a +# false positive on a custom external_label is preferable to silently +# missing a real dual-source state. +_SCRAPE_MARKER_KEYS: frozenset[str] = frozenset( + {"job", "service", "pod", "container", "namespace", "cluster", "replica"} +) + + +# --------------------------------------------------------------------------- +# Auth / client construction +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DiagnoseAuth: + """How to authenticate against the TSDB. + + All fields default to ``None`` / ``False`` so callers without auth (the + dev stack, an in-cluster sidecar with NetworkPolicy gating) get a + plain-HTTP client with no overhead. + + Precedence inside the dataclass: bearer overrides basic, both are + independent of TLS knobs. The CLI layer enforces "at most one of + bearer / basic" with argparse mutual-exclusion so the operator can't + accidentally combine them. + """ + + bearer_token: str | None = None + basic_user: str | None = None + basic_password: str | None = None + tls_cert: str | None = None + tls_key: str | None = None + tls_ca: str | None = None + insecure_skip_verify: bool = False + + def build_client_kwargs(self) -> dict[str, Any]: + """Translate the auth bundle into kwargs accepted by ``httpx.AsyncClient``.""" + kwargs: dict[str, Any] = {} + if self.bearer_token: + kwargs["headers"] = {"Authorization": f"Bearer {self.bearer_token}"} + elif self.basic_user is not None: + kwargs["auth"] = httpx.BasicAuth(self.basic_user, self.basic_password or "") + if self.tls_cert and self.tls_key: + kwargs["cert"] = (self.tls_cert, self.tls_key) + if self.insecure_skip_verify: + kwargs["verify"] = False + elif self.tls_ca: + kwargs["verify"] = self.tls_ca + return kwargs + + +def _build_async_client( + auth: DiagnoseAuth | None, + *, + timeout: float, + max_concurrent: int, +) -> httpx.AsyncClient: + """Construct an ``httpx.AsyncClient`` with auth + a sized connection pool. + + The pool size tracks the parallel-probe semaphore so the pool is never + the implicit bottleneck — same approach as ``source.py``. + """ + kwargs = (auth or DiagnoseAuth()).build_client_kwargs() + kwargs.setdefault("timeout", timeout) + kwargs.setdefault( + "limits", + httpx.Limits(max_connections=max_concurrent, max_keepalive_connections=max_concurrent), + ) + return httpx.AsyncClient(**kwargs) + + +# --------------------------------------------------------------------------- +# Datatypes shared across subcommands +# --------------------------------------------------------------------------- + + +class DiagnoseError(RuntimeError): + """Raised when the diagnose subcommand cannot reach the TSDB or the + response is malformed. Surfaces as exit code 2 to the shell.""" + + +@dataclass(frozen=True) +class LabelsetFamily: + """One distinct labelset shape observed for a forecast metric. + + ``keys`` is the set of label names that appear on every series in + this family (excluding ``__name__``). ``example_labels`` is one + representative series so the operator can see a concrete labelset + next to the abstract key list. ``count`` is how many series fell + into this family — useful for sizing the cleanup work and for + distinguishing the dominant family from a stragglers-only family. + """ + + keys: frozenset[str] + example_labels: dict[str, str] + count: int + looks_scrape_shaped: bool + + +@dataclass +class MetricFinding: + """One forecast metric and the families it appears under. + + Always has at least one family (a metric that returns zero series + is excluded upstream). ``is_duplicate`` is true when the metric is + in the dual-source state — that is, at least one scrape-shaped + family *and* at least one sink-shaped family are both present. + + Intra-source variance (two families that are both scrape-shaped or + both sink-shaped, differing only in some optional label like the + conformal ``calibrated`` flag) is *not* a duplicate. The story this + detector serves is specifically the scrape+sink dual-source trap; + flagging every label-shape change inside one source would drown + real findings under benign noise. + """ + + metric: str + families: list[LabelsetFamily] = field(default_factory=list) + + @property + def is_duplicate(self) -> bool: + scrape_shaped = any(f.looks_scrape_shaped for f in self.families) + sink_shaped = any(not f.looks_scrape_shaped for f in self.families) + return scrape_shaped and sink_shaped + + +@dataclass +class DiagnoseReport: + """All findings from one ``duplicates`` scan, plus metadata for the + report header.""" + + datasource_url: str + metrics_scanned: int + findings: list[MetricFinding] + selector: str | None + since_seconds: int | None + + @property + def duplicates(self) -> list[MetricFinding]: + return [f for f in self.findings if f.is_duplicate] + + @property + def has_duplicates(self) -> bool: + return bool(self.duplicates) + + +@dataclass(frozen=True) +class SampleRateAnomaly: + """One forecast series whose sample count over the window exceeded the cap.""" + + metric: str + labels: dict[str, str] + samples_in_window: int + + +@dataclass +class SampleRateReport: + """All anomalies from one ``sample-rate`` scan plus the scan's parameters.""" + + datasource_url: str + window_seconds: int + max_samples_per_window: int + metrics_scanned: int + series_examined: int + anomalies: list[SampleRateAnomaly] + selector: str | None + + @property + def has_anomalies(self) -> bool: + return bool(self.anomalies) + + +# --------------------------------------------------------------------------- +# Public entry points (sync wrappers for the CLI; tests can call the async +# functions directly with ``pytest.mark.asyncio``) +# --------------------------------------------------------------------------- + + +def run_diagnose_duplicates( + *, + datasource_url: str, + selector: str | None = None, + since_seconds: int | None = None, + auth: DiagnoseAuth | None = None, + out: IO[str] | None = None, + json_output: bool = False, + http_client: httpx.AsyncClient | None = None, + max_concurrent: int = 8, + timeout: float = 30.0, +) -> int: + """Execute the diagnose-duplicates scan and write the report. + + Returns the process exit code (0 / 1 / 2). ``http_client`` is the + seam tests use to inject a mock transport; production callers leave + it ``None`` and a default client is built per invocation. + """ + return asyncio.run( + _run_diagnose_duplicates_async( + datasource_url=datasource_url, + selector=selector, + since_seconds=since_seconds, + auth=auth, + out=out, + json_output=json_output, + http_client=http_client, + max_concurrent=max_concurrent, + timeout=timeout, + ) + ) + + +def run_diagnose_sample_rate( + *, + datasource_url: str, + window_seconds: int, + max_samples_per_window: int, + selector: str | None = None, + auth: DiagnoseAuth | None = None, + out: IO[str] | None = None, + json_output: bool = False, + http_client: httpx.AsyncClient | None = None, + max_concurrent: int = 8, + timeout: float = 30.0, +) -> int: + """Execute the sample-rate scan and write the report. Returns exit code.""" + return asyncio.run( + _run_diagnose_sample_rate_async( + datasource_url=datasource_url, + window_seconds=window_seconds, + max_samples_per_window=max_samples_per_window, + selector=selector, + auth=auth, + out=out, + json_output=json_output, + http_client=http_client, + max_concurrent=max_concurrent, + timeout=timeout, + ) + ) + + +# --------------------------------------------------------------------------- +# Async cores +# --------------------------------------------------------------------------- + + +async def _run_diagnose_duplicates_async( + *, + datasource_url: str, + selector: str | None, + since_seconds: int | None, + auth: DiagnoseAuth | None, + out: IO[str] | None, + json_output: bool, + http_client: httpx.AsyncClient | None, + max_concurrent: int, + timeout: float, +) -> int: + stream: IO[str] = out if out is not None else sys.stdout + err = sys.stderr + + def write(line: str) -> None: + stream.write(line + "\n") + + try: + report = await scan_for_duplicates( + datasource_url=datasource_url, + selector=selector, + since_seconds=since_seconds, + auth=auth, + http_client=http_client, + max_concurrent=max_concurrent, + timeout=timeout, + ) + except DiagnoseError as exc: + err.write(f"error: {exc}\n") + return 2 + + if json_output: + write(json.dumps(_report_to_json(report), indent=2, sort_keys=True)) + else: + _write_text_report(report, write) + + return 1 if report.has_duplicates else 0 + + +async def _run_diagnose_sample_rate_async( + *, + datasource_url: str, + window_seconds: int, + max_samples_per_window: int, + selector: str | None, + auth: DiagnoseAuth | None, + out: IO[str] | None, + json_output: bool, + http_client: httpx.AsyncClient | None, + max_concurrent: int, + timeout: float, +) -> int: + stream: IO[str] = out if out is not None else sys.stdout + err = sys.stderr + + def write(line: str) -> None: + stream.write(line + "\n") + + try: + report = await scan_for_sample_rate( + datasource_url=datasource_url, + window_seconds=window_seconds, + max_samples_per_window=max_samples_per_window, + selector=selector, + auth=auth, + http_client=http_client, + max_concurrent=max_concurrent, + timeout=timeout, + ) + except DiagnoseError as exc: + err.write(f"error: {exc}\n") + return 2 + + if json_output: + write(json.dumps(_sample_rate_report_to_json(report), indent=2, sort_keys=True)) + else: + _write_sample_rate_text_report(report, write) + + return 1 if report.has_anomalies else 0 + + +async def scan_for_duplicates( + *, + datasource_url: str, + selector: str | None, + since_seconds: int | None, + auth: DiagnoseAuth | None = None, + http_client: httpx.AsyncClient | None = None, + max_concurrent: int = 8, + timeout: float = 30.0, +) -> DiagnoseReport: + """Query the TSDB and return the structured finding set. + + Per-metric series probes fan out under a small semaphore so the scan + completes in roughly one TSDB round-trip rather than N. The semaphore + cap matches the TSDB read pool sizing used by ``source.py``. + """ + base = datasource_url.rstrip("/") + owns_client = http_client is None + client = ( + http_client + if http_client is not None + else _build_async_client(auth, timeout=timeout, max_concurrent=max_concurrent) + ) + try: + metric_names = await _list_forecast_metric_names(client, base) + semaphore = asyncio.Semaphore(max_concurrent) + + async def probe(metric: str) -> tuple[str, list[dict[str, str]]]: + async with semaphore: + match = _build_selector(metric, selector) + series = await _list_series(client, base, match, since_seconds=since_seconds) + return metric, series + + results = await asyncio.gather(*[probe(m) for m in metric_names]) if metric_names else [] + findings: list[MetricFinding] = [] + for metric, series in results: + if not series: + continue + families = _group_into_families(series) + findings.append(MetricFinding(metric=metric, families=families)) + return DiagnoseReport( + datasource_url=base, + metrics_scanned=len(metric_names), + findings=findings, + selector=selector, + since_seconds=since_seconds, + ) + finally: + if owns_client: + await client.aclose() + + +async def scan_for_sample_rate( + *, + datasource_url: str, + window_seconds: int, + max_samples_per_window: int, + selector: str | None, + auth: DiagnoseAuth | None = None, + http_client: httpx.AsyncClient | None = None, + max_concurrent: int = 8, + timeout: float = 30.0, +) -> SampleRateReport: + """Probe per-series sample rate to find suspected duplicate ingest. + + Covers the same-labelset case ``scan_for_duplicates`` cannot detect: + when the scrape and sink push samples into the *same* series (identical + labelsets), the only remaining signal is that the series has more + samples per window than one ingest path alone should produce. + + The threshold is operator-tuned because the "expected" rate depends on + ``refresh_interval`` and the forecast step, neither of which is + recoverable from the TSDB. A conservative default + (``max_samples_per_window=5`` over ``window=10m``) catches scrape + contamination (which typically adds 20-40 samples per 10min at a + 15-30s scrape_interval) while leaving slack for the sink's own + multi-sample-per-fit emission pattern. + """ + base = datasource_url.rstrip("/") + owns_client = http_client is None + client = ( + http_client + if http_client is not None + else _build_async_client(auth, timeout=timeout, max_concurrent=max_concurrent) + ) + try: + metric_names = await _list_forecast_metric_names(client, base) + semaphore = asyncio.Semaphore(max_concurrent) + + async def probe(metric: str) -> tuple[str, list[tuple[dict[str, str], int]]]: + async with semaphore: + counts = await _query_count_over_time( + client, base, metric, window_seconds=window_seconds, selector=selector + ) + return metric, counts + + results = await asyncio.gather(*[probe(m) for m in metric_names]) if metric_names else [] + anomalies: list[SampleRateAnomaly] = [] + series_examined = 0 + for metric, counts in results: + for labels, count in counts: + series_examined += 1 + if count > max_samples_per_window: + anomalies.append( + SampleRateAnomaly(metric=metric, labels=labels, samples_in_window=count) + ) + # Stable ordering: by metric, then by descending count, then by labels. + anomalies.sort(key=lambda a: (a.metric, -a.samples_in_window, sorted(a.labels.items()))) + return SampleRateReport( + datasource_url=base, + window_seconds=window_seconds, + max_samples_per_window=max_samples_per_window, + metrics_scanned=len(metric_names), + series_examined=series_examined, + anomalies=anomalies, + selector=selector, + ) + finally: + if owns_client: + await client.aclose() + + +# --------------------------------------------------------------------------- +# HTTP layer +# --------------------------------------------------------------------------- + + +def _is_retryable(exc: BaseException) -> bool: + """Retry transport errors and 5xx; let 4xx (auth, bad query) fail fast. + + Mirrors the same policy ``source.py`` uses for the runtime PromQL + client. Retrying a 401/403 just multiplies the operator-visible + latency before the real error surfaces. + """ + if isinstance(exc, httpx.TransportError): + return True + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code >= _HTTP_5XX_FLOOR + return False + + +async def _get_with_retry( + client: httpx.AsyncClient, + url: str, + *, + params: list[tuple[str, str | int | float | bool | None]] | dict[str, str] | None = None, +) -> httpx.Response: + """GET ``url`` with up to 3 attempts; transient flakes are smoothed over. + + Backoff cap matches ``source.py``: ~0.5/1/2/4s envelope. 4xx is raised + immediately by the inner ``raise_for_status`` and *not* retried because + ``_is_retryable`` returns False for it. + """ + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=0.5, max=4), + retry=retry_if_exception(_is_retryable), + reraise=True, + ): + with attempt: + response = await client.get(url, params=params) + if response.status_code >= _HTTP_5XX_FLOOR: + # Raise an HTTPStatusError so tenacity sees a retryable + # exception, not a successful return with a 5xx body. + response.raise_for_status() + return response + raise DiagnoseError("retries exhausted") # pragma: no cover + + +async def _list_forecast_metric_names(client: httpx.AsyncClient, base: str) -> list[str]: + """Enumerate every metric name matching the forecast regex. + + Uses ``/api/v1/label/__name__/values`` with a server-side ``match[]`` + so the TSDB does the regex filter before responding. On a TSDB with + tens of thousands of metric names, this turns a multi-megabyte body + into a kilobyte-class one. Both Prometheus and VictoriaMetrics + accept ``match[]`` on this endpoint; servers that ignore it fall + back to returning everything and we client-side filter as before. + """ + url = f"{base}/api/v1/label/__name__/values" + # Send the server-side filter as a single ``match[]`` parameter. The + # regex form ``{__name__=~""}`` is preferred over an anchored + # match because some TSDBs match the regex anchored already and adding + # ``^/$`` would result in a double anchor. + params: list[tuple[str, str | int | float | bool | None]] = [ + ("match[]", '{__name__=~"' + FORECAST_METRIC_REGEX + '"}') + ] + try: + response = await _get_with_retry(client, url, params=params) + except httpx.HTTPError as exc: + raise DiagnoseError( + f"datasource unreachable: {url}: {exc.__class__.__name__}: {exc}" + ) from exc + if response.status_code != _HTTP_OK: + raise DiagnoseError( + f"datasource returned HTTP {response.status_code} from {url}: {response.text[:200]}" + ) + payload = _parse_json(response, url) + data = payload.get("data") + if not isinstance(data, list): + raise DiagnoseError(f"unexpected response shape from {url}: missing 'data' list") + pattern = re.compile(f"^{FORECAST_METRIC_REGEX}$") + # Defensive client-side filter: some TSDBs (and proxies) silently + # ignore unknown ``match[]`` params on the label-values endpoint and + # return the full catalogue. The post-filter costs nothing on a + # response the server already trimmed and keeps the result correct + # when it didn't. + return sorted(name for name in data if isinstance(name, str) and pattern.match(name)) + + +async def _list_series( + client: httpx.AsyncClient, + base: str, + match: str, + *, + since_seconds: int | None, +) -> list[dict[str, str]]: + """Fetch all series matching ``match`` and return their label maps.""" + url = f"{base}/api/v1/series" + params: list[tuple[str, str | int | float | bool | None]] = [("match[]", match)] + if since_seconds is not None and since_seconds > 0: + now = int(time.time()) + params.append(("start", str(now - since_seconds))) + params.append(("end", str(now))) + try: + response = await _get_with_retry(client, url, params=params) + except httpx.HTTPError as exc: + raise DiagnoseError( + f"datasource unreachable: {url}: {exc.__class__.__name__}: {exc}" + ) from exc + if response.status_code != _HTTP_OK: + raise DiagnoseError( + f"datasource returned HTTP {response.status_code} from {url}: {response.text[:200]}" + ) + payload = _parse_json(response, url) + data = payload.get("data") + if not isinstance(data, list): + raise DiagnoseError(f"unexpected response shape from {url}: missing 'data' list") + series: list[dict[str, str]] = [] + for entry in data: + if not isinstance(entry, dict): + continue + labels = {k: v for k, v in entry.items() if k != "__name__" and isinstance(v, str)} + series.append(labels) + return series + + +async def _query_count_over_time( + client: httpx.AsyncClient, + base: str, + metric: str, + *, + window_seconds: int, + selector: str | None, +) -> list[tuple[dict[str, str], int]]: + """Run ``count_over_time({}[])`` and return per-series counts. + + The PromQL instant-query result is a vector — one entry per series + matching the selector. Each entry's value is the sample count over + the window for that series. Returns ``(labels_without___name__, count)`` + tuples preserving the TSDB's ordering. + """ + base_selector = _build_selector(metric, selector) + promql = f"count_over_time({base_selector}[{window_seconds}s])" + url = f"{base}/api/v1/query" + try: + response = await _get_with_retry(client, url, params={"query": promql}) + except httpx.HTTPError as exc: + raise DiagnoseError( + f"datasource unreachable: {url}: {exc.__class__.__name__}: {exc}" + ) from exc + if response.status_code != _HTTP_OK: + raise DiagnoseError( + f"datasource returned HTTP {response.status_code} from {url}: {response.text[:200]}" + ) + payload = _parse_json(response, url) + data = payload.get("data") or {} + if not isinstance(data, dict): + raise DiagnoseError(f"unexpected response shape from {url}: 'data' is not an object") + if data.get("resultType") != "vector": + raise DiagnoseError( + f"unexpected resultType from {url}: {data.get('resultType')!r}; expected 'vector'" + ) + result = data.get("result") or [] + if not isinstance(result, list): + return [] + counts: list[tuple[dict[str, str], int]] = [] + for entry in result: + if not isinstance(entry, dict): + continue + metric_labels = entry.get("metric", {}) + if not isinstance(metric_labels, dict): + continue + labels = {k: v for k, v in metric_labels.items() if k != "__name__" and isinstance(v, str)} + value = entry.get("value") + if not isinstance(value, list) or len(value) != _VALUE_PAIR_LEN: + continue + try: + count = int(float(value[1])) + except (TypeError, ValueError): + continue + counts.append((labels, count)) + return counts + + +# --------------------------------------------------------------------------- +# Selector composition + family bucketing +# --------------------------------------------------------------------------- + + +def _build_selector(metric: str, user_selector: str | None) -> str: + """Compose ``{__name__="", }``. + + The ``--selector '{job="foo"}'`` flag is a fully-formed PromQL series + selector; we splice its inner matchers into the metric-name selector + so callers can pre-filter to one cluster / namespace / tenant. The + helper is paranoid about the brace forms so an operator passing + either ``{a="b"}`` or ``a="b"`` (no braces) lands on the same + selector — the second form is what shells often produce when the + user forgot the quoting. + """ + if not user_selector: + return "{" + f'__name__="{metric}"' + "}" + inner = user_selector.strip() + if inner.startswith("{") and inner.endswith("}"): + inner = inner[1:-1].strip() + if not inner: + return "{" + f'__name__="{metric}"' + "}" + return "{" + f'__name__="{metric}",{inner}' + "}" + + +def _group_into_families(series: list[dict[str, str]]) -> list[LabelsetFamily]: + """Bucket series by label-key fingerprint, then summarise each bucket. + + Two series belong to the same family iff they carry the same *set + of label names*, regardless of the values. This catches the canonical + dual-source case where the scraped copy carries ``job``/``instance`` + that the sink-pushed copy doesn't, without false-positiving on + series that vary only in label *values* (e.g. one node having + ``device=sda`` and another having ``device=nvme0n1``). + """ + buckets: dict[frozenset[str], list[dict[str, str]]] = {} + for labels in series: + key = frozenset(labels.keys()) + buckets.setdefault(key, []).append(labels) + families: list[LabelsetFamily] = [] + for keys, members in buckets.items(): + families.append( + LabelsetFamily( + keys=keys, + example_labels=dict(sorted(members[0].items())), + count=len(members), + looks_scrape_shaped=bool(keys & _SCRAPE_MARKER_KEYS), + ) + ) + families.sort(key=lambda f: (-f.count, sorted(f.keys))) + return families + + +def _parse_json(response: httpx.Response, url: str) -> dict[str, Any]: + try: + payload = response.json() + except ValueError as exc: + raise DiagnoseError(f"datasource returned non-JSON body from {url}: {exc}") from exc + if not isinstance(payload, dict): + raise DiagnoseError(f"unexpected response shape from {url}: top-level not an object") + if payload.get("status") != "success": + raise DiagnoseError( + f"datasource returned status={payload.get('status')!r} from {url}: " + f"{payload.get('error') or payload.get('errorType') or ''}" + ) + return payload + + +# --------------------------------------------------------------------------- +# Report rendering — duplicates +# --------------------------------------------------------------------------- + + +def _report_to_json(report: DiagnoseReport) -> dict[str, Any]: + """Stable JSON shape for CI consumption.""" + return { + "datasource_url": report.datasource_url, + "metrics_scanned": report.metrics_scanned, + "selector": report.selector, + "since_seconds": report.since_seconds, + "has_duplicates": report.has_duplicates, + "findings": [ + { + "metric": f.metric, + "is_duplicate": f.is_duplicate, + "families": [ + { + "keys": sorted(fam.keys), + "count": fam.count, + "looks_scrape_shaped": fam.looks_scrape_shaped, + "example_labels": fam.example_labels, + } + for fam in f.families + ], + } + for f in report.findings + ], + } + + +def _write_text_report(report: DiagnoseReport, write: Callable[[str], None]) -> None: + """Human-friendly report. JSON is preferred for CI; text for terminals.""" + write(f"Datasource: {report.datasource_url}") + write(f"Metrics scanned: {report.metrics_scanned}") + if report.selector: + write(f"Selector filter: {report.selector}") + if report.since_seconds: + write(f"Lookback window: {report.since_seconds}s") + write("") + if not report.findings: + write("No forecast metrics found in the datasource.") + return + if not report.has_duplicates: + write( + f"OK no duplicate forecast labelsets detected across {len(report.findings)} metric(s)." + ) + return + write( + f"FAIL {len(report.duplicates)} of {len(report.findings)} forecast metric(s) " + "appear under more than one labelset family." + ) + write("") + for finding in report.duplicates: + write(f" {finding.metric}:") + for fam in finding.families: + tag = "scrape-shaped" if fam.looks_scrape_shaped else "sink-shaped" + write(f" [{tag}] count={fam.count} keys={sorted(fam.keys)}") + write(f" example: {fam.example_labels}") + write("") + _write_remediation(report, write) + + +def _write_remediation(report: DiagnoseReport, write: Callable[[str], None]) -> None: + """Copy-pasteable next steps for an operator on the duplicate path.""" + write("Remediation:") + write("") + write(" 1. Prometheus scrape — drop forecast metrics from the forecaster job.") + write(" Add to the forecaster's scrape_config (externally-managed Prometheus)") + write(" or to the ServiceMonitor (prometheus-operator):") + write("") + write(" metric_relabel_configs:") + write(" - source_labels: [__name__]") + write(" regex: '.+_forecast(_lower|_upper)?'") + write(" action: drop") + write("") + write(" 2. Helm chart values — flip the chart-managed drop on.") + write(" Standalone chart:") + write("") + write(" sink:") + write(" remote_write:") + write(" enabled: true") + write(" serviceMonitor:") + write(" dropForecastSnapshots: true # null = follow sink.enabled") + write("") + write(" 3. Backfill cleanup — once the scrape-drop is in place, delete the") + write(" scrape-shaped historical samples from VictoriaMetrics:") + write("") + for finding in report.duplicates: + scrape_fam = next((f for f in finding.families if f.looks_scrape_shaped), None) + if scrape_fam is None: + continue + selector = _build_delete_selector(finding.metric, scrape_fam) + write( + f" curl -X POST '{report.datasource_url}/api/v1/admin/tsdb/delete_series' " + f"--data-urlencode 'match[]={selector}'" + ) + write("") + write( + " 4. Re-run ``promforecast diagnose duplicates`` to confirm the duplicate " + "labelset family no longer appears." + ) + write("") + write( + " See docs/how-to/migrate-to-sink-first.md for the full migration " + "walkthrough and docs/operations/diagnose.md for this subcommand." + ) + + +# Preferred-first ordering for the delete selector's pin label. +_DELETE_PIN_PREFERENCE: tuple[str, ...] = ( + "job", + "service", + "pod", + "container", + "namespace", + "cluster", + "replica", +) + + +def _build_delete_selector(metric: str, scrape_family: LabelsetFamily) -> str: + """Pin the delete to the scrape-shaped family only. + + Picks the most-specific scrape-marker label observed in this family + so the deletion targets the duplicate without touching the + sink-pushed series or adjacent sink-pushed metrics that happen to + share a wide external_label like ``cluster``. + """ + for key in _DELETE_PIN_PREFERENCE: + if key in scrape_family.keys: + value = scrape_family.example_labels.get(key, "") + return "{" + f'__name__="{metric}",{key}="{value}"' + "}" + return "{" + f'__name__="{metric}"' + "}" + + +# --------------------------------------------------------------------------- +# Report rendering — sample-rate +# --------------------------------------------------------------------------- + + +def _sample_rate_report_to_json(report: SampleRateReport) -> dict[str, Any]: + """Stable JSON shape for CI consumption.""" + return { + "datasource_url": report.datasource_url, + "window_seconds": report.window_seconds, + "max_samples_per_window": report.max_samples_per_window, + "metrics_scanned": report.metrics_scanned, + "series_examined": report.series_examined, + "selector": report.selector, + "has_anomalies": report.has_anomalies, + "anomalies": [ + { + "metric": a.metric, + "labels": a.labels, + "samples_in_window": a.samples_in_window, + } + for a in report.anomalies + ], + } + + +def _write_sample_rate_text_report(report: SampleRateReport, write: Callable[[str], None]) -> None: + """Human-friendly sample-rate report.""" + write(f"Datasource: {report.datasource_url}") + write(f"Metrics scanned: {report.metrics_scanned}") + write(f"Series examined: {report.series_examined}") + write(f"Window: {report.window_seconds}s") + write(f"Per-series cap: {report.max_samples_per_window} samples per window") + if report.selector: + write(f"Selector filter: {report.selector}") + write("") + if report.metrics_scanned == 0: + write("No forecast metrics found in the datasource.") + return + if not report.has_anomalies: + write( + "OK no series exceeded the per-window sample cap " + f"across {report.series_examined} series." + ) + return + write( + f"FAIL {len(report.anomalies)} of {report.series_examined} series " + "exceeded the configured sample-rate cap." + ) + write("") + last_metric: str | None = None + for anomaly in report.anomalies: + if anomaly.metric != last_metric: + write(f" {anomaly.metric}:") + last_metric = anomaly.metric + write( + f" count={anomaly.samples_in_window} labels={dict(sorted(anomaly.labels.items()))}" + ) + write("") + write( + " Likely cause: two ingest paths (snapshot scrape + remote_write sink) " + "writing to the same series. Apply the scrape-drop " + "(``serviceMonitor.dropForecastSnapshots: true`` or the equivalent " + "``metric_relabel_configs`` for externally-managed Prometheus); " + "see docs/how-to/migrate-to-sink-first.md." + ) + + +# Re-exports kept for backwards-compatibility with existing test imports. +# ``_DELETE_PIN_PREFERENCE`` and the dataclasses above are the public-ish +# surface; keeping the underscored helpers importable is a deliberate +# choice that lets test_diagnose.py exercise the bucketing logic without +# spinning up an HTTP transport. None of these are user-facing. +__all__ = [ + "FORECAST_METRIC_REGEX", + "DiagnoseAuth", + "DiagnoseError", + "DiagnoseReport", + "LabelsetFamily", + "MetricFinding", + "SampleRateAnomaly", + "SampleRateReport", + "run_diagnose_duplicates", + "run_diagnose_sample_rate", + "scan_for_duplicates", + "scan_for_sample_rate", +] diff --git a/forecaster/src/promforecast/main.py b/forecaster/src/promforecast/main.py index 276eb49..7150ce2 100644 --- a/forecaster/src/promforecast/main.py +++ b/forecaster/src/promforecast/main.py @@ -18,6 +18,7 @@ from fastapi.responses import JSONResponse, PlainTextResponse from . import config as config_module +from . import diagnose as diagnose_module from . import ha_cache as ha_cache_module from . import preview as preview_module from . import telemetry as telemetry_module @@ -25,7 +26,7 @@ from .cache import MemoryQueryCache, QueryCache, QueryCacheBackend, RedisQueryCache from .exporter import Exporter from .leader import KubernetesLeaseClient, LeaderElector, LeaderElectorConfig, default_identity -from .reload import ConfigMapWatcher, ConfigReloader, ReloadError +from .reload import ConfigMapWatcher, ConfigReloader, ReloadError, log_group_queries_loaded from .runner import Runner from .sink import ForecastSink, RemoteWriteSink from .source import PromSource @@ -53,6 +54,41 @@ def cli() -> None: ) sys.exit(rc) + if args.command == "diagnose": + if args.diagnose_command == "duplicates": + datasource_url = _resolve_diagnose_datasource(args) + auth = _resolve_diagnose_auth(args) + since = _parse_prom_duration(args.since, flag_name="--since") if args.since else None + rc = diagnose_module.run_diagnose_duplicates( + datasource_url=datasource_url, + selector=args.selector, + since_seconds=since, + auth=auth, + json_output=args.json, + max_concurrent=args.max_concurrent, + ) + sys.exit(rc) + if args.diagnose_command == "sample-rate": + datasource_url = _resolve_diagnose_datasource(args) + auth = _resolve_diagnose_auth(args) + window_seconds = _parse_prom_duration(args.window, flag_name="--window") + rc = diagnose_module.run_diagnose_sample_rate( + datasource_url=datasource_url, + window_seconds=window_seconds, + max_samples_per_window=args.max_samples, + selector=args.selector, + auth=auth, + json_output=args.json, + max_concurrent=args.max_concurrent, + ) + sys.exit(rc) + # No diagnose subcommand given → print help and exit non-zero so a + # CI pipeline that mistyped the subcommand fails loudly. + sys.stderr.write( + "error: 'diagnose' requires a subcommand (e.g. 'duplicates' or 'sample-rate')\n" + ) + sys.exit(2) + run(config_path=args.config, log_level=args.log_level) @@ -109,9 +145,275 @@ def _build_parser() -> argparse.ArgumentParser: "when the production URL is unreachable from CI." ), ) + + diagnose_p = sub.add_parser( + "diagnose", + help="Operator diagnostics against the long-term TSDB.", + ) + diagnose_sub = diagnose_p.add_subparsers(dest="diagnose_command") + dup_p = diagnose_sub.add_parser( + "duplicates", + help=( + "Scan the configured TSDB for forecast metrics that appear under " + "more than one labelset family — the dual-source state from " + "running both the /metrics scrape and the remote_write sink at " + "the same time." + ), + ) + _add_diagnose_common_flags(dup_p) + dup_p.add_argument( + "--since", + default=None, + help=( + "Optional lookback window for the series scan as a Prometheus " + "duration (e.g. ``6h``, ``2d``). Without this the TSDB applies " + "its own default (Prometheus: ~2h; VictoriaMetrics: configured " + "retention) — tighten it when a known-clean past window would " + "otherwise drown the live duplicate." + ), + ) + + rate_p = diagnose_sub.add_parser( + "sample-rate", + help=( + "Probe per-series sample rate to find suspected duplicate ingest. " + "Covers the same-labelset case the ``duplicates`` subcommand " + "cannot see from labelset shape alone." + ), + ) + _add_diagnose_common_flags(rate_p) + rate_p.add_argument( + "--window", + default="10m", + help=( + "Backward window for the per-series sample count (Prometheus " + "duration, e.g. ``10m``, ``1h``). Default ``10m``." + ), + ) + rate_p.add_argument( + "--max-samples", + type=_positive_int, + default=5, + help=( + "Per-series sample-count threshold over the window. Any series " + "above this is flagged as likely-duplicate ingestion. Default " + "``5`` — high enough to allow a sink push that emits a curve, " + "low enough that a 15-30s scrape interval (which adds ~20-40 " + "samples per 10min) trips the detector." + ), + ) return parser +def _add_diagnose_common_flags(parser: argparse.ArgumentParser) -> None: + """Apply the flags shared by every ``diagnose`` subcommand. + + Auth + datasource + output knobs are identical across subcommands; + splitting them into a helper avoids drift between subparsers and lets + new subcommands inherit the same surface area for free. + """ + parser.add_argument( + "--datasource", + default=None, + help=( + "Base URL of the long-term TSDB (e.g. http://victoriametrics:8428). " + "Either ``--datasource`` or ``--config`` must be provided; if " + "both are given, ``--datasource`` wins for the URL." + ), + ) + parser.add_argument( + "--config", + default=None, + type=Path, + help=( + "Optional path to a forecaster YAML config. When provided, " + "``datasource.url`` is inherited for the scan; useful in cluster " + "where the operator already has the same YAML mounted into the " + "forecaster pod and doesn't want to retype the URL." + ), + ) + parser.add_argument( + "--selector", + default=None, + help=( + "Optional PromQL series-selector body (e.g. '{cluster=\"prod\"}') " + "spliced into each per-metric lookup so the scan can be pinned " + "to a single tenant / cluster / namespace." + ), + ) + parser.add_argument( + "--json", + action="store_true", + help=( + "Emit the report as machine-readable JSON instead of the " + "human-friendly text format. Stable shape — safe to consume " + "from CI." + ), + ) + parser.add_argument( + "--max-concurrent", + type=_positive_int, + default=8, + help=( + "Cap on simultaneous in-flight TSDB requests. Matches the " + "forecaster's own ``datasource.max_concurrent_requests`` default." + ), + ) + + auth_group = parser.add_argument_group("authentication") + cred_group = auth_group.add_mutually_exclusive_group() + cred_group.add_argument( + "--bearer-token", + default=None, + help=( + "Bearer token sent as ``Authorization: Bearer ``. Falls " + "back to ``$PROMFORECAST_BEARER_TOKEN`` when the flag is omitted " + "so secrets stay off the shell history." + ), + ) + cred_group.add_argument( + "--basic-user", + default=None, + help=( + "Username for HTTP Basic auth. Pair with ``--basic-password`` " + "(or ``$PROMFORECAST_BASIC_PASSWORD``)." + ), + ) + auth_group.add_argument( + "--basic-password", + default=None, + help=("Password for HTTP Basic auth. Falls back to ``$PROMFORECAST_BASIC_PASSWORD``."), + ) + auth_group.add_argument( + "--tls-cert", + default=None, + type=Path, + help="Path to a client certificate for mTLS.", + ) + auth_group.add_argument( + "--tls-key", + default=None, + type=Path, + help="Path to the private key for ``--tls-cert``.", + ) + auth_group.add_argument( + "--tls-ca", + default=None, + type=Path, + help=( + "Path to a custom CA bundle for verifying the TSDB's server " + "certificate. Mutually exclusive with " + "``--tls-insecure-skip-verify``." + ), + ) + auth_group.add_argument( + "--tls-insecure-skip-verify", + action="store_true", + help=( + "Disable TLS verification entirely. Off by default — only flip " + "this on for ad-hoc debugging against a self-signed development " + "endpoint." + ), + ) + + +def _positive_int(value: str) -> int: + """argparse type for ``--max-samples`` / ``--max-concurrent``. + + Raises argparse's own error type so the helpful "argument: invalid + int value: 'foo'" message is consistent with other failing flags. + """ + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"must be a positive integer; got {value!r}") from exc + if parsed <= 0: + raise argparse.ArgumentTypeError(f"must be > 0; got {parsed}") + return parsed + + +def _resolve_diagnose_datasource(args: argparse.Namespace) -> str: + """``--datasource`` wins; otherwise pull ``datasource.url`` from ``--config``. + + Exits non-zero when neither is set so the CLI fails fast with a + targeted error instead of a confusing ``DiagnoseError`` from the + HTTP layer about an empty URL. + """ + if args.datasource: + return str(args.datasource) + if args.config is not None: + try: + cfg = config_module.load(args.config) + except Exception as exc: + raise SystemExit(f"error: failed to load --config {args.config}: {exc}") from exc + return cfg.datasource.url + raise SystemExit("error: one of --datasource or --config must be provided to identify the TSDB") + + +def _resolve_diagnose_auth(args: argparse.Namespace) -> diagnose_module.DiagnoseAuth: + """Build the diagnose auth bundle from flags + env-var fallbacks. + + Env-var precedence is intentionally below CLI flags: explicit + ``--bearer-token`` overrides ``$PROMFORECAST_BEARER_TOKEN`` so a + one-off invocation with a different token doesn't need an + ``unset`` first. + """ + bearer = args.bearer_token or os.environ.get("PROMFORECAST_BEARER_TOKEN") or None + basic_password = args.basic_password or os.environ.get("PROMFORECAST_BASIC_PASSWORD") or None + if args.tls_ca is not None and args.tls_insecure_skip_verify: + raise SystemExit("error: --tls-ca and --tls-insecure-skip-verify are mutually exclusive") + if (args.tls_cert is None) != (args.tls_key is None): + raise SystemExit("error: --tls-cert and --tls-key must be provided together") + return diagnose_module.DiagnoseAuth( + bearer_token=bearer, + basic_user=args.basic_user, + basic_password=basic_password, + tls_cert=str(args.tls_cert) if args.tls_cert is not None else None, + tls_key=str(args.tls_key) if args.tls_key is not None else None, + tls_ca=str(args.tls_ca) if args.tls_ca is not None else None, + insecure_skip_verify=args.tls_insecure_skip_verify, + ) + + +def _load_config_for_boot(config_path: Path) -> config_module.Config: + """Load the YAML, emit the boot-time visibility lines, return the config. + + Two structured log events fire in a deterministic order: + + * ``config_loaded`` with the path and the group count. + * ``group_queries_loaded`` once per group with the query IDs that + were actually loaded — the operator's signal that a partial + bind-mount projection wasn't truncated mid-read (the macOS Docker + Desktop footgun documented in ``docs/operations/reload.md``). + + Extracted from ``run()`` so the ordering is testable without standing + up the full FastAPI + scheduler stack. + """ + cfg = config_module.load(config_path) + logger.info("config_loaded", path=str(config_path), groups=len(cfg.groups)) + log_group_queries_loaded(cfg, source="boot") + return cfg + + +def _parse_prom_duration(value: str, *, flag_name: str = "duration") -> int: + """Resolve a Prometheus-duration string to a positive integer second count. + + Re-uses the parser in :mod:`config` so the CLI accepts the same + ``5m``/``1h``/``14d`` forms as YAML configs. ``flag_name`` is the + user-facing name of the flag that supplied the value, so the error + message can point at the right flag (``--since``, ``--window``, ...). + """ + parsed = config_module.parse_duration(value) + if not hasattr(parsed, "total_seconds"): + raise SystemExit( + f"error: {flag_name} must be a Prometheus duration like '6h' / '2d'; got {value!r}" + ) + seconds = int(parsed.total_seconds()) + if seconds <= 0: + raise SystemExit(f"error: {flag_name} must be a positive duration; got {value!r}") + return seconds + + def run(config_path: Path, log_level: str) -> None: """Boot the forecaster. @@ -119,8 +421,7 @@ def run(config_path: Path, log_level: str) -> None: the leader lease (HA only), starts FastAPI, and schedules per-group runs. """ _configure_logging(log_level) - cfg = config_module.load(config_path) - logger.info("config_loaded", path=str(config_path), groups=len(cfg.groups)) + cfg = _load_config_for_boot(config_path) exporter = Exporter() # Snapshot the SHA-256 of the bytes we just loaded so /metrics carries diff --git a/forecaster/src/promforecast/reload.py b/forecaster/src/promforecast/reload.py index 195b6e9..5246038 100644 --- a/forecaster/src/promforecast/reload.py +++ b/forecaster/src/promforecast/reload.py @@ -130,6 +130,13 @@ async def reload(self) -> ReloadOutcome: changed=result.changed, config_hash=new_hash, ) + # Visibility for partial-config loads: macOS Docker Desktop has + # been observed to project a truncated bind-mount mid-reload, + # leaving the forecaster running fewer queries than the source + # YAML declares. Emitting one line per group with the query + # IDs that were actually loaded lets an operator catch the + # truncation from the logs alone (see docs/operations/reload.md). + log_group_queries_loaded(new_config, source="reload") return ReloadOutcome(result=result, timestamp=now) @@ -251,6 +258,25 @@ def _content_hash(path: Path) -> str | None: return None +def log_group_queries_loaded(config: config_module.Config, *, source: str) -> None: + """Emit one ``group_queries_loaded`` line per group in ``config``. + + ``source`` is the trigger that produced this load (``"boot"`` or + ``"reload"``) so an operator filtering the journal can tell a + fresh-boot truncation apart from a configmap-driven reload truncation. + Empty-query groups still emit a line (with ``queries=[]``) — the + point is to detect *missing* queries, and silence on an empty group + would be ambiguous. + """ + for group in config.groups: + logger.info( + "group_queries_loaded", + group=group.name, + queries=[q.id for q in group.queries], + source=source, + ) + + def _hash_config_file(path: Path) -> str: """Hex SHA-256 of the on-disk config bytes; ``""`` if the file vanishes. diff --git a/forecaster/tests/test_chart_servicemonitor.py b/forecaster/tests/test_chart_servicemonitor.py new file mode 100644 index 0000000..3b9b890 --- /dev/null +++ b/forecaster/tests/test_chart_servicemonitor.py @@ -0,0 +1,435 @@ +"""Chart-template tests for the standalone ``promforecast`` chart's +ServiceMonitor and the umbrella ``promforecast-stack`` sink-first +defaults. + +The standalone chart exposes ``serviceMonitor.dropForecastSnapshots``, +which defaults to ``null``. ``null`` is the "follow the sink toggle" +state: when ``config.sink.remote_write.enabled`` is true the template +renders a ``metricRelabelings`` block dropping the +``.+_forecast(_lower|_upper)?`` snapshot families; when the sink is +off, no relabeling is rendered. A literal ``true`` / ``false`` pins the +behaviour irrespective of the sink. The umbrella chart pre-sets both +the sink and ServiceMonitor toggles so a fresh install lands on +sink+VM with the snapshot drop applied (the v1.5 default). + +These tests shell out to ``helm template`` because the alternative +(implementing a Go template renderer in Python) would diverge from the +real chart behaviour. The cost is a hard dependency on ``helm`` being +on PATH; CI installs it via ``azure/setup-helm`` and the dev environment +has it from ``brew install helm``. Tests skip cleanly when helm is +absent so a contributor without it can still run the rest of the +forecaster suite. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +STANDALONE_CHART = REPO_ROOT / "charts" / "promforecast" +UMBRELLA_CHART = REPO_ROOT / "charts" / "promforecast-stack" + +helm = shutil.which("helm") +pytestmark = pytest.mark.skipif( + helm is None, reason="`helm` not on PATH — chart-template tests are skipped" +) + + +def _umbrella_deps_resolved() -> bool: + """Check whether ``charts/promforecast-stack/charts/`` carries both subcharts. + + A fresh checkout (and the CI ``test`` job, which doesn't run + ``helm dependency update``) starts with the ``charts/`` directory + empty. ``helm template`` then bails with ``found in Chart.yaml, + but missing in charts/ directory``. The fixture below resolves + them lazily; this helper is the cheap idempotency check. + """ + subcharts_dir = UMBRELLA_CHART / "charts" + if not subcharts_dir.is_dir(): + return False + names = {p.name for p in subcharts_dir.iterdir()} + has_vm = any(n.startswith("victoria-metrics-single") and n.endswith(".tgz") for n in names) + has_pf = any(n.startswith("promforecast-") and n.endswith(".tgz") for n in names) + return has_vm and has_pf + + +@pytest.fixture(scope="session", autouse=True) +def _resolve_umbrella_chart_deps() -> None: + """Resolve the umbrella chart's subchart dependencies once per session. + + The CI ``test`` job (pytest) runs without the ``helm dependency + update`` step that ``chart-lint`` runs, so ``helm template + charts/promforecast-stack`` would fail with missing ``victoria- + metrics-single`` / ``promforecast`` subcharts. The chart-template + tests live next to the rest of the forecaster suite, so the + resolution has to happen here rather than be assumed at job-level. + + The VictoriaMetrics chart repo is registered idempotently (helm's + ``repo add`` errors on a pre-existing entry; ``|| true`` is wrong + because we want the real error if it's a different problem, so we + inspect the exit code). The dependency-update itself is cheap on a + warm cache and unconditional so a stale ``charts/*.tgz`` from a + different version always gets refreshed. + """ + if helm is None or _umbrella_deps_resolved(): + return + # Register the upstream repo. ``helm repo add`` exits 0 on a fresh + # add and 1 if the same name is already present with a different + # URL — for our case the URL is fixed, so a non-zero on the + # "already exists" path is benign. We discriminate by checking the + # stderr message rather than swallowing every failure. + repo_add = subprocess.run( + [helm, "repo", "add", "victoria-metrics", "https://victoriametrics.github.io/helm-charts/"], + capture_output=True, + text=True, + check=False, + ) + if repo_add.returncode != 0 and "already exists" not in (repo_add.stderr or ""): + pytest.skip( + "could not register the victoria-metrics helm repo, skipping chart-template tests: " + f"{repo_add.stderr.strip()}" + ) + subprocess.run([helm, "repo", "update"], capture_output=True, text=True, check=False) + subprocess.run( + [helm, "dependency", "update", str(UMBRELLA_CHART)], + capture_output=True, + text=True, + check=True, + ) + + +def _helm_template(chart: Path, *sets: str) -> list[dict]: + """Render ``chart`` with the supplied ``--set`` flags and return docs.""" + cmd = [helm, "template", str(chart)] + for s in sets: + cmd.extend(["--set", s]) + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return [d for d in yaml.safe_load_all(result.stdout) if isinstance(d, dict)] + + +def _find(docs: list[dict], kind: str, *, name_contains: str | None = None) -> dict | None: + """Return the first rendered doc matching ``kind`` (and optional name substring).""" + for d in docs: + if d.get("kind") != kind: + continue + if name_contains is None or name_contains in d.get("metadata", {}).get("name", ""): + return d + return None + + +def _relabelings(servicemonitor: dict) -> list[dict]: + """Return the ServiceMonitor's first endpoint's metricRelabelings list.""" + eps = servicemonitor["spec"]["endpoints"] + assert eps, "ServiceMonitor has no endpoints" + return eps[0].get("metricRelabelings", []) or [] + + +def _has_snapshot_drop(servicemonitor: dict) -> bool: + for r in _relabelings(servicemonitor): + if ( + r.get("action") == "drop" + and r.get("regex") == ".+_forecast(_lower|_upper)?" + and r.get("sourceLabels") == ["__name__"] + ): + return True + return False + + +# --------------------------------------------------------------------------- +# Standalone chart: dropForecastSnapshots derivation +# --------------------------------------------------------------------------- + + +def test_sink_on_renders_snapshot_drop_by_default() -> None: + """Default ``dropForecastSnapshots: null`` + sink on → drop block present.""" + docs = _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "config.sink.remote_write.enabled=true", + ) + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert _has_snapshot_drop(sm), _relabelings(sm) + + +def test_sink_off_omits_snapshot_drop() -> None: + """Default ``dropForecastSnapshots: null`` + sink off → no drop block.""" + docs = _helm_template(STANDALONE_CHART, "serviceMonitor.enabled=true") + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert not _has_snapshot_drop(sm), _relabelings(sm) + + +def test_explicit_false_overrides_sink_on() -> None: + """``dropForecastSnapshots: false`` with sink on → drop block suppressed. + + The opt-out is for the rare case where the snapshot end-of-horizon + gauges are wanted alongside the sink curves (e.g. a parallel-cutover + install verifying the two paths line up before flipping the + scrape-drop on). + """ + docs = _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "config.sink.remote_write.enabled=true", + "serviceMonitor.dropForecastSnapshots=false", + ) + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert not _has_snapshot_drop(sm), _relabelings(sm) + + +def test_explicit_true_overrides_sink_off() -> None: + """``dropForecastSnapshots: true`` with sink off → drop block forced. + + Mirror of the opt-out path: pinning ``true`` keeps the drop in place + even when the sink is off (e.g. an externally-managed sink that the + forecaster does not drive itself). + """ + docs = _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "serviceMonitor.dropForecastSnapshots=true", + ) + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert _has_snapshot_drop(sm), _relabelings(sm) + + +def test_user_supplied_metricrelabelings_compose_with_drop() -> None: + """User-extras are preserved alongside the derived drop block. + + Drop is prepended so the dropped forecast snapshots cannot satisfy a + subsequent ``keep`` rule; user-extras run on whatever survives. + """ + docs = _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "config.sink.remote_write.enabled=true", + "serviceMonitor.metricRelabelings[0].action=keep", + "serviceMonitor.metricRelabelings[0].sourceLabels[0]=__name__", + "serviceMonitor.metricRelabelings[0].regex=foo", + ) + sm = _find(docs, "ServiceMonitor") + assert sm is not None + relabelings = _relabelings(sm) + assert len(relabelings) == 2 + assert relabelings[0]["action"] == "drop" + assert relabelings[1]["action"] == "keep" + assert relabelings[1]["regex"] == "foo" + + +# --------------------------------------------------------------------------- +# Umbrella chart: sink-first default lands the drop block automatically +# --------------------------------------------------------------------------- + + +def test_umbrella_default_install_renders_sink_and_drop() -> None: + """The umbrella's defaults must produce both halves of the sink+VM pair. + + The ConfigMap must carry ``sink.remote_write.enabled: true`` (so the + forecaster pushes the curve) and the rendered ServiceMonitor must + carry the snapshot drop (so the scrape doesn't duplicate the curve + under the forecaster's own labelset). + """ + docs = _helm_template(UMBRELLA_CHART) + # Look up the forecaster's config ConfigMap by data key rather than + # name substring — the umbrella also renders a ``-datasources`` + # ConfigMap whose name contains ``promforecast`` and helm template + # ordering between subcharts is not guaranteed across Helm versions. + cm = next( + (d for d in docs if d.get("kind") == "ConfigMap" and "config.yaml" in d.get("data", {})), + None, + ) + assert cm is not None, "umbrella did not render the forecaster config ConfigMap" + cfg = yaml.safe_load(cm["data"]["config.yaml"]) + assert cfg["sink"]["remote_write"]["enabled"] is True + sm = _find(docs, "ServiceMonitor") + assert sm is not None, "umbrella must render a ServiceMonitor by default" + assert _has_snapshot_drop(sm), _relabelings(sm) + + +def test_umbrella_datasources_default_to_victoriametrics() -> None: + """The bundled Grafana datasource ConfigMap registers VM as default.""" + docs = _helm_template(UMBRELLA_CHART, "datasources.enabled=true") + cm = _find(docs, "ConfigMap", name_contains="datasources") + assert cm is not None + provisioning = yaml.safe_load(cm["data"]["promforecast-datasources.yaml"]) + vm = next(ds for ds in provisioning["datasources"] if ds["name"] == "VictoriaMetrics") + assert vm["isDefault"] is True + assert vm["uid"] == "victoriametrics" + + +def test_umbrella_helm_test_pod_renders() -> None: + """The labelset-uniqueness ``helm test`` hook is present by default. + + The pod queries VM after a real install and asserts that + ``*_forecast`` series exist under exactly one labelset family. CI + does not run ``helm test`` (no kind cluster), but ``helm template`` + + ``kubeconform`` validate that the asset renders to a schema-valid + Pod. + """ + docs = _helm_template(UMBRELLA_CHART) + pod = _find(docs, "Pod", name_contains="test-labelset") + assert pod is not None + annotations = pod["metadata"]["annotations"] + assert annotations.get("helm.sh/hook") == "test" + + +# --------------------------------------------------------------------------- +# ServiceMonitor: jobLabel + honorLabels (release-name agnostic scrape) +# --------------------------------------------------------------------------- + + +def _endpoint(servicemonitor: dict) -> dict: + eps = servicemonitor["spec"]["endpoints"] + assert eps, "ServiceMonitor has no endpoints" + return eps[0] + + +def test_servicemonitor_pins_job_label_to_app_name() -> None: + """``jobLabel: app.kubernetes.io/name`` pins the scrape job to ``promforecast``. + + Without this prometheus-operator generates a release-name-derived + job label (``//``), which makes the example + alerts and the labelset-uniqueness helm-test impossible to write + generically. + """ + docs = _helm_template(STANDALONE_CHART, "serviceMonitor.enabled=true") + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert sm["spec"].get("jobLabel") == "app.kubernetes.io/name" + + +def test_servicemonitor_honors_source_labels() -> None: + """``honorLabels: true`` preserves source ``instance`` on scraped metrics. + + The forecaster exposes its operational and derived ``*_forecast_*`` + families with the underlying signal's source labels (e.g. + ``instance="node-exporter:9100"``). Without ``honorLabels`` the + scrape would clobber that with the forecaster pod's own address, + breaking the example alerts that interpolate ``{{ $labels.instance }}`` + to identify the underlying host. + """ + docs = _helm_template(STANDALONE_CHART, "serviceMonitor.enabled=true") + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert _endpoint(sm).get("honorLabels") is True + + +# --------------------------------------------------------------------------- +# Standalone: existingConfigMap forces an explicit dropForecastSnapshots +# --------------------------------------------------------------------------- + + +def test_existing_configmap_without_explicit_drop_fails_render() -> None: + """GitOps ``existingConfigMap`` users must set ``dropForecastSnapshots``. + + With config managed out-of-band the chart cannot read the sink toggle + to derive the drop. Silently defaulting to off would land the user + in the dual-source state; we fail the render and force a decision. + """ + with pytest.raises(subprocess.CalledProcessError) as exc: + _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "existingConfigMap=external-config", + ) + stderr = exc.value.stderr or "" + assert "dropForecastSnapshots must be set explicitly" in stderr, stderr + + +def test_existing_configmap_with_explicit_drop_renders() -> None: + """Setting ``dropForecastSnapshots`` explicitly unblocks the GitOps path.""" + docs = _helm_template( + STANDALONE_CHART, + "serviceMonitor.enabled=true", + "existingConfigMap=external-config", + "serviceMonitor.dropForecastSnapshots=true", + ) + sm = _find(docs, "ServiceMonitor") + assert sm is not None + assert _has_snapshot_drop(sm), _relabelings(sm) + + +# --------------------------------------------------------------------------- +# Umbrella: release-name-agnostic VM service URL +# --------------------------------------------------------------------------- + + +def test_umbrella_vm_service_name_is_release_name_agnostic() -> None: + """The pinned ``victoria-metrics-single.fullnameOverride`` keeps the VM + Service name stable regardless of Helm release name, so the + forecaster's hardcoded ``datasource.url`` / ``sink.remote_write.url`` + resolve in any install. Without the pin, ``helm install foo`` would + create ``foo-victoria-metrics-single-server`` but the config would + still point at ``promforecast-stack-victoria-metrics-single-server``. + """ + for release in ("promforecast-stack", "foo", "team-prod"): + docs = _helm_template(UMBRELLA_CHART) # baseline + vm = next( + ( + d + for d in docs + if d.get("kind") == "Service" and "victoria-metrics-single" in d["metadata"]["name"] + ), + None, + ) + assert vm is not None + assert vm["metadata"]["name"] == "promforecast-stack-victoria-metrics-single-server", ( + f"release={release}: VM service name drifted to {vm['metadata']['name']}" + ) + + +# --------------------------------------------------------------------------- +# Umbrella: datasources.victoriaMetrics.isDefault is overridable +# --------------------------------------------------------------------------- + + +def test_umbrella_datasources_isdefault_overridable() -> None: + """Users with a Grafana that already has a default datasource (e.g. + kube-prometheus-stack registering Prometheus) need to flip VM's + ``isDefault`` off without losing the chart-provisioned VM entry. + """ + docs = _helm_template( + UMBRELLA_CHART, + "datasources.enabled=true", + "datasources.victoriaMetrics.isDefault=false", + ) + cm = _find(docs, "ConfigMap", name_contains="datasources") + assert cm is not None + provisioning = yaml.safe_load(cm["data"]["promforecast-datasources.yaml"]) + vm = next(ds for ds in provisioning["datasources"] if ds["name"] == "VictoriaMetrics") + assert vm["isDefault"] is False + assert vm["uid"] == "victoriametrics" + + +# --------------------------------------------------------------------------- +# Umbrella: helm test pod has the first-refresh poll loop +# --------------------------------------------------------------------------- + + +def test_umbrella_helm_test_pod_polls_for_first_refresh() -> None: + """The helm-test probe must tolerate the post-install delay before the + first scheduled refresh lands sink data in VM. A naive one-shot + assertion would fail every fresh install because ``helm test`` runs + immediately after ``helm install``, before the forecaster has + completed its first fit. + """ + docs = _helm_template(UMBRELLA_CHART) + pod = _find(docs, "Pod", name_contains="test-labelset") + assert pod is not None + container = pod["spec"]["containers"][0] + env = {e["name"]: e.get("value") for e in container.get("env", [])} + assert env.get("FIRST_REFRESH_TIMEOUT_SECONDS"), "poll timeout env var missing" + script = container["command"][-1] + assert "while :" in script, "helm test pod missing the poll loop" + assert "FIRST_REFRESH_TIMEOUT_SECONDS" in script + # Fail-fast on the dup case (no waiting) — but wait for sink presence. + assert "exit 1" in script + assert "sleep 5" in script diff --git a/forecaster/tests/test_diagnose.py b/forecaster/tests/test_diagnose.py new file mode 100644 index 0000000..5f8b1e8 --- /dev/null +++ b/forecaster/tests/test_diagnose.py @@ -0,0 +1,1069 @@ +"""Tests for ``promforecast diagnose``. + +The detection logic is the load-bearing part: a clean install must return +exit code 0, a dual-source install must return exit code 1 and name every +duplicated metric, and an unreachable datasource must return exit code 2. + +The TSDB is mocked at the HTTP layer with an ``httpx.MockTransport`` so +the tests run offline and against a deterministic series catalogue. The +transport works against both ``httpx.Client`` and ``httpx.AsyncClient``, +which lets us exercise the async diagnose internals from sync tests. +""" + +from __future__ import annotations + +import io +import json +from collections.abc import Callable +from typing import Any + +import httpx +import pytest + +from promforecast.diagnose import ( + DiagnoseAuth, + DiagnoseError, + LabelsetFamily, + _build_delete_selector, + _build_selector, + _group_into_families, + run_diagnose_duplicates, + run_diagnose_sample_rate, + scan_for_duplicates, + scan_for_sample_rate, +) + + +def _make_transport(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.AsyncClient: + """Build an async httpx client whose every request goes through ``handler``.""" + return httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + +def _ok(data: Any) -> httpx.Response: + return httpx.Response(200, json={"status": "success", "data": data}) + + +def _vector(entries: list[dict[str, Any]]) -> httpx.Response: + """Build an instant-query vector response (``resultType: vector``).""" + return httpx.Response( + 200, json={"status": "success", "data": {"resultType": "vector", "result": entries}} + ) + + +# --------------------------------------------------------------------------- +# Family bucketing — the heart of the detector +# --------------------------------------------------------------------------- + + +def test_group_into_families_distinguishes_sink_and_scrape_shapes() -> None: + """Series with different label keys belong to different families.""" + series = [ + # Sink-pushed copies: lean labelset. + {"group": "node_capacity", "model": "AutoARIMA", "instance": "node-a:9100"}, + {"group": "node_capacity", "model": "AutoARIMA", "instance": "node-b:9100"}, + # Scraped copies: carry the scrape-job marker. + { + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "node-a:9100", + "job": "promforecast", + }, + ] + families = _group_into_families(series) + assert len(families) == 2 + # Sorted by descending count → the two-member sink family comes first. + assert families[0].count == 2 + assert not families[0].looks_scrape_shaped + assert families[1].count == 1 + assert families[1].looks_scrape_shaped + + +def test_group_into_families_treats_value_only_variation_as_one_family() -> None: + """``device=sda`` vs ``device=nvme0n1`` is one family — same label keys.""" + series = [ + {"instance": "a", "device": "sda"}, + {"instance": "b", "device": "nvme0n1"}, + ] + families = _group_into_families(series) + assert len(families) == 1 + assert families[0].count == 2 + + +# --------------------------------------------------------------------------- +# Selector composition +# --------------------------------------------------------------------------- + + +def test_build_selector_without_user_filter() -> None: + assert _build_selector("foo_forecast", None) == '{__name__="foo_forecast"}' + + +def test_build_selector_splices_braced_user_selector() -> None: + assert ( + _build_selector("foo_forecast", '{cluster="prod"}') + == '{__name__="foo_forecast",cluster="prod"}' + ) + + +def test_build_selector_tolerates_unbraced_user_selector() -> None: + """Shells often eat the braces; accept either form.""" + assert ( + _build_selector("foo_forecast", 'cluster="prod"') + == '{__name__="foo_forecast",cluster="prod"}' + ) + + +def test_build_selector_ignores_empty_braces() -> None: + assert _build_selector("foo_forecast", "{}") == '{__name__="foo_forecast"}' + + +# --------------------------------------------------------------------------- +# Delete-selector composition +# --------------------------------------------------------------------------- + + +def test_build_delete_selector_prefers_job_over_external_labels() -> None: + """The delete selector must prefer ``job`` over broader external_labels.""" + family = LabelsetFamily( + keys=frozenset({"job", "cluster", "replica", "instance", "group"}), + example_labels={ + "cluster": "prod", + "group": "node_capacity", + "instance": "node-a:9100", + "job": "promforecast", + "replica": "prom-0", + }, + count=10, + looks_scrape_shaped=True, + ) + selector = _build_delete_selector("node_cpu_busy_pct_forecast", family) + assert "job=" in selector and "promforecast" in selector + assert "cluster=" not in selector + assert "replica=" not in selector + + +def test_build_delete_selector_pins_scrape_marker() -> None: + family = LabelsetFamily( + keys=frozenset({"job", "instance", "group", "model"}), + example_labels={ + "group": "node_capacity", + "instance": "forecaster:9091", + "job": "promforecast", + "model": "AutoARIMA", + }, + count=12, + looks_scrape_shaped=True, + ) + selector = _build_delete_selector("node_cpu_busy_pct_forecast", family) + assert selector == '{__name__="node_cpu_busy_pct_forecast",job="promforecast"}' + + +# --------------------------------------------------------------------------- +# End-to-end: clean install +# --------------------------------------------------------------------------- + + +def test_run_diagnose_returns_zero_on_clean_install() -> None: + """Sink-only install: every metric has exactly one labelset family.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["node_cpu_busy_pct_forecast", "node_cpu_busy_pct_forecast_upper"]) + if request.url.path == "/api/v1/series": + return _ok( + [ + { + "__name__": "node_cpu_busy_pct_forecast", + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "node-a:9100", + }, + { + "__name__": "node_cpu_busy_pct_forecast", + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "node-b:9100", + }, + ] + ) + return httpx.Response(404) + + client = _make_transport(handler) + out = io.StringIO() + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428/", + out=out, + http_client=client, + ) + assert rc == 0 + rendered = out.getvalue() + assert "no duplicate forecast labelsets detected" in rendered + # The report header includes the canonicalised datasource URL (no + # trailing slash) so a copy-pasted remediation snippet matches. + assert "http://vm:8428" in rendered + + +# --------------------------------------------------------------------------- +# End-to-end: dual-source install +# --------------------------------------------------------------------------- + + +def test_run_diagnose_returns_one_when_dual_source() -> None: + """The scrape + sink dual-source state must trip exit code 1.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["node_cpu_busy_pct_forecast", "node_load1_forecast"]) + if request.url.path == "/api/v1/series": + match = request.url.params.get("match[]") or "" + if "node_cpu_busy_pct_forecast" in match: + return _ok( + [ + # Sink copies. + { + "__name__": "node_cpu_busy_pct_forecast", + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "node-a:9100", + }, + # Scrape copies — extra ``job`` and changed ``instance``. + { + "__name__": "node_cpu_busy_pct_forecast", + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "forecaster:9091", + "job": "promforecast", + }, + ] + ) + return _ok( + [ + { + "__name__": "node_load1_forecast", + "group": "node_capacity", + "model": "AutoARIMA", + "instance": "node-a:9100", + } + ] + ) + return httpx.Response(404) + + client = _make_transport(handler) + out = io.StringIO() + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428/", + out=out, + http_client=client, + ) + assert rc == 1 + body = out.getvalue() + assert "node_cpu_busy_pct_forecast" in body + assert "node_load1_forecast" not in body.split("Remediation:")[0] + assert "metric_relabel_configs" in body + assert "/api/v1/admin/tsdb/delete_series" in body + assert 'job="promforecast"' in body + + +def test_run_diagnose_ignores_intra_source_label_variance() -> None: + """Two sink-shaped families differing only by an optional label are not + a duplicate. Conformal calibration legitimately adds a ``calibrated`` + label to some emitted rows but not others — this is intra-source + variance, not the dual-source state the detector targets.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast_upper"]) + if request.url.path == "/api/v1/series": + return _ok( + [ + {"__name__": "foo_forecast_upper", "group": "g", "level": "80"}, + { + "__name__": "foo_forecast_upper", + "group": "g", + "level": "80", + "calibrated": "true", + }, + ] + ) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=out, + http_client=_make_transport(handler), + ) + assert rc == 0, out.getvalue() + assert "no duplicate forecast labelsets detected" in out.getvalue() + + +def test_run_diagnose_json_output_is_stable_and_machine_parseable() -> None: + """JSON output is the CI integration point — assert the shape directly.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/series": + return _ok( + [ + {"__name__": "foo_forecast", "group": "g", "instance": "a"}, + {"__name__": "foo_forecast", "group": "g", "instance": "b", "job": "x"}, + ] + ) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + json_output=True, + out=out, + http_client=_make_transport(handler), + ) + assert rc == 1 + payload = json.loads(out.getvalue()) + assert payload["has_duplicates"] is True + finding = payload["findings"][0] + assert finding["metric"] == "foo_forecast" + assert finding["is_duplicate"] is True + assert len(finding["families"]) == 2 + assert any(fam["looks_scrape_shaped"] for fam in finding["families"]) + + +# --------------------------------------------------------------------------- +# Filter knobs +# --------------------------------------------------------------------------- + + +def test_run_diagnose_passes_selector_through_to_tsdb() -> None: + """``--selector`` is spliced into every per-metric series lookup.""" + observed_matches: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/series": + observed_matches.append(request.url.params.get("match[]") or "") + return _ok([]) + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + selector='{cluster="prod"}', + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + assert observed_matches == ['{__name__="foo_forecast",cluster="prod"}'] + + +def test_run_diagnose_passes_since_window_as_start_end() -> None: + """``--since 6h`` is converted to start/end query params.""" + observed_params: list[dict[str, str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/series": + observed_params.append(dict(request.url.params)) + return _ok([]) + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + since_seconds=3600, + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + assert "start" in observed_params[0] + assert "end" in observed_params[0] + assert int(observed_params[0]["end"]) - int(observed_params[0]["start"]) == 3600 + + +# --------------------------------------------------------------------------- +# Server-side ``match[]`` filter on ``/__name__/values`` +# --------------------------------------------------------------------------- + + +def test_label_values_request_carries_server_side_match_filter() -> None: + """The label-values request must include the ``match[]`` regex. + + Pushes the forecast-metric filter to the server so a TSDB with tens + of thousands of metric names returns a kilobyte-class response rather + than the full catalogue. + """ + observed_label_values_params: list[dict[str, str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + observed_label_values_params.append(dict(request.url.params)) + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/series": + return _ok([]) + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + assert observed_label_values_params, "label-values endpoint was never called" + match = observed_label_values_params[0].get("match[]") or "" + assert "__name__" in match and "_forecast" in match + + +def test_label_values_falls_back_to_client_side_filter_when_server_ignores_match() -> None: + """If the TSDB ignores ``match[]`` and returns the full catalogue, the + client-side regex must still keep only forecast metrics.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok( + [ + "node_cpu_seconds_total", + "node_cpu_busy_pct_forecast", + "forecast_quality_score", + "forecast_failures_total", + ] + ) + if request.url.path == "/api/v1/series": + return _ok([]) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=out, + http_client=_make_transport(handler), + json_output=True, + ) + assert rc == 0 + payload = json.loads(out.getvalue()) + # Only ``node_cpu_busy_pct_forecast`` matches; the operational + # ``forecast_*`` metrics and the unrelated ``node_cpu_seconds_total`` + # are correctly filtered out by the client-side fallback. + assert payload["metrics_scanned"] == 1 + + +# --------------------------------------------------------------------------- +# Error paths +# --------------------------------------------------------------------------- + + +def test_run_diagnose_returns_two_on_unreachable_datasource() -> None: + def handler(_: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused") + + rc = run_diagnose_duplicates( + datasource_url="http://nowhere:8428", + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 2 + + +@pytest.mark.asyncio +async def test_scan_raises_on_non_success_status() -> None: + """VM/Prometheus returns ``status: error`` on bad selectors.""" + + def handler(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"status": "error", "errorType": "bad", "error": "nope"}) + + with pytest.raises(DiagnoseError): + await scan_for_duplicates( + datasource_url="http://vm:8428", + selector=None, + since_seconds=None, + http_client=_make_transport(handler), + ) + + +# --------------------------------------------------------------------------- +# Retry / backoff on transient failures +# --------------------------------------------------------------------------- + + +def test_label_values_retries_on_transient_5xx_and_then_succeeds() -> None: + """A single 5xx response is smoothed over by the retry layer. + + Matches the behaviour of ``source.py``'s runtime PromQL client: 5xx + and transport errors retry with exponential backoff up to 3 attempts; + 4xx raises immediately. Without this, a momentary TSDB hiccup during + a diagnose run flips the exit code to ``2`` even though everything is + fine seconds later. + """ + attempts = {"count": 0} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + attempts["count"] += 1 + if attempts["count"] == 1: + return httpx.Response(503, text="overloaded") + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/series": + return _ok([]) + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + # Second attempt is the success; 3 attempts is the cap so anything + # between 2 and 3 is acceptable here. + assert attempts["count"] >= 2 + + +def test_4xx_fails_fast_without_retry() -> None: + """4xx (auth, bad query) is *not* retried — operator must see the real error.""" + attempts = {"count": 0} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + attempts["count"] += 1 + return httpx.Response(401, text="unauthorized") + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 2 + assert attempts["count"] == 1 + + +# --------------------------------------------------------------------------- +# Auth +# --------------------------------------------------------------------------- + + +def test_bearer_token_attaches_authorization_header() -> None: + """``DiagnoseAuth(bearer_token=...)`` propagates as a Bearer header.""" + observed_auth: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + observed_auth.append(request.headers.get("Authorization")) + if request.url.path == "/api/v1/label/__name__/values": + return _ok([]) + return httpx.Response(404) + + # Build the auth via the dataclass, materialise the kwargs into the + # transport-backed client manually so the test exercises exactly the + # same path production uses (auth → kwargs → AsyncClient). + auth = DiagnoseAuth(bearer_token="abc123") + client = httpx.AsyncClient(transport=httpx.MockTransport(handler), **auth.build_client_kwargs()) + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=client, + ) + assert rc == 0 + assert observed_auth and observed_auth[0] == "Bearer abc123" + + +def test_basic_auth_attaches_basic_credentials() -> None: + """``DiagnoseAuth(basic_user=..., basic_password=...)`` propagates as Basic auth.""" + observed_auth: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + observed_auth.append(request.headers.get("Authorization")) + if request.url.path == "/api/v1/label/__name__/values": + return _ok([]) + return httpx.Response(404) + + auth = DiagnoseAuth(basic_user="alice", basic_password="s3cret") + client = httpx.AsyncClient(transport=httpx.MockTransport(handler), **auth.build_client_kwargs()) + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=client, + ) + assert rc == 0 + assert observed_auth and observed_auth[0] is not None + assert observed_auth[0].startswith("Basic ") + + +def test_diagnose_auth_without_credentials_attaches_no_authorization() -> None: + """The plain-HTTP path must not attach an Authorization header. + + A spurious header would be an information leak to operators expecting + "no auth means no auth" in the dev stack. + """ + observed_auth: list[str | None] = [] + + def handler(request: httpx.Request) -> httpx.Response: + observed_auth.append(request.headers.get("Authorization")) + if request.url.path == "/api/v1/label/__name__/values": + return _ok([]) + return httpx.Response(404) + + auth = DiagnoseAuth() + kwargs = auth.build_client_kwargs() + assert "auth" not in kwargs + assert "headers" not in kwargs + client = httpx.AsyncClient(transport=httpx.MockTransport(handler), **kwargs) + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=client, + ) + assert rc == 0 + assert observed_auth and observed_auth[0] is None + + +def test_diagnose_auth_tls_knobs_resolve_to_httpx_kwargs() -> None: + """TLS flags map onto ``cert`` / ``verify`` as httpx expects.""" + auth = DiagnoseAuth(tls_cert="/tmp/c.pem", tls_key="/tmp/k.pem", tls_ca="/tmp/ca.pem") + kwargs = auth.build_client_kwargs() + assert kwargs["cert"] == ("/tmp/c.pem", "/tmp/k.pem") + assert kwargs["verify"] == "/tmp/ca.pem" + + skip = DiagnoseAuth(insecure_skip_verify=True, tls_ca="/tmp/ca.pem") + # ``insecure_skip_verify`` takes precedence inside the dataclass; the + # CLI layer enforces mutual exclusion, but the dataclass picks the + # safer behaviour ("skip" wins over a partial CA-only setup). + assert skip.build_client_kwargs()["verify"] is False + + +# --------------------------------------------------------------------------- +# Parallel fan-out +# --------------------------------------------------------------------------- + + +def test_per_metric_probes_run_in_parallel() -> None: + """Many forecast metrics → many concurrent series probes. + + Implementation detail check: each metric's ``/api/v1/series`` probe + runs under a shared semaphore. Concretely the assertion is that all + expected metrics were probed in a single scan — the parallelism + itself is hard to assert deterministically without timing. + """ + metric_names = [f"m{i}_forecast" for i in range(12)] + seen: set[str] = set() + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(metric_names) + if request.url.path == "/api/v1/series": + match = request.url.params.get("match[]") or "" + for name in metric_names: + if name in match: + seen.add(name) + break + return _ok([]) + return httpx.Response(404) + + rc = run_diagnose_duplicates( + datasource_url="http://vm:8428", + out=io.StringIO(), + http_client=_make_transport(handler), + max_concurrent=4, + ) + assert rc == 0 + assert seen == set(metric_names) + + +# --------------------------------------------------------------------------- +# Sample-rate subcommand +# --------------------------------------------------------------------------- + + +def test_sample_rate_clean_install_returns_zero() -> None: + """Series count below the cap → no anomalies → exit code 0.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + return _vector( + [ + { + "metric": {"__name__": "foo_forecast", "group": "g", "instance": "a"}, + "value": [1700000000, "2"], + } + ] + ) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + out=out, + http_client=_make_transport(handler), + ) + assert rc == 0 + assert "no series exceeded" in out.getvalue() + + +def test_sample_rate_above_cap_returns_one_and_names_series() -> None: + """Series count above the cap → flagged → exit code 1.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + return _vector( + [ + { + "metric": {"__name__": "foo_forecast", "group": "g", "instance": "a"}, + "value": [1700000000, "40"], # well above cap=5 + }, + { + "metric": {"__name__": "foo_forecast", "group": "g", "instance": "b"}, + "value": [1700000000, "2"], # below cap, not flagged + }, + ] + ) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + out=out, + http_client=_make_transport(handler), + ) + assert rc == 1 + body = out.getvalue() + assert "foo_forecast" in body + assert "instance" in body and "'a'" in body + # The clean series must not be named in the anomaly listing. + assert "'b'" not in body.split("Likely cause")[0] + + +def test_sample_rate_json_output_has_stable_shape() -> None: + """JSON shape is the CI integration point; assert key fields exist.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + return _vector( + [ + { + "metric": {"__name__": "foo_forecast", "group": "g"}, + "value": [1700000000, "99"], + } + ] + ) + return httpx.Response(404) + + out = io.StringIO() + rc = run_diagnose_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + json_output=True, + out=out, + http_client=_make_transport(handler), + ) + assert rc == 1 + payload = json.loads(out.getvalue()) + assert payload["has_anomalies"] is True + assert payload["window_seconds"] == 600 + assert payload["max_samples_per_window"] == 5 + assert payload["anomalies"][0]["metric"] == "foo_forecast" + assert payload["anomalies"][0]["samples_in_window"] == 99 + + +def test_sample_rate_query_uses_count_over_time_with_window() -> None: + """The probe must use ``count_over_time([])``.""" + observed_queries: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + observed_queries.append(request.url.params.get("query") or "") + return _vector([]) + return httpx.Response(404) + + rc = run_diagnose_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + assert observed_queries == ['count_over_time({__name__="foo_forecast"}[600s])'] + + +def test_sample_rate_splices_user_selector() -> None: + """``--selector '{cluster="prod"}'`` is forwarded into the PromQL query.""" + observed_queries: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + observed_queries.append(request.url.params.get("query") or "") + return _vector([]) + return httpx.Response(404) + + rc = run_diagnose_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + selector='{cluster="prod"}', + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 0 + assert observed_queries == ['count_over_time({__name__="foo_forecast",cluster="prod"}[600s])'] + + +def test_sample_rate_returns_two_on_unreachable_datasource() -> None: + def handler(_: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused") + + rc = run_diagnose_sample_rate( + datasource_url="http://nowhere:8428", + window_seconds=600, + max_samples_per_window=5, + out=io.StringIO(), + http_client=_make_transport(handler), + ) + assert rc == 2 + + +@pytest.mark.asyncio +async def test_sample_rate_raises_on_non_vector_result_type() -> None: + """``count_over_time`` must return a vector; a matrix/scalar means the + query was malformed in a way the operator should see.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/label/__name__/values": + return _ok(["foo_forecast"]) + if request.url.path == "/api/v1/query": + return httpx.Response( + 200, + json={"status": "success", "data": {"resultType": "scalar", "result": [1, "1"]}}, + ) + return httpx.Response(404) + + with pytest.raises(DiagnoseError): + await scan_for_sample_rate( + datasource_url="http://vm:8428", + window_seconds=600, + max_samples_per_window=5, + selector=None, + http_client=_make_transport(handler), + ) + + +# --------------------------------------------------------------------------- +# CLI resolver helpers (``main._resolve_diagnose_*``) +# --------------------------------------------------------------------------- +# +# These exercise the argparse → dataclass adapters that live in ``main.py`` +# alongside ``cli()``. They're CLI glue rather than diagnose-internal code, +# but they belong here so all the diagnose-subcommand surface area is tested +# in one file. + + +def _make_args(**overrides: Any) -> Any: + """Build a fake ``argparse.Namespace`` carrying every diagnose CLI flag. + + Defaults match what argparse would produce when no flag is set; tests + override only the fields they care about. Keeping the helper local + avoids importing argparse in every test that needs one of these. + """ + import argparse # noqa: PLC0415 + + defaults = { + "datasource": None, + "config": None, + "bearer_token": None, + "basic_user": None, + "basic_password": None, + "tls_cert": None, + "tls_key": None, + "tls_ca": None, + "tls_insecure_skip_verify": False, + } + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +def test_resolve_datasource_explicit_flag_wins(tmp_path: Any) -> None: + """``--datasource`` overrides ``--config`` when both are given.""" + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_datasource # noqa: PLC0415 + + cfg_yaml = tmp_path / "cfg.yaml" + cfg_yaml.write_text( + "apiVersion: promforecast.io/v1\n" + "datasource:\n" + " url: http://config-source:8428/\n" + "server:\n" + " refresh_interval: 1h\n" + "defaults:\n" + " models: [SeasonalNaive]\n" + "groups: []\n" + ) + args = _make_args(datasource="http://explicit:8428", config=Path(cfg_yaml)) + assert _resolve_diagnose_datasource(args) == "http://explicit:8428" + + +def test_resolve_datasource_inherits_from_config(tmp_path: Any) -> None: + """``--config`` alone resolves to the YAML's ``datasource.url``.""" + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_datasource # noqa: PLC0415 + + cfg_yaml = tmp_path / "cfg.yaml" + cfg_yaml.write_text( + "apiVersion: promforecast.io/v1\n" + "datasource:\n" + " url: http://from-config:8428/\n" + "server:\n" + " refresh_interval: 1h\n" + "defaults:\n" + " models: [SeasonalNaive]\n" + "groups: []\n" + ) + args = _make_args(config=Path(cfg_yaml)) + assert _resolve_diagnose_datasource(args) == "http://from-config:8428/" + + +def test_resolve_datasource_fails_when_neither_provided() -> None: + """Neither ``--datasource`` nor ``--config`` → SystemExit with a clear error.""" + from promforecast.main import _resolve_diagnose_datasource # noqa: PLC0415 + + args = _make_args() + with pytest.raises(SystemExit) as info: + _resolve_diagnose_datasource(args) + assert "one of --datasource or --config" in str(info.value) + + +def test_resolve_datasource_fails_with_targeted_error_on_bad_config(tmp_path: Any) -> None: + """Broken YAML at ``--config`` → SystemExit with the path in the error. + + The operator must see *which* file failed; a bare ``ValidationError`` + from pydantic with no file path is the kind of UX the migration doc + warns against. + """ + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_datasource # noqa: PLC0415 + + cfg_yaml = tmp_path / "broken.yaml" + cfg_yaml.write_text("not: a: valid: yaml: at: all: :\n") + args = _make_args(config=Path(cfg_yaml)) + with pytest.raises(SystemExit) as info: + _resolve_diagnose_datasource(args) + assert "failed to load --config" in str(info.value) + assert "broken.yaml" in str(info.value) + + +def test_resolve_auth_picks_up_bearer_token_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + """``$PROMFORECAST_BEARER_TOKEN`` populates the bearer field when the flag is omitted.""" + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + monkeypatch.setenv("PROMFORECAST_BEARER_TOKEN", "from-env") + auth = _resolve_diagnose_auth(_make_args()) + assert auth.bearer_token == "from-env" + + +def test_resolve_auth_cli_flag_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Explicit ``--bearer-token`` wins over the env var. + + The CLI flag is the per-invocation override; the env var is the + operator's persistent default. Without this precedence a one-off + invocation with a different token would need an ``unset`` first. + """ + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + monkeypatch.setenv("PROMFORECAST_BEARER_TOKEN", "from-env") + auth = _resolve_diagnose_auth(_make_args(bearer_token="from-flag")) + assert auth.bearer_token == "from-flag" + + +def test_resolve_auth_picks_up_basic_password_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + """``$PROMFORECAST_BASIC_PASSWORD`` populates the basic-auth password.""" + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + monkeypatch.setenv("PROMFORECAST_BASIC_PASSWORD", "env-pass") + auth = _resolve_diagnose_auth(_make_args(basic_user="alice")) + assert auth.basic_user == "alice" + assert auth.basic_password == "env-pass" + + +def test_resolve_auth_rejects_tls_ca_with_skip_verify() -> None: + """``--tls-ca`` + ``--tls-insecure-skip-verify`` → SystemExit. + + Combining "use this CA bundle" with "don't verify TLS at all" is a + contradiction the operator should see at argument parse time, not via + a silent precedence rule inside the auth dataclass. + """ + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + args = _make_args(tls_ca=Path("/tmp/ca.pem"), tls_insecure_skip_verify=True) + with pytest.raises(SystemExit) as info: + _resolve_diagnose_auth(args) + assert "--tls-ca" in str(info.value) + assert "--tls-insecure-skip-verify" in str(info.value) + + +def test_resolve_auth_rejects_tls_cert_without_tls_key() -> None: + """``--tls-cert`` without ``--tls-key`` → SystemExit. + + httpx silently accepts a cert-only call and then fails at handshake + time with an obscure error. Catching the half-configuration at parse + time keeps the operator from chasing a TLS error that's really a + missing flag. + """ + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + args = _make_args(tls_cert=Path("/tmp/cert.pem")) + with pytest.raises(SystemExit) as info: + _resolve_diagnose_auth(args) + assert "--tls-cert and --tls-key" in str(info.value) + + +def test_resolve_auth_rejects_tls_key_without_tls_cert() -> None: + """Symmetric case: key without cert is also rejected.""" + from pathlib import Path # noqa: PLC0415 + + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + args = _make_args(tls_key=Path("/tmp/key.pem")) + with pytest.raises(SystemExit) as info: + _resolve_diagnose_auth(args) + assert "--tls-cert and --tls-key" in str(info.value) + + +def test_resolve_auth_returns_empty_bundle_when_no_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No flags + no env vars → all fields default to ``None`` / ``False``. + + The plain-HTTP dev-stack path must not synthesise any credentials + when the operator hasn't asked for them. The env-var fallbacks are + explicitly cleared so the test isn't affected by the developer's + shell environment. + """ + from promforecast.main import _resolve_diagnose_auth # noqa: PLC0415 + + monkeypatch.delenv("PROMFORECAST_BEARER_TOKEN", raising=False) + monkeypatch.delenv("PROMFORECAST_BASIC_PASSWORD", raising=False) + auth = _resolve_diagnose_auth(_make_args()) + assert auth.bearer_token is None + assert auth.basic_user is None + assert auth.basic_password is None + assert auth.tls_cert is None + assert auth.tls_key is None + assert auth.tls_ca is None + assert auth.insecure_skip_verify is False diff --git a/forecaster/tests/test_example_alerts_labelset.py b/forecaster/tests/test_example_alerts_labelset.py new file mode 100644 index 0000000..e4655be --- /dev/null +++ b/forecaster/tests/test_example_alerts_labelset.py @@ -0,0 +1,412 @@ +"""Audit ``examples/alerts/promforecast-rules.yaml`` against the sink+VM +labelset that the v1.5 defaults emit. + +The forecaster ships two write paths for forecast metrics: the +``/metrics`` snapshot scraped by Prometheus (carries the forecaster's own +``instance=":9091"`` and ``job=""``) and the +``sink.remote_write`` push into a long-term TSDB (carries only the source +metric's labels — no forecaster scrape labels). With the umbrella chart +defaulting to sink-on and the standalone chart's +``serviceMonitor.dropForecastSnapshots`` default deriving from +``sink.remote_write.enabled``, the bundled alerts should only filter on +source-side labels. A literal ``instance=":9091"`` matcher would +silently match zero series under the recommended path. + +This test: + +* parses every alert's ``expr``, +* extracts every leaf metric reference and its label matchers, +* asserts no rule filters a ``*_forecast`` family metric on a label that + only the dropped snapshot copy would carry, and +* asserts every referenced metric and matcher resolves to at least one + series in a synthetic sink-shaped labelset fixture. + +It is deliberately a structural audit rather than a full PromQL +evaluator — the goal is to fail loudly when an example rule drifts away +from the labelset the sink actually emits, which is the failure mode +operators hit when they paste a stale example into a real cluster. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +RULES_PATH = REPO_ROOT / "examples" / "alerts" / "promforecast-rules.yaml" + +# --------------------------------------------------------------------------- +# Sink-shaped labelset fixture. +# +# Models a small but representative slice of what the sink+VM path emits +# after a single fit on the bundled dev stack. ``*_forecast`` / +# ``*_forecast_lower`` / ``*_forecast_upper`` carry source labels only +# (``instance`` from node-exporter, ``device`` / ``mountpoint`` for disk +# series). Operational metrics emitted only on /metrics (``forecast_*``, +# ``*_forecast_time_to_threshold_seconds``, ``*_forecast_growth_rate``) +# carry both the source labels (where applicable) and the forecaster's +# own scrape labels (``job="promforecast"``, ``instance="forecaster:9091"``). +# A rule that filters on labels the sink does not project will return +# zero series under the recommended path; this fixture catches that drift +# at PR time. +# --------------------------------------------------------------------------- + +_SINK_FORECAST_LABELS = [ + { + "id": "node_cpu_busy_pct", + "group": "node_capacity", + "model": "AutoARIMA", + "horizon": "next", + "instance": "node-exporter:9100", + }, + { + "id": "node_memory_used_bytes", + "group": "node_capacity", + "model": "SeasonalNaive", + "horizon": "next", + "instance": "node-exporter:9100", + }, + { + "id": "node_filesystem_avail_bytes", + "group": "node_storage", + "model": "AutoARIMA", + "horizon": "next", + "instance": "node-exporter:9100", + "mountpoint": "/var", + "device": "sda1", + "fstype": "ext4", + }, +] + +_SNAPSHOT_OP_LABELS = { + "instance": "forecaster:9091", + "job": "promforecast", +} + + +def _with(labels: dict[str, str], **extra: str) -> dict[str, str]: + out = dict(labels) + out.update(extra) + return out + + +def _expand_levels(base: list[dict[str, str]], levels: tuple[str, ...]) -> list[dict[str, str]]: + """Replicate a base labelset across the configured confidence levels.""" + return [_with(b, level=lvl) for b in base for lvl in levels] + + +LEVELS = ("80", "95") +THRESHOLD_NAMES = ("warn", "critical") +COMPARATORS = ("lt", "gt") + +FIXTURE: dict[str, list[dict[str, str]]] = { + # Sink-emitted forecast curves: source labels only, no forecaster + # scrape labels. + "node_cpu_busy_pct_forecast": _SINK_FORECAST_LABELS[:1] + _SINK_FORECAST_LABELS[1:2], + "node_memory_used_bytes_forecast": _SINK_FORECAST_LABELS[1:2], + "node_filesystem_avail_bytes_forecast": _SINK_FORECAST_LABELS[2:3], + # Bands at every configured level. + "node_cpu_busy_pct_forecast_lower": _expand_levels(_SINK_FORECAST_LABELS[:2], LEVELS), + "node_cpu_busy_pct_forecast_upper": _expand_levels(_SINK_FORECAST_LABELS[:2], LEVELS), + "node_memory_used_bytes_forecast_lower": _expand_levels(_SINK_FORECAST_LABELS[1:2], LEVELS), + "node_memory_used_bytes_forecast_upper": _expand_levels(_SINK_FORECAST_LABELS[1:2], LEVELS), + "node_filesystem_avail_bytes_forecast_lower": _expand_levels( + _SINK_FORECAST_LABELS[2:3], LEVELS + ), + "node_filesystem_avail_bytes_forecast_upper": _expand_levels( + _SINK_FORECAST_LABELS[2:3], LEVELS + ), + # Snapshot-only operational metrics carry forecaster scrape labels. + "forecast_last_run_timestamp_seconds": [ + _with(_SNAPSHOT_OP_LABELS, group="node_capacity"), + _with(_SNAPSHOT_OP_LABELS, group="node_storage"), + ], + "forecast_accuracy_mape": [ + _with( + _SNAPSHOT_OP_LABELS, + group="node_capacity", + id="node_cpu_busy_pct", + model="AutoARIMA", + horizon="24h", + ), + _with( + _SNAPSHOT_OP_LABELS, + group="node_storage", + id="node_filesystem_avail_bytes", + model="AutoARIMA", + horizon="24h", + ), + ], + "forecast_accuracy_mase": [ + _with( + _SNAPSHOT_OP_LABELS, + group="node_capacity", + id="node_cpu_busy_pct", + model="AutoARIMA", + horizon="24h", + ), + ], + "forecast_quality_score": [ + _with(_SNAPSHOT_OP_LABELS, group=g, id=i, model=m, model_fallback=mf, cold_start="false") + for g, i, m in [ + ("node_capacity", "node_cpu_busy_pct", "AutoARIMA"), + ("node_capacity", "node_memory_used_bytes", "SeasonalNaive"), + ("node_storage", "node_filesystem_avail_bytes", "AutoARIMA"), + ] + # Both fallback states present so ``ForecastFallbackPersistent`` + # (filters on ``model_fallback="true"``) and the cold-start / + # quality alerts (filter on ``cold_start!="true"``) both resolve. + for mf in ("false", "true") + ], + "forecast_deviation_outside_band": [ + _with(_SNAPSHOT_OP_LABELS, group=g, id=i, model=m, level=lvl) + for g, i, m in [ + ("node_capacity", "node_cpu_busy_pct", "AutoARIMA"), + ("node_storage", "node_filesystem_avail_bytes", "AutoARIMA"), + ] + for lvl in LEVELS + ], + "forecast_deviation_ratio": [ + _with(_SNAPSHOT_OP_LABELS, group=g, id=i, model=m, level=lvl) + for g, i, m in [ + ("node_capacity", "node_cpu_busy_pct", "AutoARIMA"), + ("node_storage", "node_filesystem_avail_bytes", "AutoARIMA"), + ] + for lvl in LEVELS + ], + "forecast_cold_start_siblings": [ + _with(_SNAPSHOT_OP_LABELS, group="node_capacity", query="node_cpu_busy_pct"), + ], + "node_filesystem_avail_bytes_forecast_time_to_threshold_seconds": [ + _with( + _SNAPSHOT_OP_LABELS, + id="node_filesystem_avail_bytes", + group="node_storage", + model="AutoARIMA", + instance="node-exporter:9100", + mountpoint="/var", + name=name, + comparator=cmp, + level=lvl, + ) + for name in THRESHOLD_NAMES + for cmp in COMPARATORS + for lvl in LEVELS + ], + "node_cpu_busy_pct_forecast_time_to_threshold_seconds": [ + _with( + _SNAPSHOT_OP_LABELS, + id="node_cpu_busy_pct", + group="node_capacity", + model="AutoARIMA", + instance="node-exporter:9100", + name=name, + comparator=cmp, + level=lvl, + ) + for name in THRESHOLD_NAMES + for cmp in COMPARATORS + for lvl in LEVELS + ], + "node_filesystem_avail_bytes_forecast_growth_rate": [ + _with( + _SNAPSHOT_OP_LABELS, + id="node_filesystem_avail_bytes", + group="node_storage", + model="AutoARIMA", + instance="node-exporter:9100", + mountpoint="/var", + window=w, + ) + for w in ("1d", "7d") + ], +} + +# --------------------------------------------------------------------------- +# PromQL leaf-matcher extraction. +# +# We only need to find ``metric_name{label="value", label!="value"}`` +# fragments and pull the label/value pairs out. A full parser is +# unnecessary here — Prometheus rules don't nest matchers, and the +# expressions in the bundled set use the standard literal shape. The +# regex below tolerates whitespace between matchers and across +# multi-line ``expr:`` blocks (the ``|`` style used in YAML). +# --------------------------------------------------------------------------- + +_FORECAST_NAME = re.compile(r"\b([a-zA-Z_:][a-zA-Z0-9_:]*)\s*\{") +_MATCHER = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(!=|=~|!~|=)\s*"([^"]*)"') + +# Forecaster-scrape labels that, when applied as filters to a forecast +# metric, would silently match zero series under the sink-first +# defaults. This list is the canonical "drift detector" — extend it +# only if the snapshot path grows a new scrape-side label. +_SCRAPE_ONLY_VALUES = { + "instance": re.compile(r"^[^:]*forecaster[^:]*:\d+$|^forecaster:\d+$"), + "job": re.compile(r"^promforecast$|^.*-promforecast$"), +} + + +def _iter_rule_exprs() -> list[tuple[str, str, str]]: + """Yield ``(group_name, alert_name, expr)`` for every alert in the file.""" + doc = yaml.safe_load(RULES_PATH.read_text()) + out: list[tuple[str, str, str]] = [] + for g in doc["spec"]["groups"]: + for r in g.get("rules", []): + if "alert" not in r: + continue + out.append((g["name"], r["alert"], r["expr"])) + return out + + +def _iter_metric_matchers(expr: str) -> list[tuple[str, list[tuple[str, str, str]]]]: + """Pull every ``metric{matchers}`` block from a PromQL expression. + + Returns ``[(metric_name, [(label, op, value), ...])]``. Bare metric + references (no ``{...}``) are not returned — they are unconstrained + and trivially satisfiable against any non-empty fixture entry. + """ + out: list[tuple[str, list[tuple[str, str, str]]]] = [] + # ``re.finditer`` over the ``metric_name{`` boundary; from each + # match we walk forward to the matching closing brace and slice the + # selector block. This avoids the regex-engine pathology of trying + # to match nested braces. + for m in _FORECAST_NAME.finditer(expr): + name = m.group(1) + # Skip language keywords that can immediately precede ``{`` in + # function-style calls (none of the bundled rules use ``by({...})`` + # but ``without({...})`` is a real construct). + if name in {"by", "without", "on", "ignoring", "group_left", "group_right"}: + continue + start = m.end() + depth = 1 + end = start + while end < len(expr) and depth > 0: + ch = expr[end] + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + end += 1 + if depth != 0: + continue + block = expr[start : end - 1] + matchers = [(lbl, op, val) for lbl, op, val in _MATCHER.findall(block)] + out.append((name, matchers)) + return out + + +def _labelset_matches(labels: dict[str, str], matchers: list[tuple[str, str, str]]) -> bool: + """Return True if ``labels`` would satisfy every matcher in ``matchers``.""" + for label, op, value in matchers: + present = labels.get(label, "") + if op == "=": + if present != value: + return False + elif op == "!=": + if present == value: + return False + elif op == "=~": + if not re.fullmatch(value, present): + return False + elif op == "!~" and re.fullmatch(value, present): + return False + return True + + +@pytest.mark.parametrize(("group", "alert", "expr"), _iter_rule_exprs()) +def test_rule_metric_names_avoid_reserved_colons(group: str, alert: str, expr: str) -> None: + """Forecaster output metric names must never use colons. + + Design principle 6 in ``CLAUDE.md`` reserves ``:`` for Prometheus + recording rules — exporters and direct instrumentation never use it. + ``exporter.py`` rejects colons at runtime, but bundled alerts could + drift independently (a recording-rule expression copy-pasted here + by mistake). This test catches that at PR time. + """ + for metric, _ in _iter_metric_matchers(expr): + if ":" in metric: + raise AssertionError( + f"{group}/{alert}: rule references metric {metric!r} which uses the " + "reserved ``:`` character. Forecaster output metrics never use colons " + "(see CLAUDE.md design principle 6). Did a recording-rule expression " + "slip into the bundled alerts?" + ) + + +@pytest.mark.parametrize(("group", "alert", "expr"), _iter_rule_exprs()) +def test_rule_uses_sink_compatible_label_filters(group: str, alert: str, expr: str) -> None: + """No example rule should pin a forecast metric to a scrape-only label. + + A pin like ``instance="forecaster:9091"`` on ``*_forecast`` would have + matched the snapshot scrape but matches zero series under the sink+VM + path. Operational metrics (``forecast_*``, + ``*_forecast_time_to_threshold_seconds``) are allowed to filter on the + scrape labels because they live only on /metrics. + """ + for metric, matchers in _iter_metric_matchers(expr): + if not re.fullmatch(r".+_forecast(_lower|_upper)?", metric): + continue + for label, op, value in matchers: + if op != "=": + continue + pattern = _SCRAPE_ONLY_VALUES.get(label) + if pattern and pattern.fullmatch(value): + raise AssertionError( + f"{group}/{alert}: rule pins sink-emitted metric {metric!r} " + f'on snapshot-only label {label}="{value}" — under the sink+VM ' + "defaults this would silently match zero series." + ) + + +@pytest.mark.parametrize(("group", "alert", "expr"), _iter_rule_exprs()) +def test_rule_metrics_resolve_against_sink_fixture(group: str, alert: str, expr: str) -> None: + """Every leaf metric reference should bind to at least one fixture series. + + **Scope: this is a binding test, not a correctness test.** It asserts + that each matcher resolves to at least one series in the sink-shaped + fixture — enough to catch typos (a metric renamed but the alert not + refreshed) and label-filter drift away from the sink labelset (a + matcher pinning a label value the sink never projects). It does *not* + audit alert semantics: a rule like ``count by (group) (X) > 0`` will + pass this test even if a label dimension (``cold_start``, + ``model_fallback``, ``level``) is silently aggregated over. Treat the + test as a guard against label drift, not as a substitute for review. + """ + skip_alerts = { + # Capacity templates are deliberately written for operator-tuned + # ``thresholds:`` configs the dev stack does not enable. The + # fixture would balloon needlessly trying to model every shape; + # the rule pinning above already audits the labelset. + } + if alert in skip_alerts: + pytest.skip("environment-specific template") + for metric, matchers in _iter_metric_matchers(expr): + series = FIXTURE.get(metric) + if series is None: + raise AssertionError( + f"{group}/{alert}: rule references {metric!r} but the sink-shaped " + "test fixture has no series for it — extend FIXTURE in this file " + "when adding a new emission family, or fix the rule if it drifted." + ) + if not any(_labelset_matches(ls, matchers) for ls in series): + raise AssertionError( + f"{group}/{alert}: matcher block on {metric!r} matches zero " + f"fixture series — expr={expr!r}" + ) + + +def test_header_declares_sink_assumption() -> None: + """The file's leading comment must call out the sink+VM assumption. + + Operators who paste a single rule out of this file lose the comment; + the assumption is still recoverable from the docs link the header + points at. Drift here means the header was edited; refresh both + halves so the docs reference stays accurate. + """ + head = RULES_PATH.read_text(encoding="utf-8").splitlines()[:20] + joined = "\n".join(head) + assert "sink+VM" in joined or "sink + VM" in joined, joined + assert "docs/architecture/emission-paths.md" in joined, joined diff --git a/forecaster/tests/test_reload.py b/forecaster/tests/test_reload.py index 2489587..d03812a 100644 --- a/forecaster/tests/test_reload.py +++ b/forecaster/tests/test_reload.py @@ -24,7 +24,12 @@ ServerConfig, ) from promforecast.exporter import Exporter, GroupSnapshot -from promforecast.reload import ConfigMapWatcher, ConfigReloader, ReloadError +from promforecast.reload import ( + ConfigMapWatcher, + ConfigReloader, + ReloadError, + log_group_queries_loaded, +) from promforecast.runner import Runner from promforecast.source import PromSource @@ -317,6 +322,112 @@ async def _hanging_run_group(self, group, *, queries=None): # type: ignore[no-u await runner.stop() +def test_log_group_queries_loaded_emits_one_line_per_group() -> None: + """Boot/reload emission lists every loaded query ID per group. + + The line set must mirror what's *actually* in the live config — so a + macOS Docker Desktop bind-mount that ships a truncated YAML on reload + is visible from the logs (one missing query == one missing line, or a + short ``queries`` list). + """ + import structlog # noqa: PLC0415 + + cfg = _config( + [ + GroupConfig( + name="full", + queries=[ + QueryConfig(id="m1", promql="up"), + QueryConfig(id="m2", promql="rate(up[5m])"), + ], + ), + GroupConfig(name="empty", queries=[QueryConfig(id="m3", promql="up")]), + ] + ) + with structlog.testing.capture_logs() as captured: + log_group_queries_loaded(cfg, source="boot") + events = [e for e in captured if e["event"] == "group_queries_loaded"] + assert [(e["group"], e["queries"], e["source"]) for e in events] == [ + ("full", ["m1", "m2"], "boot"), + ("empty", ["m3"], "boot"), + ] + + +@pytest.mark.asyncio +async def test_reload_logs_truncated_query_set(tmp_path: Path) -> None: + """Truncated YAML on reload → emitted log lines match the truncation. + + Re-enacts the macOS Docker Desktop bind-mount footgun: the file on + disk loses a query and the next reload picks up the shorter set. The + ``group_queries_loaded`` line emitted after ``config_reload_succeeded`` + is the operator's signal that the swap did not include the missing + query — which would otherwise be invisible behind the still-green + reload status. + """ + import structlog # noqa: PLC0415 + + full_config = ( + _BASE_CONFIG + + """ - id: m2 + promql: rate(up[5m]) +""" + ) + path = _write(tmp_path, full_config) + from promforecast import config as config_module # noqa: PLC0415 + + cfg = config_module.load(path) + exporter = Exporter() + runner = Runner(config=cfg, source=_NullSource(), exporter=exporter) + reloader = ConfigReloader(runner=runner, exporter=exporter, config_path=path) + + # Drop ``m2`` on disk — mimics a truncated bind-mount projection. + path.write_text(_BASE_CONFIG) + with structlog.testing.capture_logs() as captured: + await reloader.reload() + + events = [e for e in captured if e["event"] == "group_queries_loaded"] + assert len(events) == 1 + assert events[0]["group"] == "g1" + assert events[0]["queries"] == ["m"] + assert events[0]["source"] == "reload" + await runner.stop() + + +def test_boot_load_emits_config_loaded_then_group_queries_in_order(tmp_path: Path) -> None: + """``main._load_config_for_boot`` must emit ``config_loaded`` first, then + one ``group_queries_loaded`` per group with ``source="boot"``. + + Guards the ordering contract documented in + ``docs/operations/reload.md`` so a log filter pinned to + ``config_loaded -> group_queries_loaded`` keeps working even if the + boot path is refactored. + """ + import structlog # noqa: PLC0415 + + from promforecast.main import _load_config_for_boot # noqa: PLC0415 + + full_config = ( + _BASE_CONFIG + + """ - id: m2 + promql: rate(up[5m]) +""" + ) + path = _write(tmp_path, full_config) + with structlog.testing.capture_logs() as captured: + cfg = _load_config_for_boot(path) + + events = [e for e in captured if e["event"] in {"config_loaded", "group_queries_loaded"}] + assert events[0]["event"] == "config_loaded" + assert events[0]["groups"] == len(cfg.groups) + # Every other event must be the group line tagged ``source="boot"``. + boot_events = [e for e in events[1:] if e["event"] == "group_queries_loaded"] + assert boot_events, "boot path did not emit any group_queries_loaded line" + for event in boot_events: + assert event["source"] == "boot" + assert [e["group"] for e in boot_events] == [g.name for g in cfg.groups] + assert [e["queries"] for e in boot_events] == [[q.id for q in g.queries] for g in cfg.groups] + + @pytest.mark.asyncio async def test_configmap_watcher_triggers_reload_on_content_change(tmp_path: Path) -> None: """Inotify-style watcher: file change → reload fires.""" diff --git a/forecaster/uv.lock b/forecaster/uv.lock index f275a26..32fe545 100644 --- a/forecaster/uv.lock +++ b/forecaster/uv.lock @@ -428,6 +428,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "holidays" +version = "0.97" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/61/058f3f05dd318b9d9546df513f8f6611557919e5360cd608a3ce7c7500d4/holidays-0.97.tar.gz", hash = "sha256:8fe491270bd4aeed6f9584d459d5df506a414727ba76fdd9ebd6323def606935", size = 911304, upload-time = "2026-05-18T19:48:19.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/54/7947b0cba0c91b81801da793ea2d273aab5bdd5484a5ba6c2d3863110f07/holidays-0.97-py3-none-any.whl", hash = "sha256:0bcd55e64abddce2f9aa9224d68afe87acdd7eac7c2aef251b0ef7986bb5220b", size = 1479247, upload-time = "2026-05-18T19:48:16.881Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -995,6 +1007,7 @@ dependencies = [ [package.optional-dependencies] dev = [ + { name = "holidays" }, { name = "mypy" }, { name = "pandas-stubs" }, { name = "pytest" }, @@ -1007,6 +1020,9 @@ ha = [ { name = "kubernetes" }, { name = "redis" }, ] +holidays = [ + { name = "holidays" }, +] otel = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-sdk" }, @@ -1015,6 +1031,8 @@ otel = [ [package.metadata] requires-dist = [ { name = "fastapi", specifier = ">=0.115" }, + { name = "holidays", marker = "extra == 'dev'", specifier = ">=0.50" }, + { name = "holidays", marker = "extra == 'holidays'", specifier = ">=0.50" }, { name = "httpx", specifier = ">=0.27" }, { name = "kubernetes", marker = "extra == 'ha'", specifier = ">=31.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13" }, @@ -1039,7 +1057,7 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32" }, ] -provides-extras = ["ha", "otel", "dev"] +provides-extras = ["holidays", "ha", "otel", "dev"] [[package]] name = "protobuf"