Architecture realignment

Makefile

@@ -115,6 +115,16 @@ exit('!! forecast_query_cache_events_total{kind=\"miss\"} is zero -- the cache i
 r=json.loads(u.urlopen('http://localhost:9090/api/v1/query?query=forecast_accuracy_mape').read())['data']['result']; \
 print(f'  forecast_accuracy_mape: {len(r)} series'); \
 exit('!! Prometheus has 0 series - check the prometheus scrape config and forecaster reachability') if not r else None"
+	@echo ">> no label-divergent dual-source forecast labelsets in VM (regression guard for the scrape-drop)"
+	@cd $(FORECASTER) && .venv/bin/promforecast diagnose duplicates --datasource http://localhost:8428 --since 10m \
+	  > /tmp/promforecast-diagnose.txt 2>&1 \
+	  && echo ">> diagnose duplicates: clean" \
+	  || (echo "!! diagnose duplicates flagged a label-divergent duplicate state; see /tmp/promforecast-diagnose.txt"; tail -30 /tmp/promforecast-diagnose.txt; exit 1)
+	@echo ">> no same-labelset dual-source forecast ingest in VM (regression guard for sample-rate contamination)"
+	@cd $(FORECASTER) && .venv/bin/promforecast diagnose sample-rate --datasource http://localhost:8428 --window 10m --max-samples 5 \
+	  > /tmp/promforecast-sample-rate.txt 2>&1 \
+	  && echo ">> diagnose sample-rate: clean" \
+	  || (echo "!! diagnose sample-rate flagged at least one series above the per-window cap; see /tmp/promforecast-sample-rate.txt"; tail -30 /tmp/promforecast-sample-rate.txt; exit 1)
 	@if docker compose -f docker-compose.dev.yml ps --status running --services 2>/dev/null | grep -qx grafana; then \
 	  echo ">> dashboards reachable in Grafana at http://localhost:3000 (admin / admin)"; \
 	else \

README.md

@@ -33,6 +33,18 @@ helm install promforecast promforecast/promforecast-stack
 This installs VictoriaMetrics, the forecaster, and example dashboards.
 For an existing TSDB, install only `promforecast/promforecast`.
 
+The umbrella chart ships **sink + VictoriaMetrics** as the canonical
+forecast write path: the forecaster pushes the full forecast curve into
+VM with native future timestamps, and the bundled ServiceMonitor drops
+the redundant `*_forecast` / `*_forecast_lower` / `*_forecast_upper`
+snapshot families from the scrape so dashboards plot a smooth forward
+curve rather than averaging the sink push against a 5-minute snapshot
+staircase. See [`docs/architecture/emission-paths.md`](docs/architecture/emission-paths.md)
+for what each path emits and the dual-source trap that motivates the
+defaults. Existing snapshot-only installs upgrading to the new defaults
+should follow [`docs/how-to/migrate-to-sink-first.md`](docs/how-to/migrate-to-sink-first.md)
+and use `promforecast diagnose duplicates` to confirm the cutover.
+
 Editing the YAML config does **not** require a pod restart: the
 forecaster watches the mounted ConfigMap, validates the new config
 in-place, and rolls back on validation failure (`kubectl apply` to the

charts/promforecast-stack/README.md

@@ -22,11 +22,49 @@ accepted (VM enforces a 2-day minimum). **Tune this higher than your longest
 configured forecast horizon or your longest forecasts will be silently
 dropped.**
 
+## Sink-first defaults
+
+The chart pre-configures the **sink + VM** emission path: the forecaster
+pushes the full forecast curve into the bundled VictoriaMetrics with
+native future timestamps, and the forecaster's ServiceMonitor drops the
+`*_forecast` / `*_forecast_lower` / `*_forecast_upper` snapshot families
+from the scrape so the long-term TSDB stores one canonical labelset per
+forecast metric. Bundled dashboards default to VictoriaMetrics as their
+datasource.
+
+A `helm test` hook
+(`<release>-promforecast-stack-test-labelset`) queries VM after install
+and asserts that no `*_forecast` series carry the forecaster's scrape
+`job` label — the canonical signature of the dual-source state. Run it
+on demand:
+
+```bash
+helm test <release>
+```
+
+See [`docs/architecture/emission-paths.md`](../../docs/architecture/emission-paths.md)
+for what the two paths emit, why running both unguarded is a silent
+trap, and the `metric_relabel_configs` snippet to paste into an
+externally-managed Prometheus.
+
+**ServiceMonitor CRD required.** `promforecast.serviceMonitor.enabled`
+defaults to `true` so the chart-managed snapshot drop is applied. On
+clusters without prometheus-operator (no `monitoring.coreos.com/v1`
+ServiceMonitor CRD), `helm install` will fail; flip
+`promforecast.serviceMonitor.enabled=false` and apply the scrape-drop
+from the docs above to your plain-Prometheus config instead.
+
 ## Subchart toggles
 
 - `victoria-metrics-single.enabled`: bundle a VM single-node install.
-- `promforecast.enabled`: bundle the forecaster.
-- `dashboards.enabled`: install Grafana dashboards as ConfigMaps.
+- `promforecast.enabled`: bundle the forecaster (sink to VM by default).
+- `dashboards.enabled`: install Grafana dashboards as ConfigMaps tagged
+  for the Grafana sidecar.
+- `datasources.enabled`: ship a Grafana datasource provisioning
+  ConfigMap (also sidecar-tagged) that registers VM as the default
+  datasource the bundled dashboards target.
 - `prometheusRules.enabled`: install example `PrometheusRule` manifests.
+- `tests.labelsetUniqueness.enabled`: include the `helm test` probe
+  described above.
 
 See [`values.yaml`](values.yaml) for the full schema.

charts/promforecast-stack/templates/datasources.yaml

@@ -0,0 +1,47 @@
+{{- if .Values.datasources.enabled }}
+{{- /*
+    Grafana datasource provisioning shipped as ConfigMaps with the sidecar
+    label so a Grafana running the kiwigrid/k8s-sidecar pattern picks them
+    up automatically. VictoriaMetrics is registered as the default so the
+    bundled dashboards land on the sink-fed copy of the forecast metrics
+    (smooth curves) rather than the scraped snapshot (5-min staircase).
+    See ``docs/architecture/emission-paths.md`` for why the default matters.
+*/}}
+{{- $vm := .Values.datasources.victoriaMetrics }}
+{{- $prom := .Values.datasources.prometheus }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-datasources" (include "promforecast-stack.fullname" .) | trunc 63 | trimSuffix "-" }}
+  labels:
+{{- include "promforecast-stack.labels" . | nindent 4 }}
+{{- range $k, $v := .Values.datasources.labels }}
+    {{ $k }}: {{ $v | quote }}
+{{- end }}
+data:
+  promforecast-datasources.yaml: |-
+    apiVersion: 1
+    datasources:
+      - name: VictoriaMetrics
+        type: prometheus
+        access: proxy
+        uid: {{ $vm.uid | quote }}
+        url: {{ $vm.url | quote }}
+        isDefault: {{ $vm.isDefault }}
+        editable: true
+        jsonData:
+          httpMethod: POST
+          timeInterval: 15s
+{{- if $prom.url }}
+      - name: Prometheus
+        type: prometheus
+        access: proxy
+        uid: {{ $prom.uid | quote }}
+        url: {{ $prom.url | quote }}
+        isDefault: {{ $prom.isDefault }}
+        editable: true
+        jsonData:
+          httpMethod: POST
+          timeInterval: 15s
+{{- end }}
+{{- end }}

charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml

@@ -0,0 +1,136 @@
+{{- /*
+    Post-install test: assert that every ``*_forecast`` / ``*_forecast_lower``
+    / ``*_forecast_upper`` series in VictoriaMetrics exists under exactly
+    one labelset family — specifically, no copy carries the forecaster's
+    own scrape labels (``job=<release>-promforecast`` / a forecaster-pod
+    ``instance``) alongside the sink-emitted copy that lacks them. The
+    dual-source state silently rendered the forecast curve as an average
+    of the smooth sink copy with the 5-minute snapshot staircase before
+    the sink-first realignment. Wired as a ``helm test`` hook (not part
+    of the normal install) so an operator can run it on demand. CI does
+    not execute it — no kind cluster — but ``helm template`` still
+    renders it and ``kubeconform`` validates the Pod schema. Set
+    ``tests.labelsetUniqueness.enabled=false`` to suppress the asset for
+    managed clusters that forbid hook resources.
+
+    The probe uses VictoriaMetrics' ``/api/v1/series`` endpoint with a
+    URL-encoded ``match[]`` selector that filters in-VM on the
+    forecaster-scrape labels. The expected response under the sink-first
+    defaults is an empty result set; any series returned signals dual
+    source. We deliberately do NOT scan for the *presence* of an
+    ``instance`` label on forecast series — the sink legitimately
+    propagates the source ``instance`` (e.g. ``node-exporter:9100``),
+    so a presence-only check would false-fire on a clean install.
+*/}}
+{{- if .Values.tests.labelsetUniqueness.enabled }}
+{{- /*
+    The standalone forecaster ServiceMonitor sets
+    ``jobLabel: app.kubernetes.io/name``, so the Prometheus scrape job
+    label is the stable value ``promforecast`` regardless of Helm
+    release name. The probe's selector targets that exact job string.
+    Override ``tests.scrapeJobPattern`` only when an external scrape
+    (i.e. not the umbrella-managed ServiceMonitor) re-labels the job.
+*/}}
+{{- $jobPattern := default "promforecast" .Values.tests.scrapeJobPattern }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ printf "%s-test-labelset" (include "promforecast-stack.fullname" .) | trunc 63 | trimSuffix "-" }}
+  labels:
+{{- include "promforecast-stack.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  restartPolicy: Never
+  containers:
+    - name: assert
+      image: {{ .Values.tests.image | quote }}
+      env:
+        - name: VM_URL
+          # Pinned to the VM subchart's Service name, which the umbrella
+          # locks with ``victoria-metrics-single.fullnameOverride`` so the
+          # URL is release-name-agnostic.
+          value: {{ .Values.tests.victoriaMetricsUrl | default "http://promforecast-stack-victoria-metrics-single-server:8428" | quote }}
+        - name: SCRAPE_JOB_PATTERN
+          value: {{ $jobPattern | quote }}
+        - name: FIRST_REFRESH_TIMEOUT_SECONDS
+          value: {{ .Values.tests.firstRefreshTimeoutSeconds | default 300 | quote }}
+      command:
+        - sh
+        - -c
+        - |
+          set -eu
+          echo "Querying ${VM_URL} for the labelset-uniqueness assertion..."
+          # URL-encode the selectors by hand because busybox lacks ``jq``
+          # and a plain interpolation of the literal ``{__name__=~"..."}``
+          # characters into a URL would not survive HTTP parsing. The
+          # first selector asks VM for forecast-family series carrying
+          # the forecaster's scrape ``job`` label — under the sink-first
+          # defaults the snapshot drop strips them, so the count must be
+          # zero. The second selector asks VM for forecast-family series
+          # without that ``job`` label — those are the sink-pushed copies
+          # and the count must be non-zero (a broken sink + applied drop
+          # would silently leave VM with no forecast data at all, which
+          # is also a regression we want to surface).
+          dup_selector='%7B__name__%3D~%22.%2B_forecast%28_lower%7C_upper%29%3F%22%2Cjob%3D~%22'"$SCRAPE_JOB_PATTERN"'%22%7D'
+          sink_selector='%7B__name__%3D~%22.%2B_forecast%28_lower%7C_upper%29%3F%22%2Cjob%3D%22%22%7D'
+          # The first scheduled fit completes some seconds-to-minutes after
+          # ``helm install`` finishes; the sink push lands in VM only after
+          # that. Poll for up to ``FIRST_REFRESH_TIMEOUT_SECONDS`` (5min
+          # default) before declaring the sink-side empty. The dup check
+          # is meaningful at any point — even before the first fit, the
+          # scraped snapshot family should never reach VM under the
+          # sink-first defaults — but we re-evaluate it on each iteration
+          # so a single ``helm test`` invocation reports both failure modes.
+          deadline=$(( $(date +%s) + FIRST_REFRESH_TIMEOUT_SECONDS ))
+          while :; do
+            dup_out=$(wget -q -O - "${VM_URL}/api/v1/series?match%5B%5D=${dup_selector}") || {
+              echo "ERROR: could not query VictoriaMetrics at $VM_URL" >&2
+              exit 2
+            }
+            sink_out=$(wget -q -O - "${VM_URL}/api/v1/series?match%5B%5D=${sink_selector}") || {
+              echo "ERROR: could not query VictoriaMetrics at $VM_URL" >&2
+              exit 2
+            }
+            # ``grep -c`` over the ``__name__`` key is good enough — we only
+            # need zero-vs-non-zero. A clean install returns
+            # ``{"status":"success","data":[]}`` for the dup query and a
+            # non-empty data array for the sink query.
+            dup_n=$(echo "$dup_out" | grep -oE '"__name__":"[^"]+"' | wc -l | tr -d ' ')
+            sink_n=$(echo "$sink_out" | grep -oE '"__name__":"[^"]+"' | wc -l | tr -d ' ')
+            # Dup detection is fail-fast: the snapshot drop is a static
+            # ServiceMonitor relabel that does not depend on the first fit,
+            # so any series carrying the scrape job at any point is a
+            # config bug we should report immediately rather than waiting
+            # for the timeout to elapse.
+            if [ "$dup_n" -gt 0 ]; then
+              echo "FAIL: $dup_n forecast series carry the forecaster scrape job ($SCRAPE_JOB_PATTERN)." >&2
+              echo "      The snapshot scrape and the sink push are both being stored." >&2
+              echo "Remediation: promforecast.serviceMonitor.enabled=true (the umbrella default)" >&2
+              echo "             so the standalone chart drops the snapshot families. For an" >&2
+              echo "             externally-managed Prometheus, add the metric_relabel_configs" >&2
+              echo "             snippet from docs/architecture/emission-paths.md." >&2
+              echo "$dup_out" >&2
+              exit 1
+            fi
+            if [ "$sink_n" -gt 0 ]; then
+              echo "OK: $sink_n sink-pushed forecast series in VM, zero copies carrying the forecaster scrape job."
+              exit 0
+            fi
+            now=$(date +%s)
+            if [ "$now" -ge "$deadline" ]; then
+              echo "FAIL: no *_forecast series found without the forecaster scrape job." >&2
+              echo "      Waited ${FIRST_REFRESH_TIMEOUT_SECONDS}s for the first scheduled" >&2
+              echo "      refresh to push curves to VM. Either the forecaster's" >&2
+              echo "      ``refresh_interval`` is longer than this timeout, the sink" >&2
+              echo "      configuration is broken, or the forecaster has not become" >&2
+              echo "      ready yet. Check ``forecast_failures_total`` and" >&2
+              echo "      ``forecast_remote_write_failures_total`` on the forecaster" >&2
+              echo "      ``/metrics`` endpoint, or raise ``tests.firstRefreshTimeoutSeconds``." >&2
+              exit 1
+            fi
+            echo "waiting for first sink push (${sink_n} sink series so far, $((deadline - now))s left)..."
+            sleep 5
+          done
+{{- end }}

charts/promforecast-stack/values.schema.json

@@ -23,6 +23,49 @@
         "enabled": { "type": "boolean" },
         "labels": { "type": "object" }
       }
+    },
+    "datasources": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": { "type": "boolean" },
+        "labels": { "type": "object" },
+        "victoriaMetrics": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "uid": { "type": "string", "minLength": 1 },
+            "url": { "type": "string" },
+            "isDefault": { "type": "boolean" }
+          }
+        },
+        "prometheus": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "uid": { "type": "string", "minLength": 1 },
+            "url": { "type": "string" },
+            "isDefault": { "type": "boolean" }
+          }
+        }
+      }
+    },
+    "tests": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "image": { "type": "string", "minLength": 1 },
+        "victoriaMetricsUrl": { "type": "string" },
+        "scrapeJobPattern": { "type": "string" },
+        "firstRefreshTimeoutSeconds": { "type": "integer", "minimum": 0 },
+        "labelsetUniqueness": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": { "type": "boolean" }
+          }
+        }
+      }
     }
   }
 }