diff --git a/charts/promforecast/templates/tests/validate-config.yaml b/charts/promforecast/templates/tests/validate-config.yaml new file mode 100644 index 0000000..d7818b9 --- /dev/null +++ b/charts/promforecast/templates/tests/validate-config.yaml @@ -0,0 +1,53 @@ +{{- /* + Post-install ``helm test`` hook: runs ``promforecast validate + --strict-cardinality`` against the same ConfigMap the live + Deployment mounts. The strict flag flips the projected + cardinality overshoot from an advisory ``WARN`` line into a + non-zero exit code, so an operator can chain ``helm test`` into + a GitOps PostSync gate that rejects configs whose opt-in + emission would blow ``safety.max_total_series``. + + Wired as a ``helm test`` hook (not a normal apply) so it costs + nothing during the standard install — operators opt in by + running ``helm test ``. CI does not execute the hook + (no kind cluster), but ``helm template`` still renders it and + ``kubeconform`` validates the Pod schema. + + Set ``tests.validateConfig.enabled=false`` to suppress the + asset entirely for managed clusters that forbid hook resources. +*/}} +{{- if .Values.tests.validateConfig.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ printf "%s-test-validate" (include "promforecast.fullname" .) | trunc 63 | trimSuffix "-" }} + labels: {{- include "promforecast.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + serviceAccountName: {{ include "promforecast.serviceAccountName" . }} + restartPolicy: Never + securityContext: {{- toYaml .Values.podSecurityContext | nindent 4 }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: validate + image: {{ include "promforecast.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + args: + - validate + - --config=/etc/promforecast/config.yaml + - --strict-cardinality + resources: {{- toYaml .Values.resources | nindent 8 }} + volumeMounts: + - name: config + mountPath: /etc/promforecast + readOnly: true + volumes: + - name: config + configMap: + name: {{ include "promforecast.configMapName" . }} +{{- end }} diff --git a/charts/promforecast/values.yaml b/charts/promforecast/values.yaml index f7a3f39..2859bce 100644 --- a/charts/promforecast/values.yaml +++ b/charts/promforecast/values.yaml @@ -271,3 +271,19 @@ podLabels: {} extraEnv: [] extraVolumes: [] extraVolumeMounts: [] + +# ``helm test`` hooks. Each hook below renders as a one-shot Pod that +# the operator runs with ``helm test ``; nothing is created +# during the normal ``helm install`` apply, so the production install +# does not pay any test-pod resource cost. Disable a hook to suppress +# the corresponding rendered template entirely. +tests: + # ``validateConfig`` runs ``promforecast validate --strict-cardinality`` + # against the mounted config. The strict flag flips the projected + # cardinality overshoot from an advisory ``WARN`` line into a hard + # fail, so an operator using a wrapper like ArgoCD's PostSync hook + # can reject changes that would blow the global series cap before the + # forecaster pod ingests them. See docs/operations/cardinality.md for + # the workflow. + validateConfig: + enabled: true diff --git a/community/promforecast-prophet/src/promforecast_prophet/model.py b/community/promforecast-prophet/src/promforecast_prophet/model.py index 899717f..ae0ebbe 100644 --- a/community/promforecast-prophet/src/promforecast_prophet/model.py +++ b/community/promforecast-prophet/src/promforecast_prophet/model.py @@ -49,6 +49,12 @@ class ProphetModel: """ name = "Prophet" + # Prophet's ``fit_predict`` actually merges and consumes the + # ``regressors`` frame (see lines below) — so the forecaster's + # conditional-forecast pass needs to refit Prophet with the + # held-constant regressors instead of aliasing the unconditional + # output. + uses_regressors = True def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/community/promforecast-prophet/tests/test_prophet_model.py b/community/promforecast-prophet/tests/test_prophet_model.py index 3e53089..0a6d8f4 100644 --- a/community/promforecast-prophet/tests/test_prophet_model.py +++ b/community/promforecast-prophet/tests/test_prophet_model.py @@ -241,6 +241,78 @@ def load(self) -> Any: return [*real, _StubEP()] +def test_prophet_advertises_uses_regressors_true() -> None: + """Prophet's ``fit_predict`` actually consumes the ``regressors`` frame + (the wrapper registers each column via ``add_regressor`` and merges + it into the future frame). The forecaster's conditional-emission + path reads the ``uses_regressors`` class attribute to decide whether + to refit with held-constant regressors; Prophet must advertise + ``True`` so the refit actually happens. + """ + assert ProphetModel.uses_regressors is True + + +def test_prophet_uses_regressors_flag_via_registry_helper() -> None: + """Through the registry helper too — the forecaster reads + ``model_registry.uses_regressors(name)`` rather than reaching for + the class attribute directly. Verify the discovery + lookup path + agrees end-to-end. + """ + from promforecast import models as registry # noqa: PLC0415 + + registry.refresh() + try: + with patch("promforecast.models.entry_points", _patched_entry_points): + registry.refresh() + assert registry.uses_regressors("Prophet") is True + finally: + registry.refresh() + + +def test_prophet_legitimately_differs_when_regressors_held_constant() -> None: + """End-to-end: the conditional refit path must produce different bands + when the regressors frame is flat-held vs. the original input. + + The fake Prophet uses ``regressors[regressor_name].iloc[-1]`` as a + proxy for "the regressor's current value"; in our flat-held + contract every row of the conditional frame equals the in-sample + final value, so the post-fit predicted yhat tracks that value. We + confirm the wrapper preserves the regressors all the way through — + so the conditional refit really does see the held-constant frame — + by inspecting the per-instance ``add_regressor`` calls and the + fitted dataframe's regressor column. + """ + df = _make_df() + # Two distinct configurations of the same regressor column: + # (a) varying — the "unconditional" input the model would see + # under a real fit. + # (b) flat-held — every row pinned to the last observed value, + # matching what the runner's + # ``_flatten_regressors_to_last_value`` produces. + varying = pd.DataFrame({"ds": df["ds"], "deploys": [float(i % 5) for i in range(len(df))]}) + held = pd.DataFrame({"ds": df["ds"], "deploys": [varying["deploys"].iloc[-1]] * len(df)}) + + ProphetModel().fit_predict(df=df, horizon=3, freq="h", levels=[80], regressors=varying) + unconditional_inst = _FakeProphet.instances[-1] + assert unconditional_inst.fit_df is not None + # The fit frame has the varying regressor merged in. + assert "deploys" in unconditional_inst.fit_df.columns + assert list(unconditional_inst.fit_df["deploys"]) == list(varying["deploys"]) + + _FakeProphet.instances.clear() + ProphetModel().fit_predict(df=df, horizon=3, freq="h", levels=[80], regressors=held) + conditional_inst = _FakeProphet.instances[-1] + assert conditional_inst.fit_df is not None + # The flat-held frame is what the conditional refit feeds Prophet. + # Every row equals the last value so the regressor effect is + # "frozen at the current state". + deploys_in_fit = list(conditional_inst.fit_df["deploys"]) + assert deploys_in_fit == [held["deploys"].iloc[-1]] * len(df) + # Both fits add the regressor — the wrapper isn't accidentally + # silencing the regressor on the conditional pass. + assert conditional_inst.regressors == ["deploys"] + + def test_prophet_appears_in_promforecast_registry_via_entry_point() -> None: """End-to-end: once the package is installed, ``Prophet`` becomes a valid model name in promforecast configs without any forecaster-side diff --git a/dashboards/grafana/promforecast-capacity.json b/dashboards/grafana/promforecast-capacity.json index 980a342..20fe2ae 100644 --- a/dashboards/grafana/promforecast-capacity.json +++ b/dashboards/grafana/promforecast-capacity.json @@ -325,6 +325,83 @@ }, "overrides": [] } + }, + { + "type": "table", + "id": 8, + "title": "Next maintenance windows (lowest predicted load)", + "description": "Top-N predicted-lowest-load windows in the next ``recommended_windows.lookahead`` of the forecast horizon. Lower predicted_load is better; rank=1 is the suggested window. Operators schedule reboots / cert rotations / security patches against the ranked recommendations.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 39 + }, + "targets": [ + { + "expr": "forecast_recommended_maintenance_window{group=~\"$group\",id=~\"$id\"}", + "legendFormat": "", + "format": "table", + "instant": true + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "best_model": true, + "horizon": true, + "model": true, + "model_fallback": true, + "job": true, + "cluster": true, + "replica": true + }, + "indexByName": { + "rank": 0, + "id": 1, + "group": 2, + "instance": 3, + "start_timestamp_seconds": 4, + "duration_seconds": 5, + "Value": 6 + }, + "renameByName": { + "Value": "predicted_load", + "start_timestamp_seconds": "start (unix)", + "duration_seconds": "duration (s)" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "start (unix)" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + } + ] + } } ] } diff --git a/dashboards/grafana/promforecast-explainability.json b/dashboards/grafana/promforecast-explainability.json new file mode 100644 index 0000000..ffcd0fc --- /dev/null +++ b/dashboards/grafana/promforecast-explainability.json @@ -0,0 +1,190 @@ +{ + "title": "promforecast — Explainability (component decomposition)", + "uid": "promforecast-explainability", + "schemaVersion": 39, + "version": 1, + "editable": true, + "tags": [ + "promforecast", + "forecasting", + "explainability", + "decomposition" + ], + "timezone": "", + "time": { + "from": "now-6h", + "to": "now+24h" + }, + "refresh": "5m", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics" + } + }, + { + "name": "metric", + "type": "query", + "label": "Series", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values({__name__=~\".+_forecast_component\"}, __name__)", + "refresh": 2, + "includeAll": false, + "multi": false + }, + { + "name": "model", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": "label_values({__name__=\"$metric\"}, model)", + "refresh": 2, + "includeAll": false, + "multi": false + } + ] + }, + "panels": [ + { + "type": "timeseries", + "id": 1, + "title": "Component contributions to the predicted curve (stacked)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "{__name__=\"$metric\",model=\"$model\"}", + "legendFormat": "{{component}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "fillOpacity": 40, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "description": "Per-step decomposition of the predicted curve. Components sum to the central forecast row-by-row; an unexpected residual band signals model surprise." + }, + { + "type": "timeseries", + "id": 2, + "title": "Trend component only", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "expr": "{__name__=\"$metric\",model=\"$model\",component=\"trend\"}", + "legendFormat": "trend" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "description": "Linear-best-fit slope across the horizon. A steepening trend explains 'X fills in N days' before the deviation alert catches up." + }, + { + "type": "timeseries", + "id": 3, + "title": "Seasonal component only", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "expr": "{__name__=\"$metric\",model=\"$model\",component=\"seasonal\"}", + "legendFormat": "seasonal" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "description": "Per-position-within-cycle deviation. Peaks identify the part of the daily/weekly pattern most responsible for the predicted shape." + }, + { + "type": "stat", + "id": 4, + "title": "Decomposition skips (recent)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 20 + }, + "targets": [ + { + "expr": "sum by (model, reason) (increase(forecast_decomposition_skipped_total[1h]))", + "legendFormat": "{{model}} / {{reason}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + } + } + }, + "description": "Per-(model, reason) skip counter rate. A sustained non-zero reading means the configured model can't decompose for some series — switch models or disable the family for that query." + } + ] +} diff --git a/docker/dev-config.yaml b/docker/dev-config.yaml index bbfd1e7..bafc400 100644 --- a/docker/dev-config.yaml +++ b/docker/dev-config.yaml @@ -98,14 +98,31 @@ groups: strategy: forward_fill max_gap: 10m queries: + # Exercises the conditional-forecast emission end-to-end — the + # runner re-fits with regressors flattened to their current value + # so the parallel ``..._forecast_conditional*`` family populates + # in dev. No regressors are configured on this query, so the + # conditional curve equals the unconditional one for the built-in + # models that ignore regressors — the wiring is what's exercised. - id: node_cpu_busy_pct promql: | 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) + emission: + conditional: true - id: node_memory_used_bytes promql: | node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes - id: node_load1 promql: node_load1 + # Exercises the maintenance-window recommender against a real + # series in dev. With short lookback / step / horizon the + # recommended windows churn each refresh but the family + # populates from the first run. + recommended_windows: + enabled: true + duration: 5m + lookahead: 1h + count: 3 - id: node_load5 promql: node_load5 @@ -118,10 +135,23 @@ groups: queries: # Filesystem fill: predict where used % is heading. The capacity # dashboard uses this to surface "disk will be full in N hours". + # Also exercises two predictive emission families end-to-end: + # * thresholds + alert_within emit ``..._forecast_will_breach`` + # (operators alert on the boolean directly). + # * ``emission.decompose: true`` emits ``..._forecast_component`` + # (stacked-area explainability). - id: node_filesystem_used_pct promql: | 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"}) + thresholds: + - name: warn + value: 80 + comparator: gt + alert_within: 30m + severity: warning + emission: + decompose: true - id: node_disk_read_bytes_rate promql: rate(node_disk_read_bytes_total[2m]) - id: node_disk_written_bytes_rate @@ -173,6 +203,51 @@ groups: ) season_length: 10 data_profile: intermittent + + # Error ratio with SLO error-budget forecasting. Exercises the + # ``slo:`` config end-to-end — the three derived families + # (remaining / burn_rate / exhausted_in) populate per (series, + # window) in dev. Multiple windows so the multi-window pattern + # has data to show. + - id: http_error_ratio + promql: | + ( + sum by (endpoint) ( + rate(http_requests_total{status=~"5..",endpoint=~"/api/.*|/healthz"}[2m]) + ) + / + clamp_min( + sum by (endpoint) ( + rate(http_requests_total{endpoint=~"/api/.*|/healthz"}[2m]) + ), + 0.001 + ) + ) + season_length: 10 + slo: + objective: 99.5 + windows: [5m, 1h, 30d] + # Recommend the lowest-load 5-minute windows for scheduling + # brief downtime. The loadgen is sinusoidal, so the recommender + # has a clear seasonality to mine. + recommended_windows: + enabled: true + duration: 5m + lookahead: 1h + count: 3 + + # Synthetic backlog / drain demo. The forecaster sees a queue + # that decays with the inverse of request rate; the ``le_drain`` + # threshold produces the ``..._forecast_time_to_drain_seconds`` + # family — with +Inf samples when the queue is not draining. + - id: http_backlog_estimate + promql: | + max by (endpoint) (http_requests_in_flight) + season_length: 10 + thresholds: + - name: drained + value: 0 + comparator: le_drain # p95 latency derived from the histogram. Forecasting tail latency # is one of the more interesting demos because it's both noisy and # tied to load. diff --git a/docs/README.md b/docs/README.md index efb72a5..28f6c3c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,13 +15,18 @@ The docs follow the [Diataxis](https://diataxis.fr/) framework: - [CLI](cli.md) — `promforecast run`, `promforecast validate`, `promforecast diagnose` - [Reload verification](operations/reload.md) — confirming a config reload actually picked up the expected queries - [Duplicate-labelset diagnosis](operations/diagnose.md) — detecting and remediating the scrape+sink dual-source state +- [Cardinality budgeting](operations/cardinality.md) — per-family multipliers and the `--strict-cardinality` CI gate - [High availability](ha.md) — Lease-based leader election, Redis snapshot cache - [Query cache](cache.md) — in-memory LRU + optional Redis dedup for overlapping PromQL - [Series discovery](discovery.md) — template-driven auto-fan-out over label values - [Preprocessing](preprocessing.md) — gap imputation and outlier scrubbing - [Change-points](change-points.md) — input-side break detection and lookback adaptation - [Conformal calibration](conformal.md) — empirical-coverage band correction -- [Capacity planning](capacity.md) — damped trend, time-to-threshold ETAs, growth rate +- [Capacity planning](capacity.md) — damped trend, time-to-threshold ETAs, growth rate, will-breach predictive boolean, queue burn-down (drain) forecasting +- [SLO error-budget forecasting](slo.md) — burn rate, exhaustion projection, multi-window alerting +- [Maintenance window recommendations](operations/maintenance-windows.md) — top-N lowest-load windows for scheduling reboots, rollouts, and cert rotations +- [Conditional forecasts](conditional-forecasts.md) — baseline given the current regressor state, for "unusual *for current conditions*" alerts +- [Explainability (decomposition)](explainability.md) — trend / seasonal / level / residual split of the forecast curve - [Forecast drift](drift.md) — prediction-vs-prediction shift telemetry - [Reading the numbers](reading-the-numbers.md) — how to interpret quality and accuracy in practice (mean vs p95, horizon paradox, "wait three cycles", when to actually worry) - [Scheduling](scheduling.md) — per-query refresh intervals, max-concurrent-fits cap diff --git a/docs/architecture/emission-paths.md b/docs/architecture/emission-paths.md index e128026..b571c5b 100644 --- a/docs/architecture/emission-paths.md +++ b/docs/architecture/emission-paths.md @@ -111,9 +111,9 @@ The official charts ship that combination on by default: - The **umbrella `promforecast-stack` chart** sets `promforecast.config.sink.remote_write.enabled: true` and `promforecast.serviceMonitor.enabled: true`. The forecaster's - ServiceMonitor (Story 2) then derives the snapshot drop from the - sink toggle automatically, so a fresh `helm install` lands on - sink+VM with the dual-source trap closed. + ServiceMonitor then derives the snapshot drop from the sink toggle + automatically, so a fresh `helm install` lands on sink+VM with the + dual-source trap closed. - The **standalone `promforecast` chart** exposes `serviceMonitor.dropForecastSnapshots` (default `null`). When `null` it resolves to the value of `config.sink.remote_write.enabled`: diff --git a/docs/capacity.md b/docs/capacity.md index bb08fda..eb7ea31 100644 --- a/docs/capacity.md +++ b/docs/capacity.md @@ -121,6 +121,55 @@ ETA is reported as **0** (read: already in violation). - **Quota exhaustion**: forecast the in-use side of any quota, `comparator: gt`, `value` = the quota cap. +### Predictive will-breach boolean + +`time_to_threshold_seconds < 6h` is a perfectly good alert, but every +operator who writes one re-derives the same arithmetic. Set +`alert_within` + `severity` on a threshold and the forecaster emits a +boolean shaped exactly for `forecast_will_breach == 1`: + +```yaml +queries: + - id: node_filesystem_avail_bytes + promql: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} + thresholds: + - name: critical + value: 5368709120 # 5 GiB + comparator: lt + alert_within: 6h # opt into the will_breach gauge + severity: critical +``` + +| Field | Required | Meaning | +|------------------|------------------------|--------------------------------------------------------------------------------------| +| `alert_within` | only when paired | Lookahead window. The boolean fires whenever the ETA is finite and below this value. | +| `severity` | only when paired | Becomes the `severity` label on the emitted metric. | + +Both fields must be set together — the predictive metric is a derived +shape and partial config is rejected at load time. The metric is emitted +even when the predicate is `0`, so dashboards have a baseline series to +chart from boot: + +``` +node_filesystem_avail_bytes_forecast_will_breach{ + name="critical", severity="critical", level="95", + group="node_capacity", model="AutoETSDamped", instance="host-a" +} 1.0 +``` + +The same fan-out as the ETA family applies: a row for the central +forecast (no `level` label) plus one row per confidence level so +operators can choose how conservative to be. Alerts read like: + +```promql +forecast_will_breach{severity="critical", level="95"} == 1 +``` + +The corresponding `forecast_time_to_threshold_seconds` value at the +same labelset is the operator-visible "when" for the breach. See +`examples/alerts/promforecast-rules.yaml` for the `ForecastWillBreach` +rule template. + ## Forecast growth rate Configure per-query growth-rate emission to publish how fast the @@ -170,14 +219,77 @@ window — the alert query you wrote with `window="7d"` keeps meaning what it said. Configure shorter windows or extend `defaults.horizon` if you need them. +## Queue burn-down / drain forecasting + +A queue (Kafka consumer lag, Sidekiq job backlog, alert backlog) that +*should* drain over time is the inverse of capacity: instead of +"when will this hit X from below?", you want "when will this hit +zero from above?". Use the `le_drain` comparator on a threshold to +opt into the dedicated drain ETA family: + +```yaml +queries: + - id: kafka_consumer_lag + promql: sum by (topic) (kafka_consumergroup_lag{group="orders"}) + thresholds: + - name: drained + value: 0 + comparator: le_drain +``` + +Output (note the different metric name — `_forecast_time_to_drain_seconds`, +not `_time_to_threshold_seconds`): + +``` +kafka_consumer_lag_forecast_time_to_drain_seconds{ + name="drained", group="kafka", topic="orders" +} 5400 +``` + +`le_drain` is mechanically the same predicate as `lt` (value drops +below threshold), but with three deliberate differences: + +1. **Different metric name.** Emits under + `_forecast_time_to_drain_seconds`, parallel to the + threshold-ETA family. The two never collide on the same query. +2. **`+Inf` instead of NaN on no-crossing.** When the forecast curve + never crosses the threshold from above within the horizon, the + sample reads `+Inf`. Operators alert with + `forecast_time_to_drain_seconds == +Inf for 30m` to catch "queue + is not actually draining" as a distinct condition from "draining + slowly". With NaN (the standard `lt` behaviour), the alert would + silently never match. +3. **`yhat_upper` for the conservative band.** A larger predicted + queue drains later, so `yhat_upper` is the *latest* predicted + drain — the conservative answer when alerting on a stuck queue. + (`lt` uses `yhat_lower` for the earliest-crossing semantics that + capacity alerts want.) + +The shipped `examples/alerts/promforecast-rules.yaml` includes a +`promforecast.queue.examples` group with two reference alerts: + +```promql +# Queue is not actually draining +kafka_consumer_lag_forecast_time_to_drain_seconds{name="drained"} == +Inf + +# Drain projected to take >4h (and is happening at all) +kafka_consumer_lag_forecast_time_to_drain_seconds{name="drained"} > 4 * 60 * 60 +and +kafka_consumer_lag_forecast_time_to_drain_seconds{name="drained"} != +Inf +``` + ## Cardinality -Both new families count against the global `safety.max_total_series` -cap; the cardinality estimator in `promforecast validate` includes -them: +All the capacity-style families count against the global +`safety.max_total_series` cap; the cardinality estimator in +`promforecast validate` includes them: - Threshold ETA: `N_series × N_models × N_thresholds × (1 + N_levels)` - per query that configures thresholds. + per query that configures thresholds (excluding `le_drain`). +- Drain ETA: same shape, charged separately. Only `le_drain` thresholds + count. +- Will-breach: `N_series × N_models × N_thresholds_with_alert × (1 + N_levels)` + per query — only thresholds with `alert_within` + `severity` count. - Growth rate: `N_series × N_models × N_windows` per query that opts in. Run `promforecast validate config.yaml` after adding thresholds or diff --git a/docs/conditional-forecasts.md b/docs/conditional-forecasts.md new file mode 100644 index 0000000..cc96ff9 --- /dev/null +++ b/docs/conditional-forecasts.md @@ -0,0 +1,66 @@ +# Conditional forecasts + +The default forecast answers: “Is the actual inside the historic envelope?” +That works for most cases, but breaks when a known regressor legitimately shifts the metric (e.g. a deploy that *should* increase request rate). + +**Conditional forecasts** answer: “Given the *current* state of the regressors, what should the metric look like now?” +Use the conditional deviation when you want to detect anomalies *after accounting for known external drivers*. + +## Configuration + +Per-query, off by default: + +```yaml +groups: + - name: http + queries: + - id: http_requests_rate + promql: sum(rate(http_requests_total[1m])) by (endpoint) + regressors: + - id: deploys + promql: changes(kube_deployment_status_observed_generation[1h]) + emission: + conditional: true +``` + +The runner re-fits each emitted model while holding the most recent regressor values constant. + +## Emitted metrics + +| Metric | Description | +|---------------------------------------------|-----------| +| `_forecast_conditional` | Point forecast at horizon | +| `_forecast_conditional_lower{level}` | Lower band per level | +| `_forecast_conditional_upper{level}` | Upper band per level | +| `forecast_conditional_deviation_ratio` | Signed deviation of latest actual | +| `forecast_conditional_deviation_outside_band` | 1 if actual is outside the conditional band | + +These carry the same labels as the unconditional deviation family, so you can alert on them directly: + +```promql +# Anomaly after accounting for known regressors +forecast_conditional_deviation_outside_band{level="95"} == 1 + and on (id, group, model) forecast_quality_score > 0.6 +``` + +## Model behaviour + +Built-in models (`AutoARIMA`, `AutoETS`, `MSTL`, `SeasonalNaive`, etc.) do **not** use regressors. +They advertise `uses_regressors = False`, so the runner **short-circuits** the conditional refit and aliases the unconditional forecast. + +You still get a stable schema (conditional metrics always exist), but at **zero extra CPU cost**. + +Regressor-aware models (e.g. Prophet plug-in) perform a real second fit with held-constant regressors. + +## Cardinality + +Roughly doubles the forecast snapshot footprint for opted-in queries. +`promforecast validate` includes the extra lines in its estimate. + +See [operations/cardinality.md](operations/cardinality.md) for the full multiplier table. + +## When not to use it + +- No regressors on the query (conditional becomes a pure duplicate). +- No model in the set uses regressors (wastes cardinality). +- You want classic anomaly detection that treats regressor shifts as real deviations. diff --git a/docs/explainability.md b/docs/explainability.md new file mode 100644 index 0000000..970ad8c --- /dev/null +++ b/docs/explainability.md @@ -0,0 +1,61 @@ +# Forecast explainability + +Two related questions get two different metrics: + +* "Which **regressor** is moving the forecast?" → [`_forecast_contribution_ratio`](regressors.md) +* "Which **model component** is driving the curve?" → the decomposition family below + +The decomposition projects the forecast onto `trend / seasonal / level / residual` so you can read “disk fills in 14 days because the trend steepened last week” directly from a stacked-area Grafana panel. + +## Configuration + +Enabled per query, off by default: + +```yaml +queries: + - id: node_filesystem_avail_bytes + promql: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} + emission: + decompose: true +``` + +**Output** (one gauge per component at end-of-horizon): + +``` +_forecast_component{component="trend", group, model, ...} +_forecast_component{component="seasonal", group, model, ...} +_forecast_component{component="level", group, model, ...} +_forecast_component{component="residual", group, model, ...} +``` + +The four components always sum to `yhat` (within floating-point tolerance) for each series/model. + +## Component meanings + +| Component | What it captures | +|------------|------------------| +| **level** | In-sample baseline (constant across horizon) | +| **trend** | Linear slope across the horizon (anchored at 0 on first step) | +| **seasonal** | Repeating deviation within the seasonal cycle | +| **residual** | Remaining structure + model-specific effects | + +## Non-decomposable models + +Intermittent-demand models (`Croston`, `CrostonSBA`, `ADIDA`, `IMAPA`) do not support decomposition. Setting `emission.decompose: true` on queries using them increments: + +``` +forecast_decomposition_skipped_total{group, query, model, reason="unsupported_model"} +``` + +Other `reason` values: `fit_failed`, `components_unavailable`, `invariant_violation`. + +## Cardinality + +Adds **4×** samples per (series, model) that has decomposition enabled. +`promforecast validate` includes this in the cardinality estimate. + +## When not to use it + +- On `data_profile: intermittent` series — components lack meaning. +- On already-understood signals (e.g. clean sinusoidal patterns). +- When dashboard space or cardinality budget is tight and no stacked-area panel will consume the data. diff --git a/docs/operations/cardinality.md b/docs/operations/cardinality.md new file mode 100644 index 0000000..7688858 --- /dev/null +++ b/docs/operations/cardinality.md @@ -0,0 +1,77 @@ +# Cardinality budgeting + +promforecast emits many optional metric families. Turning several on at once multiplies the total series count faster than the input data suggests. This page gives you the multiplier table and the recommended workflow to size your budget before shipping a config. + +If your `/metrics` scrape is getting heavy or `forecast_series_dropped_total{reason="total_cap"}` is ticking, start here. + +## How the safety budget works + +- `safety.max_series_per_query` — caps each individual query. +- `safety.max_total_series` — global cap across all groups (sliced by `priority`). +- When exceeded, `series_overflow` (default: `drop_lowest_priority`) drops excess series. + +Dropped series are tracked in `forecast_series_dropped_total{reason="per_query_cap" | "total_cap"}`. + +## Estimator: `promforecast validate` + +```bash +promforecast validate --config /etc/promforecast/config.yaml + +# Fail CI on oversized configs +promforecast validate --config config.yaml --strict-cardinality +``` + +The validator assumes worst-case (`safety.max_series_per_query` per query) and prints a detailed breakdown. Use `--strict-cardinality` to turn the `WARN:` line into a non-zero exit code. + +## Multiplier table + +Notation: +- `S` = `safety.max_series_per_query` (worst-case per query) +- `M` = emitted models per series (1 with `auto_select`, all configured models otherwise) +- `L` = number of `confidence_levels` +- `H` = number of `accuracy.horizons` + +| Family | Multiplier per opted-in query | Notes | +|---------------------------------------------|----------------------------------------|-------| +| `_forecast` + lower + upper | `S × M × (1 + 2L)` | Always on | +| `forecast_accuracy_mape` + `mase` | `S × M_models × H × 2` | Requires `accuracy.evaluate: true` | +| `forecast_deviation_*` | `S × M × L × 2` | `emission.deviation: true` | +| `forecast_quality_score` | `S × M` | `emission.quality: true` | +| `_forecast_contribution_ratio` | `S × N_regressors` | Only with regressors | +| `forecast_band_calibration_offset` | `S × M × L` | `accuracy.conformal.enabled: true` | +| `_forecast_component` | `S × M × 4` | `emission.decompose: true` (trend/seasonal/level/residual) | +| `forecast_drift_score` | `S × M` | `emission.drift.enabled: true` | +| Threshold ETA / will_breach | `S × M × N_thresholds × (1 + L)` | Only configured thresholds | +| Drain ETA (`le_drain`) | `S × M × N_drain_thresholds × (1 + L)` | Charged separately from the standard ETA | +| Growth rate | `S × M × N_windows` | `emission.growth_rate.enabled: true` | +| Conditional forecast families | `S × M × (1 + 2L)` | `emission.conditional: true` (may add extra fit) | +| SLO error budget | `S × M × N_windows × (3 + L)` | `slo:` configured (3 sibling families × N_windows, plus level fan-out on exhausted_in) | +| Recommended maintenance windows | `S × M × count` | `recommended_windows.enabled: true` | +| Operational + preprocessing families | ~7 per group + ~6 per query | Mostly always on | + +## Working example + +**Setup:** 4 queries (50 series each), 2 models, 2 confidence levels, deviation + quality + decompose + conditional + 2 thresholds + growth rate. + +**Projected total ≈ 17 200 lines** + +With default `safety.max_total_series: 5000` this would fail `--strict-cardinality`. Solutions: +- Raise the cap +- Reduce models (`auto_select: true`) +- Turn off unused families +- Add `topk()` to high-cardinality PromQL queries + +## Recommended posture + +1. **Run `promforecast validate --strict-cardinality` on every config PR.** +2. Treat `safety.max_total_series` as a real knob — the default 5000 is for minimal setups. +3. Prefer `auto_select: true` to keep `M = 1`. +4. Monitor `forecast_series_dropped_total{reason="total_cap"}`. +5. Use `topk(N, ...)` in PromQL for queries prone to label cardinality spikes. + +## See also + +- [Architecture — emission paths](../architecture/emission-paths.md) +- [Capacity planning](../capacity.md) +- [Conditional forecasts](../conditional-forecasts.md) +- [Explainability](../explainability.md) diff --git a/docs/operations/maintenance-windows.md b/docs/operations/maintenance-windows.md new file mode 100644 index 0000000..e34b1da --- /dev/null +++ b/docs/operations/maintenance-windows.md @@ -0,0 +1,79 @@ +# Maintenance window recommendations + +The forecaster already knows future load. This feature turns that knowledge into a ranked list of the **quietest time windows** of a given duration within a lookahead horizon — eliminating guesswork when scheduling maintenance. + +## Configuration + +Add to any load-related query: + +```yaml +groups: + - name: node_capacity + queries: + - id: node_cpu_busy_pct + promql: | + 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) + recommended_windows: + enabled: true + duration: 30m + lookahead: 48h + count: 3 +``` + +| Field | Default | Meaning | +|-------------|---------|---------| +| `enabled` | `false` | Opt-in (family is off by default) | +| `duration` | `30m` | Length of each recommended window | +| `lookahead` | `48h` | How far into the future to search | +| `count` | `3` | Number of windows to recommend, ranked by lowest predicted load | + +## Emitted metrics + +One global family (no per-id prefix): + +``` +forecast_recommended_maintenance_window{ + id="node_cpu_busy_pct", + group="node_capacity", + model="AutoARIMA", + rank="1", + duration_seconds="1800", + start_timestamp_seconds="1763020800" +} 12.3 +``` + +- **Value** = predicted average load during the window (lower is better). +- `rank` — 1 is the best (quietest) window. +- Ties are broken by earliest start time for stable ranking. + +## Worked examples + +- **Database reboot** — `duration: 5m`, `lookahead: 24h`, `count: 3` +- **Certificate rotation** — `duration: 15m`, `lookahead: 72h`, `count: 5` +- **Fleet-wide security patch** — `duration: 1h`, `lookahead: 7d`, `count: 5` + +## Grafana usage + +The bundled capacity dashboard includes a table panel showing top-N windows. +When using multiple models, filter to one (e.g. the winner): + +```promql +forecast_recommended_maintenance_window{ + id="node_cpu_busy_pct", + model="AutoARIMA" +} +``` + +Convert `start_timestamp_seconds` to a readable time using Grafana value mappings. + +## Cardinality + +`count` rows per (series, model) per opted-in query. +With defaults (`count: 3`) and 500 series this adds ~1500 rows per query — bounded by design. + +The `start_timestamp_seconds` label changes over time. Drop it via `metric_relabel_configs` if long-term retention becomes an issue. + +## What this is not + +- **Not automation** — emits metrics only. Wire your scheduler via Prometheus/Grafana alerts if needed. +- **Not workload-aware** — based purely on the forecast. Does not consider dependencies, business impact, or maintenance fatigue. Use as candidate windows, not final commitments. diff --git a/docs/slo.md b/docs/slo.md new file mode 100644 index 0000000..3964414 --- /dev/null +++ b/docs/slo.md @@ -0,0 +1,71 @@ +# SLO error-budget forecasting + +Instead of only alerting on “how much error budget have I burned?”, forecast your error-rate metric to answer “at the current predicted burn rate, when will the budget run out?” + +This gives you forward-looking SLO exhaustion projections that match the same trajectory style used for capacity forecasting. + +## Configuration + +Add an `slo:` block to any error-rate query: + +```yaml +groups: + - name: http_slo + queries: + - id: http_request_error_ratio + promql: | + sum(rate(http_requests_total{status=~"5.."}[5m])) + / sum(rate(http_requests_total[5m])) + slo: + objective: 99.9 + windows: [5m, 1h, 30d] +``` + +| Field | Default | Meaning | +|-------------|-------------|---------| +| `objective` | required | SLO target as percentage (e.g. `99.9`) | +| `windows` | `[30d]` | One or more SLO windows (short + long windows recommended) | + +## Emitted metrics + +Per `(series, model, window)`: + +| Metric | Description | +|--------|-----------| +| `_forecast_error_budget_remaining_seconds{window}` | Static total budget for the window | +| `_forecast_error_budget_burn_rate{window}` | Predicted burn rate (1.0 = exactly on pace) | +| `_forecast_error_budget_exhausted_in_seconds{window}` | Seconds until budget exhaustion (central prediction) | +| `_forecast_error_budget_exhausted_in_seconds{window, level}` | Per-confidence-level exhaustion (uses upper band) | + +- `burn_rate > 1` → consuming budget faster than allowed. +- `exhausted_in_seconds = +Inf` when predicted error rate is zero. + +## Example alerts + +```promql +# Budget projected to exhaust in <24h +http_request_error_ratio_forecast_error_budget_exhausted_in_seconds{window="30d"} < 86400 + +# Fast burn: 30d budget gone in <2 days +http_request_error_ratio_forecast_error_budget_burn_rate{window="30d"} > 14.4 +``` + +Configure multiple windows (`[5m, 1h, 30d]`) to cover both fast-burn and slow-drift cases. + +## Cardinality + +Roughly `(3 + N_levels) × N_windows` rows per (series, model) for each opted-in query. +A query with 500 series, 1 model, 2 levels, and 3 windows adds ~7 500 rows. +`promforecast validate` accounts for this in its estimate. + +## Implementation notes + +- Projections are **point-in-time** (based on the leading forecast row), not integrated over the full window. This matches operator intuition for “current predicted rate”. +- Uses `yhat_upper` for pessimistic exhaustion time when confidence levels are configured. +- Works best with `accuracy.conformal.enabled: true` for calibrated bands. + +## What this is not + +- Not a replacement for proper SLI instrumentation. +- Not a full state machine (does not integrate observed burn so far — pair with recording rules if needed). +- Not an alerting engine — it only emits metrics for Alertmanager rules. diff --git a/examples/alerts/promforecast-rules.yaml b/examples/alerts/promforecast-rules.yaml index cee7d6c..5348df1 100644 --- a/examples/alerts/promforecast-rules.yaml +++ b/examples/alerts/promforecast-rules.yaml @@ -262,6 +262,30 @@ spec: expected to cross the configured ``warn`` threshold within 4 hours. Right-size before SLO-impacting saturation. + # Predictive will-breach alert. Reads the boolean directly so the + # rule stays a one-liner — the heavy lifting (ETA + alert_within + # comparison) is done by the forecaster. Pin ``level="95"`` so the + # rule fires once per series on the conservative band; the + # corresponding ``forecast_time_to_threshold_seconds`` value at the + # same labelset is the operator-visible "when" for the breach. + - alert: ForecastWillBreach + expr: | + node_filesystem_avail_bytes_forecast_will_breach{ + severity="critical", level="95" + } == 1 + for: 10m + labels: + severity: critical + annotations: + summary: "{{ $labels.instance }}{{ $labels.mountpoint }} will breach within the configured alert_within window" + description: | + {{ $labels.id }} for {{ $labels.instance }} + {{ $labels.mountpoint }} (95% band) is predicted to cross the + ``{{ $labels.name }}`` threshold inside the configured + ``alert_within`` window. Cross-reference + ``forecast_time_to_threshold_seconds`` at the same label set + for the exact ETA. + # Growth-rate-change alert. Compares the 1d window slope # against the 7d baseline; magnitudes are kept absolute so the # alert catches sudden *deceleration* of a free-space metric @@ -288,3 +312,95 @@ spec: ({{ $labels.mountpoint }}) is more than 3x the 7-day baseline. Investigate whether a workload change or leak is driving accelerated consumption. + + - name: promforecast.slo.examples + interval: 1m + rules: + # SLO error budget will be exhausted at the current burn rate. + # Generic template aligned with the SRE workbook's burn-rate + # alerts — pair short / long windows on a single query to get + # multi-window/multi-burn-rate coverage. The example here uses + # a single 30d window; a real deployment would also configure + # 5m and 1h windows on the same query and alert on each. + - alert: SLOBudgetExhaustingSoon + expr: | + http_request_error_ratio_forecast_error_budget_exhausted_in_seconds{ + window="30d" + } < 86400 + for: 1h + labels: + severity: critical + annotations: + summary: "{{ $labels.id }} 30d error budget projected to exhaust in <24h" + description: | + At the predicted burn rate, the configured 30d error + budget for {{ $labels.id }} will be consumed within 24 + hours. Investigate the source of elevated errors and + consider whether to slow the error rate or accept the + budget overrun. + + # Fast burn — the predicted rate is multiples of the + # sustainable rate. ``burn_rate > 14.4`` corresponds to "the + # entire 30d budget would be exhausted in 2 days at this + # rate"; tune for your SLO and on-call sensitivity. + - alert: SLOFastBurn + expr: | + http_request_error_ratio_forecast_error_budget_burn_rate{ + window="30d" + } > 14.4 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.id }} burning the 30d error budget at >14.4x sustainable rate" + description: | + At the predicted burn rate for {{ $labels.id }}, the + monthly budget would be consumed in roughly 2 days. + Cross-reference with ``..._exhausted_in_seconds`` for the + projected exhaustion time. + + - name: promforecast.queue.examples + interval: 1m + rules: + # Queue is not actually draining — the projected curve is flat + # or growing, so the drain ETA is +Inf. Without this rule a + # stuck consumer is invisible because alerts on + # ``forecast_time_to_drain_seconds < N`` silently never match. + - alert: QueueNotDraining + expr: | + kafka_consumer_lag_forecast_time_to_drain_seconds{ + name="drained" + } == +Inf + for: 30m + labels: + severity: warning + annotations: + summary: "{{ $labels.id }} is not projected to drain within the horizon" + description: | + The forecast for {{ $labels.id }} predicts no crossing + below the configured drain threshold within the horizon + — the queue is not actually being worked off. Check + consumer health and parallelism. + + # Drain projected to happen far in the future. Useful as an + # early-warning alert on a backlog that *will* eventually + # drain but not before SLAs are at risk. + - alert: QueueDrainSlow + expr: | + kafka_consumer_lag_forecast_time_to_drain_seconds{ + name="drained" + } > 4 * 60 * 60 + and + kafka_consumer_lag_forecast_time_to_drain_seconds{ + name="drained" + } != +Inf + for: 30m + labels: + severity: info + annotations: + summary: "{{ $labels.id }} projected to take >4h to drain" + description: | + The forecast for {{ $labels.id }} predicts a drain ETA + above 4h. The queue is being worked off, but the + backlog is too large to clear before downstream SLAs + are at risk. Add consumer capacity or shed load. diff --git a/examples/configs/business-kpis.yaml b/examples/configs/business-kpis.yaml index 44473be..48cd10f 100644 --- a/examples/configs/business-kpis.yaml +++ b/examples/configs/business-kpis.yaml @@ -46,6 +46,28 @@ groups: promql: | sum by (service) (rate(http_requests_total{status=~"5.."}[5m])) + # Error ratio with SLO error-budget forecasting. The forecast is + # the predicted error rate; the slo block turns it into burn rate + # and exhaustion ETAs. Pair short and long windows for + # multi-window/multi-burn-rate alerting per the SRE workbook. + - id: http_request_error_ratio + promql: | + sum by (service) (rate(http_requests_total{status=~"5.."}[5m])) + / + sum by (service) (rate(http_requests_total[5m])) + slo: + objective: 99.9 + windows: [5m, 1h, 30d] + # Suggest the lowest-load 15-minute windows for scheduling + # certificate rotations or other brief downtime; rate of + # requests is a reasonable proxy for impact when the dominant + # cost is request count. + recommended_windows: + enabled: true + duration: 15m + lookahead: 48h + count: 3 + - id: http_request_p95_seconds promql: | histogram_quantile(0.95, diff --git a/examples/configs/kube-resources.yaml b/examples/configs/kube-resources.yaml index 34f83c5..f6a9f05 100644 --- a/examples/configs/kube-resources.yaml +++ b/examples/configs/kube-resources.yaml @@ -27,3 +27,16 @@ groups: - id: persistentvolumeclaim_used_bytes promql: | sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes) + + # Queue burn-down example. The le_drain comparator changes the + # ETA family in two ways: the metric name becomes + # ``..._forecast_time_to_drain_seconds`` (parallel to the + # standard ETA), and the no-crossing sentinel is +Inf rather + # than NaN so the "queue is not draining" alert can match it. + - id: kafka_consumer_lag + promql: | + sum by (consumergroup, topic) (kafka_consumergroup_lag) + thresholds: + - name: drained + value: 0 + comparator: le_drain diff --git a/forecaster/src/promforecast/_time_axis.py b/forecaster/src/promforecast/_time_axis.py new file mode 100644 index 0000000..d064a61 --- /dev/null +++ b/forecaster/src/promforecast/_time_axis.py @@ -0,0 +1,40 @@ +"""Shared time-axis helper for forecast-frame post-processing. + +Both :mod:`promforecast.capacity` (threshold ETAs, drain ETAs, growth +rates) and :mod:`promforecast.maintenance` (recommended windows) need to +project a forecast's ``ds`` column onto seconds-from-the-leading-row. +The conversion is identical, defensively handles malformed timestamps +the same way, and lives here so the two callers can stay in sync. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import pandas as pd + + +def seconds_from_start(ds: pd.Series[Any]) -> list[float] | None: + """Return per-row seconds since the leading timestamp, or ``None``. + + Anchors at row 0 because the operator's mental model for everything + derived from a forecast curve is "time from now", and "now" is the + moment the forecast was issued. ``None`` is returned if the input is + empty or any row carries a value that doesn't quack like a datetime + (no ``.timestamp()``), so the caller can treat the failure mode the + same way an empty forecast is treated. + """ + if ds.empty: + return None + try: + start_ts = float(ds.iloc[0].timestamp()) + except (AttributeError, ValueError, TypeError): + return None + out: list[float] = [] + for item in ds.tolist(): + try: + out.append(float(item.timestamp()) - start_ts) + except (AttributeError, ValueError, TypeError): + return None + return out diff --git a/forecaster/src/promforecast/capacity.py b/forecaster/src/promforecast/capacity.py index 715916e..31a40fa 100644 --- a/forecaster/src/promforecast/capacity.py +++ b/forecaster/src/promforecast/capacity.py @@ -26,16 +26,27 @@ import math from collections.abc import Callable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Literal + +from ._time_axis import seconds_from_start as _seconds_from_start if TYPE_CHECKING: import pandas as pd from .config import ThresholdConfig -Comparator = Literal["gt", "lt"] +Comparator = Literal["gt", "lt", "le_drain"] GrowthUnits = Literal["units_per_second", "percent"] +# Comparators that emit the dedicated "queue is draining" ETA family +# (``_forecast_time_to_drain_seconds``) instead of the standard +# threshold ETA. The drain family reports ``+inf`` on no-crossing — +# operators alert with ``... == +Inf for 30m`` to catch "queue is not +# actually draining". The lt/gt families report NaN on no-crossing +# (dropped at render time) because the typical alert against them is +# ``... < N`` and a missing sample is the right "no concern" signal. +DRAIN_COMPARATORS: frozenset[str] = frozenset({"le_drain"}) + # Forecast frames smaller than this can't define a meaningful slope or # crossing — one row is a point, not a curve. _MIN_ROWS_FOR_DURATION = 2 @@ -67,6 +78,66 @@ class GrowthRate: rate: float +@dataclass(frozen=True) +class WillBreach: + """0/1 predictive sample derived from an ETA + an ``alert_within`` window. + + ``value`` is ``1.0`` when the threshold ETA is finite and below the + operator's ``alert_within`` lookahead, ``0.0`` otherwise. ``level`` is + ``None`` for the central forecast, a numeric level for band-edge + samples — same convention as :class:`ThresholdEta`. + """ + + name: str + severity: str + level: int | None + value: float + + +def compute_will_breach( + *, + etas: list[ThresholdEta], + thresholds: list[ThresholdConfig], +) -> list[WillBreach]: + """Project ``etas`` onto 0/1 will-breach samples for the configured thresholds. + + Walks the (threshold, level) ETAs already produced by + :func:`compute_threshold_etas` and emits a :class:`WillBreach` for + each (threshold, level) whose threshold opted into the predictive + family (``alert_within`` + ``severity`` both set). ETAs whose + threshold did not opt in are skipped; non-finite ETAs (no crossing + in the horizon) emit ``0`` rather than being suppressed so an alert + on ``forecast_will_breach == 1`` has a stable zero-baseline series + to chart against. + """ + if not etas or not thresholds: + return [] + by_name = { + t.name: t for t in thresholds if t.alert_within is not None and t.severity is not None + } + if not by_name: + return [] + out: list[WillBreach] = [] + for eta in etas: + threshold = by_name.get(eta.name) + if threshold is None: + continue + # mypy: narrowed by ``by_name`` filter above. + assert threshold.alert_within is not None + assert threshold.severity is not None + window_seconds = threshold.alert_within.total_seconds() + value = 1.0 if math.isfinite(eta.seconds) and eta.seconds < window_seconds else 0.0 + out.append( + WillBreach( + name=eta.name, + severity=threshold.severity, + level=eta.level, + value=value, + ) + ) + return out + + def compute_threshold_etas( *, forecast: pd.DataFrame, @@ -112,10 +183,17 @@ def compute_threshold_etas( ), ) ) - # The conservative band side depends on the direction: for "above" - # crossings, the *upper* band tells you the earliest possible - # crossing; for "below", the *lower* band does. - band_side = "yhat_upper" if comparator == "gt" else "yhat_lower" + # The conservative band side depends on the direction: + # * ``gt`` (e.g. "disk fills"): ``yhat_upper`` reports the + # earliest possible crossing. + # * ``lt`` (e.g. "free space drops below floor"): + # ``yhat_lower`` reports the earliest possible crossing. + # * ``le_drain`` (e.g. "queue drains to zero from above"): + # ``yhat_upper`` is the *latest* drain time — the conservative + # answer when alerting on "queue is not draining". A drain + # that uses the lower band would tell us the optimistic + # answer, which is the opposite of what the alert needs. + band_side = "yhat_upper" if comparator in ("gt", "le_drain") else "yhat_lower" for level in levels: col = f"{band_side}_{level}" if col not in forecast.columns: @@ -178,30 +256,6 @@ def compute_growth_rates( return out -def _seconds_from_start(ds: pd.Series[Any]) -> list[float] | None: - """Return per-row seconds since the first timestamp, or None on failure. - - Forecast rows always carry timestamps in monotonically increasing - order. We anchor at row 0 because the operator's mental model is - "ETA = time from now", and "now" is the moment the forecast was - issued. - """ - if ds.empty: - return None - try: - start = ds.iloc[0] - start_ts = float(start.timestamp()) - except (AttributeError, ValueError, TypeError): - return None - out: list[float] = [] - for item in ds.tolist(): - try: - out.append(float(item.timestamp()) - start_ts) - except (AttributeError, ValueError, TypeError): - return None - return out - - def _first_crossing( *, seconds: list[float], @@ -209,7 +263,7 @@ def _first_crossing( threshold: float, comparator: Comparator, ) -> float: - """Return the seconds-from-start of the first crossing, or NaN. + """Return the seconds-from-start of the first crossing, or NaN / +Inf. A "crossing" is the first adjacent pair where the predicate (``> v`` or ``< v``) flips from False to True. Linear interpolation between @@ -217,12 +271,26 @@ def _first_crossing( behaviour operators expect from reading a continuous curve in Grafana. If the series starts already on the wrong side of the threshold, the ETA is 0. + + The no-crossing sentinel depends on the comparator: + + * ``gt`` / ``lt``: NaN (dropped at render time so an alert on + ``... < N`` doesn't silently match). + * ``le_drain``: ``+inf``. Operators alert on + ``... == +Inf for 30m`` to catch "queue is not draining" — the + sentinel must round-trip through Prometheus and stay queryable + rather than being filtered out at render time. """ if not values or not seconds or len(values) != len(seconds): - return float("nan") - triggered: Callable[[float], bool] = ( - (lambda v: v > threshold) if comparator == "gt" else (lambda v: v < threshold) - ) + return _no_crossing_sentinel(comparator) + # ``gt`` and ``lt`` use strict inequality (a value exactly at the + # threshold is "approaching", not "crossed"). ``le_drain`` uses + # non-strict ``<=`` so a forecast that touches the drain value + # (e.g. a queue forecast that hits exactly zero) registers as + # drained — otherwise an already-empty queue would report + # ``+Inf`` "not draining" and the QueueNotDraining alert would + # fire on a healthy queue. + triggered = _triggered_for(comparator, threshold) if triggered(values[0]) and math.isfinite(values[0]): return 0.0 for i in range(1, len(values)): @@ -238,7 +306,43 @@ def _first_crossing( ratio = (threshold - prev_v) / span ratio = max(0.0, min(1.0, ratio)) return seconds[i - 1] + ratio * (seconds[i] - seconds[i - 1]) - return float("nan") + return _no_crossing_sentinel(comparator) + + +def _triggered_for(comparator: Comparator, threshold: float) -> Callable[[float], bool]: + """Return the per-step predicate that decides "has the curve crossed?" + + Strict inequality for ``gt`` / ``lt`` (a value exactly at the + threshold is approaching, not crossed). Non-strict ``<=`` for + ``le_drain`` so a queue forecast that lands exactly on the drain + value is recognised as drained. + """ + if comparator == "gt": + + def gt_predicate(v: float) -> bool: + return v > threshold + + return gt_predicate + if comparator == "le_drain": + + def le_predicate(v: float) -> bool: + return v <= threshold + + return le_predicate + + def lt_predicate(v: float) -> bool: + return v < threshold + + return lt_predicate + + +def _no_crossing_sentinel(comparator: Comparator) -> float: + """Sentinel value for a curve that never crosses ``threshold``. + + Centralised so the two callers (the leading-row triggered? guard + and the loop exit) agree on which sentinel each comparator uses. + """ + return float("inf") if comparator in DRAIN_COMPARATORS else float("nan") def _slope_over_window( diff --git a/forecaster/src/promforecast/config.py b/forecaster/src/promforecast/config.py index af974a5..e474769 100644 --- a/forecaster/src/promforecast/config.py +++ b/forecaster/src/promforecast/config.py @@ -375,13 +375,114 @@ class ThresholdConfig(BaseModel): so capacity ETAs are conservative at the upper band — "disk fills no later than 24h from now" rather than "the central forecast hits full in 24h". + + ``alert_within`` + ``severity`` opt the threshold into the + ``_forecast_will_breach`` predictive boolean: a derived 0/1 gauge + that fires when the time-to-threshold ETA is below ``alert_within``. + Both fields are required together (one without the other is + rejected at load time) — the predictive metric is a derived shape and + cannot be partially configured. + + The ``le_drain`` comparator targets the queue-burn-down case. It + differs from ``lt`` along four axes: + + * ``v <= threshold`` predicate (non-strict) so a forecast at exactly + the drain value (e.g. queue lands on zero) registers as drained. + * No-crossing sentinel is ``+Inf`` (rather than NaN) so the + ``forecast_time_to_drain_seconds == +Inf for 30m`` alert can match + "queue is not actually draining". + * Band side picks ``yhat_upper`` (the *latest* drain — the + conservative answer when alerting on stuck queues). + * Emitted under ``_forecast_time_to_drain_seconds`` instead of + ``..._time_to_threshold_seconds`` so the two families never + collide on the same query. + + ``le_drain`` thresholds **must not** carry ``alert_within`` / + ``severity``: the derived ``forecast_will_breach`` gauge fires when + the ETA is small, which for a drain means "queue will drain soon" + — the opposite of a "predictive breach". The combination is + rejected at config-load time so the meaning of will_breach stays + unambiguous. """ model_config = ConfigDict(extra="forbid") name: str value: float - comparator: Literal["gt", "lt"] = "gt" + comparator: Literal["gt", "lt", "le_drain"] = "gt" + alert_within: Duration | None = None + severity: Literal["warning", "critical"] | None = None + + +class SLOConfig(BaseModel): + """Per-query SLO error-budget forecasting block. + + Off-by-default. When configured on a query whose forecast represents + an error rate (fraction of failing requests, fraction of time in + error), the forecaster emits three derived families per configured + window: + + * ``_forecast_error_budget_remaining_seconds{window}`` — the + static allowed-error budget for the window + (``(1 - objective/100) * window``). + * ``_forecast_error_budget_burn_rate{window}`` — the predicted + consumption rate divided by the allowed rate. ``1.0`` means + "consuming at exactly the budget-exhausting rate". + * ``_forecast_error_budget_exhausted_in_seconds{window[, level]}`` — + projected seconds until cumulative consumption at the forecast's + current rate exhausts the budget. Central row carries no + ``level``; band rows use ``yhat_upper`` (the conservative side — + a higher predicted rate exhausts sooner). + + Aligns with the SRE workbook's multi-window/multi-burn-rate + pattern: pair short and long ``windows`` (e.g. ``[5m, 1h, 30d]``) + and alert on the burn rate against each. + + Cardinality multiplier ``N_windows * (N_levels + 1)`` per opted-in + query, counted by ``promforecast validate``. + """ + + model_config = ConfigDict(extra="forbid") + + # Objective is the SLO target as a percentage in (0, 100): 99.9 + # means "allow 0.1% errors". The bounds exclude both endpoints so a + # config can't accidentally request "100% success" (zero budget, + # divide-by-zero on burn rate) or "0% success" (every observation + # is an error, budget meaningless). + objective: float = Field(gt=0.0, lt=100.0) + windows: list[Duration] = Field(default_factory=lambda: [timedelta(days=30)]) + + +class RecommendedWindowsConfig(BaseModel): + """Maintenance-window recommendation block. + + Off-by-default. When configured, the forecaster sweeps the next + ``lookahead`` of the forecast horizon for the lowest-predicted-load + windows of ``duration`` and emits a single ranked family — + ``forecast_recommended_maintenance_window{id, group, rank, + duration_seconds, start_timestamp_seconds}`` — whose value is the + mean predicted load within that window. Rank 1 is the best + (lowest-load) window; subsequent ranks are emitted up to ``count``. + + The forecast already has all the data; this turns the "stare at a + weekly seasonality panel at 11 PM" decision into a config-knob + emission family operators can pin to a Grafana panel or feed into + a workflow. + + Cardinality multiplier ``count`` per opted-in query — bounded by + construction. The ``start_timestamp_seconds`` label churns each + refresh (different windows become available as time moves + forward); operators on retention-sensitive TSDBs can drop the label + via ``metric_relabel_configs`` if the churn outpaces their + retention without losing the metric's value. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + duration: Duration = timedelta(minutes=30) + lookahead: Duration = timedelta(hours=48) + count: int = Field(default=3, ge=1) class GrowthRateEmissionConfig(BaseModel): @@ -459,7 +560,7 @@ class BandCoverageConfig(BaseModel): class QueryEmissionConfig(BaseModel): """Per-query emission toggles. - Today five families of settings live here: + Today six families of settings live here: * :class:`GrowthRateEmissionConfig` — forecast growth-rate metric. * ``clip_non_negative`` and ``round_to_integer`` — count-aware @@ -468,6 +569,8 @@ class QueryEmissionConfig(BaseModel): drift telemetry. * :class:`BandCoverageConfig` — empirical band coverage rolling gauges. + * ``conditional`` — parallel ``_forecast_conditional*`` family + that re-runs the fit with regressors held at their current value. Every family is kept on the per-query :class:`EmissionConfig` block so operators opt in *per metric*: a config that mixes a count-valued @@ -481,6 +584,14 @@ class QueryEmissionConfig(BaseModel): An operator who wants the count profile but explicit continuous emission still gets the override by setting the toggle to ``False`` directly. + + ``conditional`` is off by default. When on, the runner re-fits every + emitted model with a regressors frame whose values are flat-held at + the most recent observation. Models that ignore the regressors + parameter (the built-in ARIMA/ETS/SeasonalNaive wrappers) produce + a conditional forecast equal to the unconditional one — the family + still emits so downstream alerts have a stable schema; regressor-aware + models register additional gain. """ model_config = ConfigDict(extra="forbid") @@ -490,6 +601,12 @@ class QueryEmissionConfig(BaseModel): round_to_integer: bool | None = None drift: DriftEmissionConfig = Field(default_factory=DriftEmissionConfig) band_coverage: BandCoverageConfig = Field(default_factory=BandCoverageConfig) + conditional: bool = False + # Component decomposition emission (trend/seasonal/level/residual). Off by + # default; opt-in per query because decomposition adds N_components rows + # per (series, model) per horizon step. See `models.py` for which models + # actually decompose; non-decomposable models bump a skip counter. + decompose: bool = False class ColdStartBorrowConfig(BaseModel): @@ -594,6 +711,11 @@ class QueryConfig(BaseModel): emission: QueryEmissionConfig = Field(default_factory=QueryEmissionConfig) data_profile: Literal["intermittent", "count", "continuous"] | None = None cold_start: ColdStartConfig = Field(default_factory=ColdStartConfig) + # Per-query SLO error-budget forecasting. ``None`` (default) skips + # the family entirely; set the block to opt in. See :class:`SLOConfig`. + slo: SLOConfig | None = None + # Per-query maintenance-window recommendation. Off by default. + recommended_windows: RecommendedWindowsConfig = Field(default_factory=RecommendedWindowsConfig) # Per-query refresh override. When ``None`` (default) the query # inherits ``server.refresh_interval`` from its group. Setting it # explicitly schedules an independent timer for this query, so a @@ -1100,6 +1222,32 @@ def _validate_query_overrides(*, group_name: str, query: QueryConfig) -> None: "each threshold name must be unique within a query" ) threshold_names.add(threshold.name) + has_alert_within = threshold.alert_within is not None + has_severity = threshold.severity is not None + if has_alert_within != has_severity: + raise ValueError( + f"{label}.thresholds[{threshold.name}]: alert_within and severity " + "must be set together (the will_breach metric is a derived shape " + "and cannot be partially configured)" + ) + if threshold.alert_within is not None and threshold.alert_within.total_seconds() <= 0: + raise ValueError( + f"{label}.thresholds[{threshold.name}]: alert_within must be a positive duration" + ) + # The will_breach gauge derives ``1`` from "ETA is finite and + # below alert_within". For a drain threshold the ETA shrinks as + # the queue drains faster — so will_breach would fire on a + # *healthy* fast-draining queue, the opposite of its + # "predictive breach" meaning. Reject the combination at + # load-time rather than emitting metrics whose meaning depends + # on the comparator. + if threshold.comparator == "le_drain" and has_alert_within: + raise ValueError( + f"{label}.thresholds[{threshold.name}]: alert_within + severity are not " + "supported on le_drain thresholds; the will_breach gauge would invert " + "(it would fire on a queue draining quickly). Use the +Inf-anchored " + "QueueNotDraining-style alert on forecast_time_to_drain_seconds instead." + ) _validate_cold_start(label=label, query=query) growth = query.emission.growth_rate if growth.enabled: @@ -1130,6 +1278,47 @@ def _validate_query_overrides(*, group_name: str, query: QueryConfig) -> None: ) if query.refresh_interval is not None and query.refresh_interval.total_seconds() <= 0: raise ValueError(f"{label}.refresh_interval must be a positive duration") + _validate_slo(label=label, query=query) + _validate_recommended_windows(label=label, query=query) + + +def _validate_slo(*, label: str, query: QueryConfig) -> None: + """Semantic checks for the SLO block of a query. + + The numeric bounds on ``objective`` are enforced by pydantic; this + helper validates the per-window durations and the relationship + between windows and the forecast horizon. Failing at load time + means an over-budget config never reaches the runner. + """ + if query.slo is None: + return + if not query.slo.windows: + raise ValueError(f"{label}.slo: at least one window is required") + for window in query.slo.windows: + if window.total_seconds() <= 0: + raise ValueError(f"{label}.slo.windows: must be positive durations") + + +def _validate_recommended_windows(*, label: str, query: QueryConfig) -> None: + """Semantic checks for the recommended-windows block of a query. + + The duration must fit inside the lookahead and ``count`` must be + positive (pydantic enforces ``count >= 1``; we still guard against + a non-positive ``duration`` or ``lookahead`` slipping through). + """ + rw = query.recommended_windows + if not rw.enabled: + return + if rw.duration.total_seconds() <= 0: + raise ValueError(f"{label}.recommended_windows.duration must be a positive duration") + if rw.lookahead.total_seconds() <= 0: + raise ValueError(f"{label}.recommended_windows.lookahead must be a positive duration") + if rw.duration > rw.lookahead: + raise ValueError( + f"{label}.recommended_windows.duration ({rw.duration}) must be <= " + f"lookahead ({rw.lookahead}); a window longer than the lookahead " + "can never be recommended." + ) def _validate_cold_start(*, label: str, query: QueryConfig) -> None: diff --git a/forecaster/src/promforecast/decomposition.py b/forecaster/src/promforecast/decomposition.py new file mode 100644 index 0000000..3b9d61a --- /dev/null +++ b/forecaster/src/promforecast/decomposition.py @@ -0,0 +1,264 @@ +"""Forecast component decomposition. + +Implements a model-agnostic decomposition of the forecast curve into +``trend`` / ``seasonal`` / ``level`` / ``residual`` components. Used by +the runner when ``emission.decompose: true`` is set on a query. + +The decomposition is **derived from the forecast itself**, not from the +underlying model's internal state, so it works uniformly across the +built-in model registry and any plug-in model whose forecast frame +exposes the standard ``yhat`` column. The trade-off is that the split +between components is a heuristic projection rather than the model's +own attribution — operators get a useful "what is shaping this curve" +visual, not a parameter-level introspection. + +Two pieces of structure are recovered: + +* **Level** = the in-sample mean (a single number broadcast over the + horizon). Captures the baseline the forecast oscillates around. +* **Trend** = the linear best-fit slope of the forecast curve over the + horizon, anchored at zero at the first step so the level isn't + double-counted. +* **Seasonal** = the deviation from (level + trend) averaged at each + position within the configured ``season_length`` cycle. Recovers the + recurring "Mondays are higher" shape. +* **Residual** = what's left after subtracting the other three — + enforces the contract that ``trend + seasonal + level + residual == + yhat`` row-by-row up to floating-point noise. + +The contract is enforced by :func:`decompose`; tests assert it on every +fit. + +A registry of *non-decomposable* model names is maintained alongside so +the runner can short-circuit the decomposition for, e.g., intermittent +wrappers or neural extensions that don't have a meaningful trend/season +split. Names listed in :data:`NON_DECOMPOSABLE_MODELS` are reported as +``forecast_decomposition_skipped_total{reason="unsupported_model"}`` +without an attempt being made. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd + + +COMPONENT_NAMES = ("trend", "seasonal", "level", "residual") + + +# Models whose forecast shape doesn't admit a meaningful trend/season +# decomposition. Intermittent-demand methods spread sparse mass across +# the horizon — separating that into trend + seasonal would just label +# noise. Operators see the skip via +# ``forecast_decomposition_skipped_total{reason="unsupported_model"}``. +NON_DECOMPOSABLE_MODELS: frozenset[str] = frozenset( + { + "Croston", + "CrostonSBA", + "ADIDA", + "IMAPA", + "ColdStart", + } +) + + +@dataclass(frozen=True) +class DecompositionResult: + """Per-component value lists, all aligned to the forecast frame's rows. + + ``components`` keys are exactly :data:`COMPONENT_NAMES`. Each list + has length equal to the input forecast's row count. + """ + + components: dict[str, list[float]] + + +class DecompositionError(RuntimeError): + """Raised when the forecast frame can't be decomposed. + + Carries a short ``reason`` token that becomes a label value on + ``forecast_decomposition_skipped_total``. + """ + + def __init__(self, reason: str, message: str) -> None: + super().__init__(message) + self.reason = reason + + +def is_decomposable(model_name: str) -> bool: + """Return whether ``model_name`` is in the decomposable registry.""" + return model_name not in NON_DECOMPOSABLE_MODELS + + +def decompose( + *, + forecast: pd.DataFrame, + in_sample_values: list[float], + season_length: int, +) -> DecompositionResult: + """Project a forecast frame onto trend/seasonal/level/residual components. + + Args: + forecast: The forecast frame produced by ``model.fit_predict``. + Must contain a ``yhat`` column with one value per horizon + step. + in_sample_values: The training-side ``y`` values used to + establish the baseline level. The mean of this list becomes + the ``level`` component (broadcast over the horizon). + season_length: Number of forecast steps per cycle. Used to fold + the (yhat - level - trend) residual into per-position + seasonal averages. + + Returns a :class:`DecompositionResult` whose component lists each + match the forecast frame's row count. + + Raises :class:`DecompositionError` with a bounded ``reason`` token + when the inputs can't be decomposed. The runner translates the + token into a label value on + ``forecast_decomposition_skipped_total``. + """ + if forecast.empty or "yhat" not in forecast.columns: + raise DecompositionError( + "components_unavailable", + "forecast frame is empty or missing the 'yhat' column", + ) + yhat = [float(v) for v in forecast["yhat"].tolist()] + n = len(yhat) + if n == 0: + raise DecompositionError( + "components_unavailable", + "forecast frame has zero rows", + ) + + # Level: in-sample mean. Falls back to the forecast mean when no + # in-sample data is supplied (the runner always supplies it; the + # branch is defensive for direct callers). + level_value = _finite_mean(in_sample_values) if in_sample_values else _finite_mean(yhat) + level = [level_value] * n + + # Trend: linear best-fit slope across the horizon, anchored at zero + # on step 0 so the average isn't double-counted with ``level``. + slope = _least_squares_slope(yhat) + trend = [slope * i for i in range(n)] + + # Seasonal: average residual at each position within the cycle of + # ``season_length``. Steps that share a position contribute to the + # same bucket; the bucket mean becomes the seasonal value for every + # step at that position. A non-positive season collapses to zero so + # the residual component picks up everything (correct contract). + detrended = [yhat[i] - level[i] - trend[i] for i in range(n)] + if season_length and season_length > 0: + buckets: list[list[float]] = [[] for _ in range(season_length)] + for i, v in enumerate(detrended): + if math.isfinite(v): + buckets[i % season_length].append(v) + bucket_means = [sum(b) / len(b) if b else 0.0 for b in buckets] + seasonal = [bucket_means[i % season_length] for i in range(n)] + else: + seasonal = [0.0] * n + + residual = [yhat[i] - level[i] - trend[i] - seasonal[i] for i in range(n)] + + components = { + "trend": trend, + "seasonal": seasonal, + "level": level, + "residual": residual, + } + _enforce_sum_invariant(yhat=yhat, components=components) + return DecompositionResult(components=components) + + +# Tolerances for the sum-to-yhat invariant. The closed-form math is exact +# under IEEE-754 once you account for the trailing residual subtraction; +# anything beyond a few ULPs scaled by the row magnitude is a logic bug, +# not float noise. The check is the *larger* of an absolute floor +# (handles rows near zero where relative tolerance would be too strict) +# and a relative band (handles large-magnitude metrics like disk bytes +# where the absolute floor would falsely trip even though the math is +# correct to nine significant digits). A pure absolute tolerance was the +# original implementation but produced false ``invariant_violation`` +# skips on legitimate large-magnitude forecasts. +_INVARIANT_ABS_TOLERANCE = 1e-6 +_INVARIANT_REL_TOLERANCE = 1e-9 + + +def _enforce_sum_invariant(*, yhat: list[float], components: dict[str, list[float]]) -> None: + """Validate that ``sum(components, row=i) ≈ yhat[i]`` for every row. + + The runner relies on this to keep the stacked-area panel honest. A + component list that drifts off the invariant silently would be worse + than emitting nothing — operators would attribute the missing mass to + the wrong cause. We raise instead so the caller emits the skip + counter (``reason="invariant_violation"``) and the family stays off + /metrics for the offending fit. + + The tolerance is ``max(abs_tol, rel_tol * |yhat|)`` so the check + holds both on near-zero rows (where relative would be too strict) + and on large-magnitude metrics like disk bytes (where a fixed + absolute floor would falsely trip on harmless float noise). + """ + summed = [0.0] * len(yhat) + for values in components.values(): + for i, v in enumerate(values): + summed[i] += v + for i, (s, y) in enumerate(zip(summed, yhat, strict=True)): + if not (math.isfinite(s) and math.isfinite(y)): + continue + tolerance = max(_INVARIANT_ABS_TOLERANCE, _INVARIANT_REL_TOLERANCE * abs(y)) + if abs(s - y) > tolerance: + raise DecompositionError( + "invariant_violation", + f"component sum diverges from yhat at row {i}: " + f"|{s} - {y}| = {abs(s - y)} > {tolerance}", + ) + + +def reconstruct(components: dict[str, list[float]]) -> list[float]: + """Sum the components row-by-row. + + Helper used by tests to assert + ``sum(components, row=i) ≈ yhat[i]`` for every ``i``. + """ + if not components: + return [] + rows = max((len(vs) for vs in components.values()), default=0) + out = [0.0] * rows + for values in components.values(): + for i in range(rows): + if i < len(values): + out[i] += values[i] + return out + + +def _finite_mean(values: list[float]) -> float: + """Mean over the finite entries; 0.0 when none are finite.""" + finite = [v for v in values if math.isfinite(v)] + if not finite: + return 0.0 + return sum(finite) / len(finite) + + +def _least_squares_slope(values: list[float]) -> float: + """Closed-form OLS slope of ``values`` regressed on row index. + + Returns 0 when the input is too short or every row is non-finite, + so the trend component degrades silently to "no slope" rather than + raising. + """ + pairs = [(float(i), float(v)) for i, v in enumerate(values) if math.isfinite(v)] + if len(pairs) < 2: # noqa: PLR2004 + return 0.0 + n = float(len(pairs)) + sum_x = sum(x for x, _ in pairs) + sum_y = sum(y for _, y in pairs) + sum_xy = sum(x * y for x, y in pairs) + sum_xx = sum(x * x for x, _ in pairs) + denom = n * sum_xx - sum_x * sum_x + if denom == 0: + return 0.0 + return (n * sum_xy - sum_x * sum_y) / denom diff --git a/forecaster/src/promforecast/exporter.py b/forecaster/src/promforecast/exporter.py index d2e382a..289cced 100644 --- a/forecaster/src/promforecast/exporter.py +++ b/forecaster/src/promforecast/exporter.py @@ -33,10 +33,14 @@ "AccuracyPoint", "BandCoveragePoint", "CalibrationOffsetPoint", + "ComponentDecompositionPoint", + "ConditionalDeviationPoint", + "ConditionalForecastPoint", "ContributionPoint", "DeviationPoint", "DriftPoint", "EnsembleWeightPoint", + "ErrorBudgetPoint", "Exporter", "ForecastPoint", "GroupSnapshot", @@ -44,7 +48,10 @@ "InvalidMetricNameError", "PreprocessStats", "QualityPoint", + "RecommendedMaintenanceWindowPoint", "ThresholdEtaPoint", + "TimeToDrainPoint", + "WillBreachPoint", "validate_metric_name", ] @@ -141,6 +148,122 @@ class ThresholdEtaPoint: seconds: float +@dataclass(frozen=True) +class ConditionalForecastPoint: + """A single conditional forecast row destined for /metrics. + + Mirrors :class:`ForecastPoint` but its metric name carries the + ``_forecast_conditional`` / ``_forecast_conditional_lower`` / + ``_forecast_conditional_upper`` suffix family. Emitted by the + conditional re-fit path when ``emission.conditional`` is on for the + query. + """ + + metric: str + labels: dict[str, str] + value: float + + +@dataclass(frozen=True) +class ConditionalDeviationPoint: + """A single conditional deviation observation. + + Rendered as ``forecast_conditional_deviation_ratio`` and + ``forecast_conditional_deviation_outside_band`` gauges. Mirrors the + existing :class:`DeviationPoint` shape — the band reference is the + conditional fit's leading row rather than the backtest trailing + band. + """ + + labels: dict[str, str] + ratio: float + outside_band: bool + + +@dataclass(frozen=True) +class ComponentDecompositionPoint: + """A single ``_forecast_component`` sample. + + The metric name is per-id (mirrors the forecast family). ``labels`` + carry ``component`` (one of ``trend``/``seasonal``/``level``/ + ``residual``) plus the standard ``model`` / ``horizon`` discriminators + and the source series labels. One sample is emitted per horizon step + per component when ``emission.decompose`` is on for a decomposable + model. + """ + + metric: str + labels: dict[str, str] + value: float + + +@dataclass(frozen=True) +class TimeToDrainPoint: + """One ``_forecast_time_to_drain_seconds`` sample. + + Sibling of :class:`ThresholdEtaPoint` for the ``le_drain`` comparator. + The metric name is per-id (mirrors the ETA family) and the + no-crossing sentinel is ``+Inf`` rather than NaN — operators alert + on ``forecast_time_to_drain_seconds == +Inf`` to catch "queue is + not actually draining". + """ + + metric: str + labels: dict[str, str] + seconds: float + + +@dataclass(frozen=True) +class ErrorBudgetPoint: + """One row in the SLO error-budget family destined for /metrics. + + Rendered through three per-id metric names: + + * ``_forecast_error_budget_remaining_seconds{window}`` + * ``_forecast_error_budget_burn_rate{window}`` + * ``_forecast_error_budget_exhausted_in_seconds{window[, level]}`` + + The metric field carries the full name including the ``_`` prefix + so the exporter can bucket and render one ``GaugeMetricFamily`` per + name (matching the forecast / ETA / will-breach family shape). + """ + + metric: str + labels: dict[str, str] + value: float + + +@dataclass(frozen=True) +class RecommendedMaintenanceWindowPoint: + """One ``forecast_recommended_maintenance_window`` sample. + + Global metric name (no per-id prefix) — the ``id`` and ``group`` are + discriminator labels so a single Grafana panel can render the + ranked windows across every query that opts in. The sample value is + the predicted load at the window (mean ``yhat`` across the window's + forecast rows); labels carry ``rank``, ``duration_seconds`` and + ``start_timestamp_seconds`` so dashboards can plot the window's + position on the timeline. + """ + + labels: dict[str, str] + predicted_load: float + + +@dataclass(frozen=True) +class WillBreachPoint: + """One ``_forecast_will_breach`` 0/1 sample. + + The metric name is per-id (mirrors the ETA family). ``labels`` carry + ``name``, ``severity``, and ``model``; ``level`` is added for + per-band rows and absent for the central forecast. + """ + + metric: str + labels: dict[str, str] + value: float + + @dataclass(frozen=True) class GrowthRatePoint: """One ``_forecast_growth_rate`` sample. @@ -244,6 +367,37 @@ class GroupSnapshot: # exposition until they reappear. threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) growth_rates: list[GrowthRatePoint] = field(default_factory=list) + # Predictive will-breach gauge: per-(series, model, threshold[, level]) + # 0/1 sample derived from the ETA + threshold ``alert_within`` window. + # Snapshot-style — only the most recent run's samples belong on + # /metrics; thresholds that drop out of config disappear from the + # exposition until they reappear. + will_breach_points: list[WillBreachPoint] = field(default_factory=list) + # Queue burn-down / drain ETA: parallel ``_forecast_time_to_drain_seconds`` + # family for thresholds with comparator=``le_drain``. ``+Inf`` samples + # are kept so the "queue is not draining" alert can match them. + time_to_drain_points: list[TimeToDrainPoint] = field(default_factory=list) + # SLO error-budget snapshot rows (remaining, burn rate, exhausted_in). + # Snapshot-style — only the most recent run's samples belong on + # /metrics; queries that disappear from config disappear from the + # exposition until they reappear. + error_budget_points: list[ErrorBudgetPoint] = field(default_factory=list) + # Recommended maintenance windows (top-N lowest-load windows in the + # forecast horizon). Snapshot-style; the next refresh rotates the + # set as time moves forward and earlier windows leave the lookahead. + recommended_maintenance_window_points: list[RecommendedMaintenanceWindowPoint] = field( + default_factory=list + ) + # Parallel conditional forecast family. Same shape as ``points`` — + # rendered through one ``GaugeMetricFamily`` per distinct metric name — + # but the metric names carry the ``_forecast_conditional`` suffix + # family. + conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) + # Conditional deviation gauges (one per emitted (series, model, level)). + conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) + # Per-horizon-step component decomposition (per emitted series, model). + # Rendered as ``_forecast_component{component="trend|...|residual"}``. + component_points: list[ComponentDecompositionPoint] = field(default_factory=list) last_run_timestamp: float = 0.0 run_duration_seconds: float = 0.0 # Per-query run duration in seconds for the most recent fit of each @@ -366,6 +520,25 @@ def __init__(self) -> None: # Per-(group, query) expansion count (gauge): how many concrete # PromQL fetches the most recent run derived from this template. self._discovery_expansions: dict[tuple[str, str], int] = {} + # Per-(group, query, model, reason) decomposition-skipped counter. + # Bumped when ``emission.decompose: true`` is set on a query but + # the chosen model does not decompose cleanly. The ``reason`` set + # is bounded by the runner (``unsupported_model``, ``fit_failed``, + # ``components_unavailable``, ``invariant_violation``); the + # ``group``/``query`` labels let operators alert "which query + # keeps tripping the skip path" without log scraping. The + # cardinality multiplier stays small (typical decompose-opted-in + # query has 1-2 emitted models, and skips only fire when + # something is wrong). + self._decomposition_skipped: dict[tuple[str, str, str, str], int] = {} + # Per-(group, query, model) conditional-aliased counter. Bumped + # whenever the conditional emission path short-circuits the + # refit (because the model advertises ``uses_regressors=False`` + # or no regressors are configured). Lets operators confirm the + # CPU saving from the short-circuit is actually firing in + # their deployment; a flat-zero counter means every conditional + # emission is paying the refit cost. + self._conditional_aliased: dict[tuple[str, str, str], int] = {} # Per-(group, query, source) season_length auto-detection counter. # ``source`` is bounded to {detected, snapped, fallback, too_short} # so cardinality is small and predictable. Useful for alerting on @@ -514,6 +687,38 @@ def retain_discovery_groups(self, names: set[str]) -> None: k: v for k, v in self._season_length_detections.items() if k[0] in names } + def increment_conditional_aliased(self, group: str, query: str, model: str) -> None: + """Bump the per-(group, query, model) conditional-aliased counter. + + Called by the runner whenever the conditional emission path + aliases the unconditional forecast instead of refitting. The + operator can chart ``rate(forecast_conditional_aliased_total[5m])`` + to confirm the short-circuit is firing. + """ + with self._lock: + key = (group, query, model) + self._conditional_aliased[key] = self._conditional_aliased.get(key, 0) + 1 + + def increment_decomposition_skipped( + self, + group: str, + query: str, + model: str, + reason: str, + ) -> None: + """Bump the per-(group, query, model, reason) decomposition-skipped counter. + + Bounded ``reason`` set lives on the runner (``unsupported_model``, + ``fit_failed``, ``components_unavailable``, ``invariant_violation``) + so cardinality stays small even with the four-label fan-out. + Operators can alert per (group, query) on persistent skips + instead of relying on a global "something is being skipped" + signal. + """ + with self._lock: + key = (group, query, model, reason) + self._decomposition_skipped[key] = self._decomposition_skipped.get(key, 0) + 1 + def increment_season_length_detection(self, group: str, query: str, source: str) -> None: """Bump the auto-seasonality detection counter for one outcome. @@ -578,6 +783,14 @@ def _discovery_state( with self._lock: return dict(self._discovery_expansions), dict(self._discovery_failures) + def _decomposition_skipped_state(self) -> dict[tuple[str, str, str, str], int]: + with self._lock: + return dict(self._decomposition_skipped) + + def _conditional_aliased_state(self) -> dict[tuple[str, str, str], int]: + with self._lock: + return dict(self._conditional_aliased) + def _season_length_state(self) -> dict[tuple[str, str, str], int]: with self._lock: return dict(self._season_length_detections) @@ -600,13 +813,22 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: cache_state = self._exporter._query_cache_state() discovery_state = self._exporter._discovery_state() season_length_state = self._exporter._season_length_state() + decomposition_skipped_state = self._exporter._decomposition_skipped_state() + conditional_aliased_state = self._exporter._conditional_aliased_state() yield from _forecast_families(groups) + yield from _conditional_forecast_families(groups) + yield from _conditional_deviation_families(groups) + yield from _component_families(groups) yield from _accuracy_families(groups) yield from _deviation_families(groups) yield from _quality_families(groups) yield from _ensemble_families(groups) yield from _contribution_families(groups) yield from _threshold_eta_families(groups) + yield from _will_breach_families(groups) + yield from _time_to_drain_families(groups) + yield from _error_budget_families(groups) + yield from _recommended_window_families(groups) yield from _growth_rate_families(groups) yield from _calibration_families(groups) yield from _preprocess_families(groups) @@ -624,6 +846,8 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: yield from _query_cache_families(cache_state) yield from _discovery_families(discovery_state) yield from _season_length_families(season_length_state) + yield from _decomposition_skipped_families(decomposition_skipped_state) + yield from _conditional_aliased_families(conditional_aliased_state) def _emit_gauge_family( @@ -828,6 +1052,240 @@ def _threshold_eta_families( ) +def _conditional_forecast_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render the ``_forecast_conditional*`` family, one per distinct name. + + Same bucketing as the unconditional forecast family. The metric + names that show up here are ``_forecast_conditional``, + ``_forecast_conditional_lower``, ``_forecast_conditional_upper``. + """ + by_metric: dict[str, list[ConditionalForecastPoint]] = {} + for snap in groups: + for point in snap.conditional_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + metric_name, + "promforecast conditional forecast value — model re-fit with " + "regressors held at their most-recent value.", + value_fn=lambda p: p.value, + ) + + +def _conditional_deviation_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render the conditional-deviation ratio + outside-band gauges. + + Mirrors the unconditional ``_deviation_families`` shape; the + conditional ratio drops NaN samples while the boolean outside-band + still emits a 0 / 1 reading so alerts can detect "outside the + conditional band" cleanly. + """ + points = [p for snap in groups for p in snap.conditional_deviation_points] + yield from _emit_gauge_family( + points, + "forecast_conditional_deviation_ratio", + "Signed deviation of the most recent actual from the conditional " + "forecast band (regressors held at their current value).", + value_fn=lambda p: p.ratio, + skip_fn=lambda p: not math.isfinite(p.ratio), + ) + yield from _emit_gauge_family( + points, + "forecast_conditional_deviation_outside_band", + "1 if the most recent actual is outside the conditional forecast band, else 0.", + value_fn=lambda p: 1.0 if p.outside_band else 0.0, + ) + + +def _component_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render the ``_forecast_component`` decomposition family per id.""" + by_metric: dict[str, list[ComponentDecompositionPoint]] = {} + for snap in groups: + for point in snap.component_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + metric_name, + "Per-component decomposition of the forecast curve " + "(trend / seasonal / level / residual). Components sum to " + "the central forecast within numerical tolerance.", + value_fn=lambda p: p.value, + skip_fn=lambda p: not math.isfinite(p.value), + ) + + +def _conditional_aliased_families( + state: dict[tuple[str, str, str], int], +) -> Iterator[CounterMetricFamily]: + """Render ``forecast_conditional_aliased_total{group, query, model}``. + + Bumped every time the conditional emission path skips the refit + and aliases the unconditional forecast frame instead. Lets + operators chart how often the short-circuit is firing + (and therefore how much CPU it is saving them); a deployment + where the counter stays at 0 is paying the doubled-fit cost for + every conditional emission. + + Emitted only after at least one alias has fired so /metrics stays + quiet for deployments that don't enable ``emission.conditional``. + """ + if not state: + return + counter = CounterMetricFamily( + "forecast_conditional_aliased", + "Number of times the conditional emission path aliased the unconditional " + "forecast frame instead of refitting with held-constant regressors. " + "Bumped per (series, model) when no regressors are configured or the model " + "advertises uses_regressors=False.", + labels=["group", "query", "model"], + ) + for (group, query, model), count in state.items(): + counter.add_metric([group, query, model], float(count)) + yield counter + + +def _decomposition_skipped_families( + state: dict[tuple[str, str, str, str], int], +) -> Iterator[CounterMetricFamily]: + """Render ``forecast_decomposition_skipped_total{group, query, model, reason}``. + + Emitted only when at least one skip has occurred so /metrics stays + quiet for deployments that don't enable decomposition (or only + enable it for decomposable models). + """ + if not state: + return + counter = CounterMetricFamily( + "forecast_decomposition_skipped", + "Number of times the runner skipped component-decomposition emission " + "for the (group, query, model, reason) tuple. Reason is one of " + "(unsupported_model | fit_failed | components_unavailable | " + "invariant_violation).", + labels=["group", "query", "model", "reason"], + ) + for (group, query, model, reason), count in state.items(): + counter.add_metric([group, query, model, reason], float(count)) + yield counter + + +def _will_breach_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render ``_forecast_will_breach`` gauges, one family per id. + + The metric name is per-id (mirrors the ETA family). Values are + always 0 or 1 — operators alert with + ``forecast_will_breach{severity="critical"} == 1 for 10m`` and the + zero-baseline samples keep the panel/chart well-defined even before + the first crossing. + """ + by_metric: dict[str, list[WillBreachPoint]] = {} + for snap in groups: + for point in snap.will_breach_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + metric_name, + "1 if the forecast predicts a threshold crossing within the " + "configured `alert_within` window for the (name, severity[, level]) " + "label set; 0 otherwise.", + value_fn=lambda p: p.value, + ) + + +def _time_to_drain_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render ``_forecast_time_to_drain_seconds`` gauges, one family per id. + + Sibling of :func:`_threshold_eta_families` for the ``le_drain`` + comparator. Unlike the ETA family this *preserves* the + no-crossing sentinel (``+Inf``) because the operator-facing alert + is ``forecast_time_to_drain_seconds == +Inf for 30m`` — dropping + non-finite samples at render time would make that rule silently + never match. NaN values (degenerate inputs) are still dropped + because the alert can't act on them. + """ + by_metric: dict[str, list[TimeToDrainPoint]] = {} + for snap in groups: + for point in snap.time_to_drain_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + metric_name, + "Seconds from the forecast's start until the projected curve first " + "crosses ``value`` from above (queue drains). ``+Inf`` means no " + "crossing within the horizon — alert on ``... == +Inf for 30m`` to " + "catch 'queue is not draining'.", + value_fn=lambda p: p.seconds, + skip_fn=lambda p: math.isnan(p.seconds), + ) + + +def _error_budget_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render the SLO error-budget family per-id. + + Three sibling metric names are bucketed by name then emitted as + individual gauge families. The exhausted-in family carries an + explicit ``+Inf`` sample when the forecast says no errors — that + propagates the "no concern" answer through to alerts on + ``... > N`` instead of dropping the sample. NaN is still skipped + because alerts can't act on it. + """ + by_metric: dict[str, list[ErrorBudgetPoint]] = {} + for snap in groups: + for point in snap.error_budget_points: + by_metric.setdefault(point.metric, []).append(point) + for metric_name, points in sorted(by_metric.items()): + yield from _emit_gauge_family( + points, + metric_name, + "SLO error-budget projection derived from the forecast. The " + "remaining-seconds family is the total allowed budget for the " + "window; the burn-rate family is the predicted rate divided by " + "the allowed rate; the exhausted-in family projects when the " + "budget would be consumed at the current trajectory.", + value_fn=lambda p: p.value, + skip_fn=lambda p: math.isnan(p.value), + ) + + +def _recommended_window_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render ``forecast_recommended_maintenance_window`` gauges. + + Single global family (no per-id prefix) — ``id`` is a label so a + Grafana panel can render the ranked recommendations across every + opted-in query. The sample value is the predicted load at the + window; lower is better. Labels carry ``rank``, + ``duration_seconds``, ``start_timestamp_seconds``. + """ + points = [p for snap in groups for p in snap.recommended_maintenance_window_points] + yield from _emit_gauge_family( + points, + "forecast_recommended_maintenance_window", + "Predicted load (mean forecast value) for a recommended maintenance " + "window. Lower is better; ``rank=1`` is the suggested window. The " + "``start_timestamp_seconds`` and ``duration_seconds`` labels carry " + "the window's position on the forecast horizon.", + value_fn=lambda p: p.predicted_load, + skip_fn=lambda p: not math.isfinite(p.predicted_load), + ) + + def _growth_rate_families( groups: list[GroupSnapshot], ) -> Iterator[GaugeMetricFamily]: @@ -928,7 +1386,8 @@ def _operational_families( # noqa: PLR0912 "forecast_failures", "Forecast run failures, by reason " "(query_error | query_timeout | fit_error | fit_timeout | " - "invalid_metric_name | group_error).", + "invalid_metric_name | group_error | conditional_fit_error | " + "conditional_fit_timeout).", labels=["group", "reason"], ) for snap in groups: diff --git a/forecaster/src/promforecast/main.py b/forecaster/src/promforecast/main.py index 7150ce2..e7ae778 100644 --- a/forecaster/src/promforecast/main.py +++ b/forecaster/src/promforecast/main.py @@ -51,6 +51,7 @@ def cli() -> None: config_path=args.config, dry_run=args.dry_run, datasource_url_override=args.datasource_url, + strict_cardinality=args.strict_cardinality, ) sys.exit(rc) @@ -145,6 +146,16 @@ def _build_parser() -> argparse.ArgumentParser: "when the production URL is unreachable from CI." ), ) + validate_p.add_argument( + "--strict-cardinality", + action="store_true", + help=( + "Fail (exit 1) when the projected cardinality estimate exceeds " + "safety.max_total_series instead of emitting an advisory WARN. " + "Use in CI to reject configs whose opt-in emission would blow " + "the global series cap. See docs/operations/cardinality.md." + ), + ) diagnose_p = sub.add_parser( "diagnose", diff --git a/forecaster/src/promforecast/maintenance.py b/forecaster/src/promforecast/maintenance.py new file mode 100644 index 0000000..43239cd --- /dev/null +++ b/forecaster/src/promforecast/maintenance.py @@ -0,0 +1,140 @@ +"""Recommended-maintenance-window picker. + +Given a forecast curve and a ``(duration, lookahead, count)`` request, +return the top-N predicted-lowest-load windows of ``duration`` within +the next ``lookahead`` of the forecast horizon. Each candidate window +is scored by the *mean* of the forecast values inside it; the lowest +mean wins. + +Pure-function implementation — the runner reaches for this once per +emitted (model, series) when ``recommended_windows`` is configured on +the query, and the math is unit-testable without a full Runner fixture. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +from ._time_axis import seconds_from_start as _seconds_from_start + +if TYPE_CHECKING: + import pandas as pd + + from .config import RecommendedWindowsConfig + + +@dataclass(frozen=True) +class RecommendedWindow: + """One ranked window suggestion derived from a forecast. + + ``rank`` is 1-indexed (1 = lowest predicted load). ``predicted_load`` + is the mean ``yhat`` across the rows that fall inside the window — + the same metric an operator reading the Grafana panel would use to + eyeball the window themselves. + """ + + rank: int + start_timestamp: float + duration_seconds: float + predicted_load: float + + +def recommend_windows( + *, + forecast: pd.DataFrame, + config: RecommendedWindowsConfig, +) -> list[RecommendedWindow]: + """Score every aligned candidate window in the lookahead and return top-N. + + Candidate windows are anchored at each forecast step (so the + resolution is the forecast step itself) and span ``duration``. The + last fully-fitting window's start is the latest valid anchor; later + anchors would overflow either the lookahead or the available + forecast horizon and are skipped silently. + + Returns at most ``config.count`` windows, ranked ascending by + predicted load. Ties are broken by start timestamp (earlier first) + so successive refreshes produce a stable ranking when the curve is + flat — without this an operator's "next window" recommendation + would jitter across equally-good slots. + """ + if forecast.empty or "yhat" not in forecast.columns or "ds" not in forecast.columns: + return [] + if config.count <= 0 or config.duration.total_seconds() <= 0: + return [] + duration_seconds = config.duration.total_seconds() + lookahead_seconds = config.lookahead.total_seconds() + seconds = _seconds_from_start(forecast["ds"]) + if seconds is None or not seconds: + return [] + start_unix = _unix_seconds(forecast["ds"].iloc[0]) + if start_unix is None: + return [] + yhat = [float(v) for v in forecast["yhat"].tolist()] + candidates: list[RecommendedWindow] = [] + n = len(seconds) + for i in range(n): + window_start = seconds[i] + if window_start > lookahead_seconds: + # All subsequent anchors are also out of the lookahead — the + # forecast timestamps are monotone, so we can stop early. + break + window_end = window_start + duration_seconds + # j: half-open upper bound — the first row strictly past the + # window's end. If we never find one, the window overflows the + # forecast horizon and the candidate is skipped. + j = i + while j < n and seconds[j] < window_end: + j += 1 + if j == n and (seconds[-1] - window_start) < duration_seconds: + # The forecast ends before the window does; skip rather than + # using a partial mean (we'd recommend windows we can't see + # the full load of). + continue + values_in_window = [v for v in yhat[i:j] if math.isfinite(v)] + if not values_in_window: + continue + predicted_load = sum(values_in_window) / len(values_in_window) + candidates.append( + RecommendedWindow( + rank=0, # filled in after sorting + start_timestamp=start_unix + window_start, + duration_seconds=duration_seconds, + predicted_load=predicted_load, + ) + ) + # Stable sort: ascending load, then ascending start_timestamp for + # deterministic ranking when several windows tie on load. + candidates.sort(key=lambda w: (w.predicted_load, w.start_timestamp)) + top = candidates[: config.count] + return [ + RecommendedWindow( + rank=i + 1, + start_timestamp=w.start_timestamp, + duration_seconds=w.duration_seconds, + predicted_load=w.predicted_load, + ) + for i, w in enumerate(top) + ] + + +def _unix_seconds(ts: object) -> float | None: + """Return ``ts.timestamp()`` (seconds since the epoch) or ``None``. + + Accepts ``datetime`` and pandas ``Timestamp`` (which is duck-typed as + a datetime but doesn't subclass it in older pandas releases); the + fallback path uses ``getattr`` to avoid asserting an attribute that + ``object`` doesn't carry. + """ + if isinstance(ts, datetime): + return ts.timestamp() + timestamp_fn = getattr(ts, "timestamp", None) + if timestamp_fn is None: + return None + try: + return float(timestamp_fn()) + except (AttributeError, ValueError, TypeError): + return None diff --git a/forecaster/src/promforecast/models/__init__.py b/forecaster/src/promforecast/models/__init__.py index 5f76fc4..3a426ef 100644 --- a/forecaster/src/promforecast/models/__init__.py +++ b/forecaster/src/promforecast/models/__init__.py @@ -114,6 +114,26 @@ def _discover() -> dict[str, type[ForecastModel]]: f"{discovered[name].__module__}.{discovered[name].__qualname__} " f"and {obj.__module__}.{obj.__qualname__} both registered" ) + # Plug-ins discovered through this code path that pre-date the + # ``uses_regressors`` marker default to the safe "refit" branch + # of the conditional-emission path. Flag the missing attribute + # at registry-build time so operators see *which* plug-in is + # paying the doubled-fit cost; the conditional emission still + # works, it just doesn't benefit from the short-circuit. + if not hasattr(obj, "uses_regressors"): + logger.info( + "model_missing_uses_regressors_attribute", + entry_point=ep.name, + value=ep.value, + model_name=name, + hint=( + "set 'uses_regressors = False' on the class if the " + "model ignores the regressors parameter so the " + "conditional-emission path can short-circuit the refit; " + "leave it True (or unset, the safe default) when the " + "model genuinely consumes regressors" + ), + ) discovered[name] = obj return discovered @@ -156,6 +176,28 @@ def available() -> list[str]: return sorted(_registry().keys()) +def uses_regressors(name: str) -> bool: + """Whether the model registered under ``name`` consumes the regressors arg. + + Read off the class-level ``uses_regressors`` attribute (see + :class:`ForecastModel`). Defaults to ``False`` for plug-ins that + predate the marker — the conditional emission path will still + *refit* such plug-ins, so the safe default is "regressors-aware so we + don't skip the refit by mistake"; we invert: a missing marker means + "treat as unknown -> refit to be safe". + + Returns ``True`` when the model is unknown to the registry — the + runner's conditional path uses this signal to decide whether to skip + a refit, and missing entries should not be silently dropped from the + refit pass. + """ + registry = _registry() + cls = registry.get(name) + if cls is None: + return True + return bool(getattr(cls, "uses_regressors", True)) + + def build( name: str, season_length: int, @@ -204,4 +246,5 @@ def build( "available", "build", "refresh", + "uses_regressors", ] diff --git a/forecaster/src/promforecast/models/auto_arima.py b/forecaster/src/promforecast/models/auto_arima.py index 072a412..e359423 100644 --- a/forecaster/src/promforecast/models/auto_arima.py +++ b/forecaster/src/promforecast/models/auto_arima.py @@ -12,6 +12,7 @@ class AutoARIMAModel: name = "AutoARIMA" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/models/auto_arima_damped.py b/forecaster/src/promforecast/models/auto_arima_damped.py index 57c1817..7bbf6db 100644 --- a/forecaster/src/promforecast/models/auto_arima_damped.py +++ b/forecaster/src/promforecast/models/auto_arima_damped.py @@ -27,6 +27,7 @@ class AutoARIMADampedModel: name = "AutoARIMADamped" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/models/auto_ets.py b/forecaster/src/promforecast/models/auto_ets.py index 98c4533..19fbf17 100644 --- a/forecaster/src/promforecast/models/auto_ets.py +++ b/forecaster/src/promforecast/models/auto_ets.py @@ -18,6 +18,7 @@ class AutoETSModel: name = "AutoETS" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/models/auto_ets_damped.py b/forecaster/src/promforecast/models/auto_ets_damped.py index 782e69c..55faacc 100644 --- a/forecaster/src/promforecast/models/auto_ets_damped.py +++ b/forecaster/src/promforecast/models/auto_ets_damped.py @@ -33,6 +33,7 @@ class AutoETSDampedModel: name = "AutoETSDamped" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/models/base.py b/forecaster/src/promforecast/models/base.py index 3c7222f..ce4eb10 100644 --- a/forecaster/src/promforecast/models/base.py +++ b/forecaster/src/promforecast/models/base.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Protocol, runtime_checkable +from typing import TYPE_CHECKING, ClassVar, Protocol, runtime_checkable if TYPE_CHECKING: import pandas as pd @@ -18,9 +18,18 @@ class ForecastModel(Protocol): Implementations must be safe to instantiate without arguments and must not mutate inputs. + + ``uses_regressors`` advertises whether the model's ``fit_predict`` + actually consumes the ``regressors`` parameter. The conditional-forecast + emission path reads this to skip the held-constant refit for models + that ignore regressors entirely (every built-in wrapper here) — without + it, opting into ``emission.conditional: true`` doubles the per-series + fit cost for no analytical gain. Plug-ins that do consume regressors + (e.g. the Prophet wrapper) override this to ``True``. """ name: str + uses_regressors: ClassVar[bool] def fit_predict( self, diff --git a/forecaster/src/promforecast/models/intermittent.py b/forecaster/src/promforecast/models/intermittent.py index 5dd5152..b189191 100644 --- a/forecaster/src/promforecast/models/intermittent.py +++ b/forecaster/src/promforecast/models/intermittent.py @@ -122,6 +122,7 @@ class _IntermittentBase: """ name: ClassVar[str] = "" + uses_regressors: ClassVar[bool] = False def __init__(self, season_length: int = 288, **params: Any) -> None: # ``season_length`` is part of the registry-side contract — every diff --git a/forecaster/src/promforecast/models/mstl.py b/forecaster/src/promforecast/models/mstl.py index 5a7f017..526e1cd 100644 --- a/forecaster/src/promforecast/models/mstl.py +++ b/forecaster/src/promforecast/models/mstl.py @@ -20,6 +20,7 @@ class MSTLModel: name = "MSTL" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/models/seasonal_naive.py b/forecaster/src/promforecast/models/seasonal_naive.py index 457f8ee..aa68bba 100644 --- a/forecaster/src/promforecast/models/seasonal_naive.py +++ b/forecaster/src/promforecast/models/seasonal_naive.py @@ -14,6 +14,7 @@ class SeasonalNaiveModel: name = "SeasonalNaive" + uses_regressors = False def __init__(self, season_length: int = 288, **params: Any) -> None: self._season_length = season_length diff --git a/forecaster/src/promforecast/point_factory.py b/forecaster/src/promforecast/point_factory.py index 6f7ea99..8ab3e17 100644 --- a/forecaster/src/promforecast/point_factory.py +++ b/forecaster/src/promforecast/point_factory.py @@ -21,16 +21,24 @@ from . import deviation as deviation_mod from . import quality as quality_mod from .accuracy import overall_mape, overall_mase +from .capacity import DRAIN_COMPARATORS from .exporter import ( AccuracyPoint, CalibrationOffsetPoint, + ComponentDecompositionPoint, + ConditionalDeviationPoint, + ConditionalForecastPoint, ContributionPoint, DeviationPoint, EnsembleWeightPoint, + ErrorBudgetPoint, ForecastPoint, GrowthRatePoint, QualityPoint, + RecommendedMaintenanceWindowPoint, ThresholdEtaPoint, + TimeToDrainPoint, + WillBreachPoint, ) from .sink import ForecastCurvePoint @@ -40,9 +48,11 @@ import pandas as pd from .accuracy import AccuracyResult - from .capacity import GrowthRate, ThresholdEta + from .capacity import GrowthRate, ThresholdEta, WillBreach from .contribution import ContributionResult + from .maintenance import RecommendedWindow from .selector import SelectionResult + from .slo import BudgetSample def format_window(seconds: float) -> str: @@ -177,6 +187,188 @@ def forecast_points( value=float(last[hi_col]), ) + def conditional_forecast_points( + self, + forecast: pd.DataFrame, + model_name: str, + selection: SelectionResult | None, + ) -> Iterable[ConditionalForecastPoint]: + """Mirror of :meth:`forecast_points` for the conditional family. + + Same end-of-horizon convention — one snapshot value per (series, + model, level). The metric names carry the + ``_forecast_conditional`` suffix family instead of ``_forecast``. + """ + if forecast.empty: + return + last = forecast.iloc[-1] + common = self._build_common_labels(model_name, selection) + yield ConditionalForecastPoint( + metric=f"{self._metric_id}_forecast_conditional", + labels=common, + value=float(last["yhat"]), + ) + for level in self._levels: + lo_col = f"yhat_lower_{level}" + hi_col = f"yhat_upper_{level}" + if lo_col in forecast.columns: + yield ConditionalForecastPoint( + metric=f"{self._metric_id}_forecast_conditional_lower", + labels={**common, "level": str(level)}, + value=float(last[lo_col]), + ) + if hi_col in forecast.columns: + yield ConditionalForecastPoint( + metric=f"{self._metric_id}_forecast_conditional_upper", + labels={**common, "level": str(level)}, + value=float(last[hi_col]), + ) + + def conditional_forecast_curve_points( + self, + forecast: pd.DataFrame, + model_name: str, + selection: SelectionResult | None, + ) -> Iterable[ForecastCurvePoint]: + """Yield every horizon step as a push point for the conditional sink curve. + + Mirrors :meth:`forecast_curve_points` but emits under the + ``_forecast_conditional`` metric-name family so the sink lands + the conditional curves in a parallel TSDB family. Caller-side + renaming is no longer needed once this method is used. + + Rows with non-finite timestamps or values are dropped — TSDBs + reject NaN samples, and emitting them would just generate noise + in the sink's failure counter. + """ + yield from self._curve_points(forecast, model_name, selection, suffix_family="_conditional") + + def conditional_deviation_points( + self, + forecast: pd.DataFrame, + model_name: str, + actual: float, + ) -> Iterable[ConditionalDeviationPoint]: + """Yield one conditional deviation sample per emitted (model, level). + + ``actual`` is the trailing in-sample observation (the most + recent value of the input series). ``forecast`` is the + conditional-fit forecast frame — we read its **first** horizon + step as "what the model would predict immediately given the + held-constant regressors". + """ + if forecast.empty: + return + from . import deviation as deviation_mod # noqa: PLC0415 + + first = forecast.iloc[0] + try: + yhat = float(first["yhat"]) + except (KeyError, ValueError): + return + common = self._common_with_id() + for level in self._levels: + lo_col = f"yhat_lower_{level}" + hi_col = f"yhat_upper_{level}" + if lo_col not in forecast.columns or hi_col not in forecast.columns: + continue + try: + lo = float(first[lo_col]) + hi = float(first[hi_col]) + except (KeyError, ValueError): + continue + dev = deviation_mod.compute( + actual=actual, + yhat=yhat, + yhat_lower=lo, + yhat_upper=hi, + ) + yield ConditionalDeviationPoint( + labels={**common, "model": model_name, "level": str(level)}, + ratio=dev.ratio, + outside_band=dev.outside_band, + ) + + def component_curve_points( + self, + forecast: pd.DataFrame, + model_name: str, + components: dict[str, list[float]], + selection: SelectionResult | None, + ) -> Iterable[ForecastCurvePoint]: + """Yield per-(step, component) push-format points for the sink. + + Mirrors :meth:`forecast_curve_points` but emits one row per + component per horizon step. Consumed by the remote_write sink so + a Grafana stacked-area panel can plot how trend / seasonal / + level / residual layer up over the predicted curve. + + Rows with non-finite timestamps or values are dropped — TSDBs + reject NaN samples and emitting them just generates noise in the + sink's failure counter. + """ + if forecast.empty or not components: + return + rows = len(forecast) + common = self._build_common_labels(model_name, selection) + metric_name = f"{self._metric_id}_forecast_component" + for i in range(rows): + try: + ts = forecast["ds"].iloc[i] + ts_ms = int(float(ts.timestamp()) * 1000) + except (KeyError, AttributeError, TypeError, ValueError): + continue + for component_name, values in components.items(): + if len(values) != rows: + continue + raw = values[i] + if not math.isfinite(raw): + continue + yield ForecastCurvePoint( + metric=metric_name, + labels={**common, "component": component_name}, + value=float(raw), + timestamp_ms=ts_ms, + ) + + def component_points( + self, + forecast: pd.DataFrame, + model_name: str, + components: dict[str, list[float]], + selection: SelectionResult | None, + ) -> Iterable[ComponentDecompositionPoint]: + """Yield one end-of-horizon decomposition sample per component. + + Snapshot convention (mirrors :meth:`forecast_points`): emit the + value at the configured horizon as a single gauge sample per + (series, model, component). ``components`` maps each component + name (``trend`` / ``seasonal`` / ``level`` / ``residual``) to + the per-step value list — same length as the forecast frame. + + The full per-step curve flows through + :meth:`component_curve_points`; this method only produces the + scrape-side snapshot row used by /metrics, matching how + ``_forecast`` (a single snapshot point) and the sink curve + (every step) split today. + """ + if forecast.empty or not components: + return + rows = len(forecast) + common = self._build_common_labels(model_name, selection) + metric_name = f"{self._metric_id}_forecast_component" + for component_name, values in components.items(): + if len(values) != rows or not values: + continue + last_val = float(values[-1]) + if not math.isfinite(last_val): + continue + yield ComponentDecompositionPoint( + metric=metric_name, + labels={**common, "component": component_name}, + value=last_val, + ) + def drift_labels(self, model_name: str) -> dict[str, str]: """Common label set for a :class:`DriftPoint` in this series. @@ -222,9 +414,32 @@ def forecast_curve_points( NaN samples, and emitting them would just generate noise in the sink's failure counter. """ + yield from self._curve_points(forecast, model_name, selection, suffix_family="") + + def _curve_points( + self, + forecast: pd.DataFrame, + model_name: str, + selection: SelectionResult | None, + *, + suffix_family: str, + ) -> Iterable[ForecastCurvePoint]: + """Internal helper shared by every per-step curve emission method. + + ``suffix_family`` is inserted between ``_forecast`` and the + ``_lower`` / ``_upper`` band suffix — empty for the standard + family (``_forecast``, ``_forecast_lower``, + ``_forecast_upper``), ``"_conditional"`` for the conditional + family. Keeping the rendering logic in one place stops the two + public callers from drifting on band-handling or NaN-skip + semantics as new options arrive. + """ if forecast.empty: return common = self._build_common_labels(model_name, selection) + base_metric = f"{self._metric_id}_forecast{suffix_family}" + lower_metric = f"{self._metric_id}_forecast{suffix_family}_lower" + upper_metric = f"{self._metric_id}_forecast{suffix_family}_upper" for _, row in forecast.iterrows(): ts = row.get("ds") if ts is None: @@ -237,7 +452,7 @@ def forecast_curve_points( yhat_raw = row.get("yhat") if yhat_raw is not None and math.isfinite(float(yhat_raw)): yield ForecastCurvePoint( - metric=f"{self._metric_id}_forecast", + metric=base_metric, labels=common, value=float(yhat_raw), timestamp_ms=ts_ms, @@ -249,7 +464,7 @@ def forecast_curve_points( lo = row[lo_col] if math.isfinite(float(lo)): yield ForecastCurvePoint( - metric=f"{self._metric_id}_forecast_lower", + metric=lower_metric, labels={**common, "level": str(level)}, value=float(lo), timestamp_ms=ts_ms, @@ -258,7 +473,7 @@ def forecast_curve_points( hi = row[hi_col] if math.isfinite(float(hi)): yield ForecastCurvePoint( - metric=f"{self._metric_id}_forecast_upper", + metric=upper_metric, labels={**common, "level": str(level)}, value=float(hi), timestamp_ms=ts_ms, @@ -307,7 +522,7 @@ def threshold_eta_points( etas: list[ThresholdEta], selection: SelectionResult | None, ) -> Iterable[ThresholdEtaPoint]: - """Yield one ``_forecast_time_to_threshold_seconds`` per ETA. + """Yield one ``_forecast_time_to_threshold_seconds`` per non-drain ETA. ``etas`` carries the per-(threshold, level) ETA computed by :func:`promforecast.capacity.compute_threshold_etas`. The metric @@ -317,12 +532,20 @@ def threshold_eta_points( plus optional ``best_model``/``model_fallback``) and add ``name`` + ``comparator`` for the threshold, and ``level`` when the ETA was computed against a band. + + ETAs whose comparator is one of :data:`promforecast.capacity.DRAIN_COMPARATORS` + are *routed* to the drain family — see :meth:`time_to_drain_points` — + and not emitted here. Keeping the routing in the factory (rather + than the runner) means a future ``ge_fill`` or similar can land + with one new dispatch line and no runner change. """ if not etas: return common = self._build_common_labels(model_name, selection) metric_name = f"{self._metric_id}_forecast_time_to_threshold_seconds" for eta in etas: + if eta.comparator in DRAIN_COMPARATORS: + continue labels = { **common, "name": eta.name, @@ -336,6 +559,168 @@ def threshold_eta_points( seconds=eta.seconds, ) + def time_to_drain_points( + self, + model_name: str, + etas: list[ThresholdEta], + selection: SelectionResult | None, + ) -> Iterable[TimeToDrainPoint]: + """Yield one ``_forecast_time_to_drain_seconds`` per drain ETA. + + Sibling of :meth:`threshold_eta_points` for the ``le_drain`` + comparator. Different metric name (and the ``+Inf`` no-crossing + sentinel passes through to the exporter intact so the "queue is + not draining" alert can match it). + + Unlike the threshold-ETA family we do *not* emit a ``comparator`` + label: the metric name itself already carries the routing + ("drain"), and every row in this family would carry the same + value (``le_drain``) — a label that's constant across the family + is pure noise. + """ + if not etas: + return + common = self._build_common_labels(model_name, selection) + metric_name = f"{self._metric_id}_forecast_time_to_drain_seconds" + for eta in etas: + if eta.comparator not in DRAIN_COMPARATORS: + continue + labels = { + **common, + "name": eta.name, + } + if eta.level is not None: + labels["level"] = str(eta.level) + yield TimeToDrainPoint( + metric=metric_name, + labels=labels, + seconds=eta.seconds, + ) + + def error_budget_points( + self, + model_name: str, + samples: list[BudgetSample], + selection: SelectionResult | None, + ) -> Iterable[ErrorBudgetPoint]: + """Yield SLO error-budget snapshot rows for one (model, series). + + Each :class:`promforecast.slo.BudgetSample` fans out to up to + ``(2 + 1 + N_levels)`` rows: + + * one ``_forecast_error_budget_remaining_seconds{window}`` + row (the static allowed-error budget), + * one ``_forecast_error_budget_burn_rate{window}`` row + (the predicted consumption rate divided by the allowed rate), + * one ``_forecast_error_budget_exhausted_in_seconds{window}`` + row for the central forecast (no ``level`` label), plus + * one ``... _exhausted_in_seconds{window, level=}`` row per + confidence level the forecast carried. + + Mirrors the ETA family's per-id metric-name convention so + dashboards can join the underlying forecast against its derived + budget projection on ``id``. + """ + if not samples: + return + common = self._build_common_labels(model_name, selection) + remaining_metric = f"{self._metric_id}_forecast_error_budget_remaining_seconds" + burn_metric = f"{self._metric_id}_forecast_error_budget_burn_rate" + exh_metric = f"{self._metric_id}_forecast_error_budget_exhausted_in_seconds" + for sample in samples: + window_label = format_window(sample.window_seconds) + base = {**common, "window": window_label} + yield ErrorBudgetPoint( + metric=remaining_metric, + labels=base, + value=sample.remaining_seconds, + ) + yield ErrorBudgetPoint( + metric=burn_metric, + labels=base, + value=sample.burn_rate, + ) + # Central exhausted-in: no ``level`` label so an alert on + # the central forecast doesn't need to filter for a missing + # level. + yield ErrorBudgetPoint( + metric=exh_metric, + labels=base, + value=sample.exhausted_in_point, + ) + for level, exhausted_in in sample.exhausted_in_per_level.items(): + yield ErrorBudgetPoint( + metric=exh_metric, + labels={**base, "level": str(level)}, + value=exhausted_in, + ) + + def recommended_maintenance_window_points( + self, + model_name: str, + windows: list[RecommendedWindow], + selection: SelectionResult | None, + ) -> Iterable[RecommendedMaintenanceWindowPoint]: + """Yield one ``forecast_recommended_maintenance_window`` row per ranked window. + + ``rank``, ``start_timestamp_seconds`` and ``duration_seconds`` + all live on the label set so a single Grafana panel can render + the ranked recommendations across every opted-in query. The + sample value is the window's predicted load (mean ``yhat`` + across the window's rows); lower is better. ``rank=1`` is the + suggested window. + """ + if not windows: + return + common = self._build_common_labels(model_name, selection) + # The operational families add ``id`` as a discriminator label; + # do the same here so the metric is queryable by id across + # groups. ``_build_common_labels`` does not include ``id`` (it + # lives on the metric name for the snapshot families) so we + # splice it in explicitly. + common_with_id = {**common, "id": self._metric_id} + for window in windows: + yield RecommendedMaintenanceWindowPoint( + labels={ + **common_with_id, + "rank": str(window.rank), + "start_timestamp_seconds": str(int(window.start_timestamp)), + "duration_seconds": str(int(window.duration_seconds)), + }, + predicted_load=window.predicted_load, + ) + + def will_breach_points( + self, + model_name: str, + breaches: list[WillBreach], + selection: SelectionResult | None, + ) -> Iterable[WillBreachPoint]: + """Yield one ``_forecast_will_breach`` per derived 0/1 sample. + + Labels mirror the ETA family (``model``, ``horizon``, plus + optional ``best_model``/``model_fallback``) and add ``name`` + + ``severity`` for the threshold, plus ``level`` when the sample + was derived from a band ETA. + """ + if not breaches: + return + common = self._build_common_labels(model_name, selection) + metric_name = f"{self._metric_id}_forecast_will_breach" + for breach in breaches: + labels = { + **common, + "name": breach.name, + "severity": breach.severity, + } + if breach.level is not None: + labels["level"] = str(breach.level) + yield WillBreachPoint( + metric=metric_name, + labels=labels, + value=breach.value, + ) + def growth_rate_points( self, model_name: str, diff --git a/forecaster/src/promforecast/runner.py b/forecaster/src/promforecast/runner.py index ba5c919..c2b3980 100644 --- a/forecaster/src/promforecast/runner.py +++ b/forecaster/src/promforecast/runner.py @@ -26,15 +26,18 @@ from . import cold_start as cold_start_mod from . import conformal as conformal_mod from . import contribution as contribution_mod +from . import decomposition as decomposition_mod from . import deviation as deviation_mod from . import discovery as discovery_mod from . import drift as drift_mod +from . import maintenance as maintenance_mod from . import models as model_registry from . import point_factory as point_factory_mod from . import preprocess as preprocess_mod from . import quality as quality_mod from . import regressors as regressors_mod from . import seasonality as seasonality_mod +from . import slo as slo_mod from . import telemetry as telemetry_mod from .accuracy import ( AccuracyResult, @@ -59,10 +62,14 @@ AccuracyPoint, BandCoveragePoint, CalibrationOffsetPoint, + ComponentDecompositionPoint, + ConditionalDeviationPoint, + ConditionalForecastPoint, ContributionPoint, DeviationPoint, DriftPoint, EnsembleWeightPoint, + ErrorBudgetPoint, Exporter, ForecastPoint, GroupSnapshot, @@ -70,7 +77,10 @@ InvalidMetricNameError, PreprocessStats, QualityPoint, + RecommendedMaintenanceWindowPoint, ThresholdEtaPoint, + TimeToDrainPoint, + WillBreachPoint, validate_metric_name, ) from .point_factory import BacktestOutcome, BandSummary, SeriesPointFactory @@ -148,6 +158,42 @@ class _FitResult: # aggregated into the run context without monotonic merging. threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) growth_rates: list[GrowthRatePoint] = field(default_factory=list) + # Per-(model, threshold[, level]) 0/1 predictive will-breach samples + # derived from the ETAs above. Snapshot-style; folded straight into + # the run context without monotonic merging. + will_breach: list[WillBreachPoint] = field(default_factory=list) + # Drain-ETA family for ``le_drain`` thresholds. Sibling of + # ``threshold_etas`` — separate field rather than a re-used one so + # the exporter's per-family rendering can preserve ``+Inf`` samples + # (the "queue is not draining" alert needs them to round-trip). + time_to_drain: list[TimeToDrainPoint] = field(default_factory=list) + # SLO error-budget snapshot rows (remaining, burn rate, exhausted_in). + # Snapshot-style; folded straight into the run context. + error_budget: list[ErrorBudgetPoint] = field(default_factory=list) + # Recommended maintenance windows — top-N lowest-load windows in the + # forecast horizon for this (series, model). Snapshot-style. + recommended_windows: list[RecommendedMaintenanceWindowPoint] = field(default_factory=list) + # Conditional forecast emission: snapshot + deviation + curve. All + # three are produced together when ``emission.conditional: true`` is + # on for the query. The curve flows into the optional remote_write + # sink alongside the unconditional curve. + conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) + conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) + conditional_curve: list[ForecastCurvePoint] = field(default_factory=list) + # Component decomposition emission. Per-(series, model, component) + # end-of-horizon snapshot rows for /metrics + per-step curve points + # for the optional remote_write sink (consumed by the stacked-area + # explainability dashboard). + component_points: list[ComponentDecompositionPoint] = field(default_factory=list) + component_curve: list[ForecastCurvePoint] = field(default_factory=list) + # Per-reason conditional-fit failure counter. Bumped on every fit + # that times out or raises; merged into ``ctx.failures`` so the + # operator sees them under ``forecast_failures_total`` with a + # bounded ``conditional_fit_*`` reason set. + conditional_failures: dict[str, int] = field(default_factory=dict) + # ``(model_name, reason)`` skip records the runner folds into the + # exporter's per-(model, reason) decomposition-skipped counter. + decomposition_skipped: list[tuple[str, str]] = field(default_factory=list) # Per-model forecast dataframes retained until after conformal # calibration so the capacity helpers can compute ETAs against the # *calibrated* bands (a calibration that widens a band should pull @@ -258,6 +304,26 @@ class _RunContext: # only the current run's samples land on /metrics. threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) growth_rates: list[GrowthRatePoint] = field(default_factory=list) + # Will-breach predictive boolean gauges (per series/model/threshold/level). + will_breach_points: list[WillBreachPoint] = field(default_factory=list) + # Drain-ETA points (queue burn-down). Snapshot-style; preserves + # ``+Inf`` samples for the "queue is not draining" alert. + time_to_drain_points: list[TimeToDrainPoint] = field(default_factory=list) + # SLO error-budget snapshot rows accumulated this run. + error_budget_points: list[ErrorBudgetPoint] = field(default_factory=list) + # Recommended maintenance windows accumulated this run. + recommended_maintenance_window_points: list[RecommendedMaintenanceWindowPoint] = field( + default_factory=list + ) + # Conditional forecast emission accumulators (snapshot rows + + # per-(model, level) deviation gauges + per-step curve for the sink). + conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) + conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) + conditional_curve_points: list[ForecastCurvePoint] = field(default_factory=list) + # Per-(series, model, component) end-of-horizon decomposition rows + # + per-step curve for the sink. + component_points: list[ComponentDecompositionPoint] = field(default_factory=list) + component_curve_points: list[ForecastCurvePoint] = field(default_factory=list) # Per-query preprocessing statistics for the most recent run. preprocess_stats: dict[str, PreprocessStats] = field(default_factory=dict) # Per-(query, strategy) gap-fill count for this run; merged with the @@ -815,6 +881,15 @@ async def run_group( # noqa: PLR0912, PLR0915 contribution_points=ctx.contribution_points, threshold_etas=ctx.threshold_etas, growth_rates=ctx.growth_rates, + will_breach_points=ctx.will_breach_points, + time_to_drain_points=ctx.time_to_drain_points, + error_budget_points=ctx.error_budget_points, + recommended_maintenance_window_points=( + ctx.recommended_maintenance_window_points + ), + conditional_points=ctx.conditional_points, + conditional_deviation_points=ctx.conditional_deviation_points, + component_points=ctx.component_points, last_run_timestamp=time.time(), run_duration_seconds=duration, run_duration_per_query=dict(ctx.run_duration_per_query), @@ -876,19 +951,22 @@ async def run_group( # noqa: PLR0912, PLR0915 # ``on_retry`` is bound to *this* group so transient retries # show up under the right label without the sink needing to # know about group identity. - if self._sink is not None and ctx.curve_points: + sink_payload = ( + ctx.curve_points + ctx.conditional_curve_points + ctx.component_curve_points + ) + if self._sink is not None and sink_payload: def _record_sink_retry(reason: str) -> None: self._exporter.increment_remote_write_retry(group.name, reason) try: - await self._sink.write(ctx.curve_points, on_retry=_record_sink_retry) + await self._sink.write(sink_payload, on_retry=_record_sink_retry) except SinkError as exc: log.warning( "remote_write_failed", reason=exc.reason, error=str(exc), - points=len(ctx.curve_points), + points=len(sink_payload), ) self._exporter.increment_remote_write_failure(group.name, exc.reason) except Exception as exc: @@ -1695,6 +1773,57 @@ async def _fit_series( # noqa: PLR0912, PLR0915 result=result, offsets=offsets, ) + # Component decomposition emission. Pure post-processing on the + # already-computed forecast frame, so it costs no extra model + # fits. Runs on every emitted (model, series) when + # ``emission.decompose`` is on; unsupported models log a skip + # counter rather than emit garbage. + if query.emission.decompose: + for ci in result.capacity_inputs: + self._emit_decomposition( + group_name=group_name, + query_id=query.id, + forecast=ci.forecast, + model_name=ci.model_name, + selection=ci.selection, + factory=factory, + in_sample_values=list(frame.values), + season_length=eff_season_length, + result=result, + ) + + # Conditional forecast emission. Re-fits each emitted model with + # a regressors frame whose values are flat-held at the most + # recent observation — operators alert on the divergence from the + # *conditional* baseline ("is the actual unusual given current + # regressors") rather than the unconditional band. For built-in + # models that ignore the regressors parameter the conditional + # output equals the unconditional output; the family still emits + # so downstream alerts have a stable schema. The pass reuses the + # unconditional fit's conformal offsets + count-aware safeguard + # toggles so the two families never disagree on calibrated band + # width or clipping policy. + if query.emission.conditional: + cond_clip, cond_rounder = effective_emission_safeguards(query) + await self._emit_conditional_forecasts( + df=df, + query=query, + group_name=group_name, + model_specs=model_specs, + emitted_inputs=list(result.capacity_inputs), + actual=float(frame.values[-1]) if frame.values else float("nan"), + horizon=horizon, + freq=freq, + levels=levels, + season_length=eff_season_length, + regressors=regressors_frame, + factory=factory, + result=result, + offsets_by_model=offsets_by_model, + clip=cond_clip, + rounder=cond_rounder, + ) + # Free the retained per-model dataframes — they can be MB-scale, # and ``_RunContext`` retains them across the per-series loop. result.capacity_inputs.clear() @@ -1982,8 +2111,21 @@ def _emit_capacity_metrics( """ if forecast.empty: return + # Apply the conformal offsets once and reuse the calibrated frame + # for every consumer that reads ``yhat_upper_`` / + # ``yhat_lower_`` (threshold ETAs and the SLO budget + # projection). Without the cache a query with both ``thresholds:`` + # and ``slo:`` pays the DataFrame ``.copy()`` twice per fit. + # ``_apply_offsets_to_threshold_eta_inputs`` short-circuits to the + # original frame when no calibration ran, so the common + # zero-offsets path stays zero-copy. + needs_calibrated_bands = bool(query.thresholds) or query.slo is not None + calibrated_df = ( + _apply_offsets_to_threshold_eta_inputs(forecast, offsets, levels) + if needs_calibrated_bands + else forecast + ) if query.thresholds: - calibrated_df = _apply_offsets_to_threshold_eta_inputs(forecast, offsets, levels) etas = capacity_mod.compute_threshold_etas( forecast=calibrated_df, thresholds=query.thresholds, @@ -1993,6 +2135,25 @@ def _emit_capacity_metrics( result.threshold_etas.extend( factory.threshold_eta_points(model_name, etas, selection) ) + # The drain ETA family is the parallel emission for + # ``le_drain`` thresholds — ``threshold_eta_points`` + # already filters them out so the two metric names + # don't collide. Routing happens in the factory so + # adding a future ``ge_fill`` (or similar) lands as + # one new dispatch line, not a fresh runner branch. + result.time_to_drain.extend( + factory.time_to_drain_points(model_name, etas, selection) + ) + # Derive the predictive will_breach gauge from the same + # ETA set so an alert on ``forecast_will_breach{...} == 1`` + # is always self-consistent with the + # ``forecast_time_to_threshold_seconds`` numbers + # displayed alongside it. + breaches = capacity_mod.compute_will_breach(etas=etas, thresholds=query.thresholds) + if breaches: + result.will_breach.extend( + factory.will_breach_points(model_name, breaches, selection) + ) growth_cfg = query.emission.growth_rate if growth_cfg.enabled and growth_cfg.windows: windows_seconds = [w.total_seconds() for w in growth_cfg.windows] @@ -2003,6 +2164,273 @@ def _emit_capacity_metrics( ) if rates: result.growth_rates.extend(factory.growth_rate_points(model_name, rates, selection)) + # SLO error budget projection. Off unless ``slo:`` is configured + # on the query; uses the leading forecast row as "now" so the + # burn rate / exhausted-in match what an operator reading the + # current /metrics scrape would compute by hand. Reads + # ``yhat_upper_N`` from the *calibrated* frame so the per-level + # ``exhausted_in`` agrees with the band an operator queries via + # ``yhat_upper`` at the same labelset. + if query.slo is not None: + budget_samples = slo_mod.compute_budget( + forecast=calibrated_df, + slo=query.slo, + levels=levels, + ) + if budget_samples: + result.error_budget.extend( + factory.error_budget_points(model_name, budget_samples, selection) + ) + # Recommended maintenance windows. Sweeps the lookahead for the + # top-N lowest-mean-load windows of the configured duration. + rw_cfg = query.recommended_windows + if rw_cfg.enabled: + windows = maintenance_mod.recommend_windows(forecast=forecast, config=rw_cfg) + if windows: + result.recommended_windows.extend( + factory.recommended_maintenance_window_points(model_name, windows, selection) + ) + + def _emit_decomposition( + self, + *, + group_name: str, + query_id: str, + forecast: pd.DataFrame, + model_name: str, + selection: SelectionResult | None, + factory: SeriesPointFactory, + in_sample_values: list[float], + season_length: int, + result: _FitResult, + ) -> None: + """Project ``forecast`` into trend/seasonal/level/residual snapshot rows. + + Pure post-processing on the already-computed forecast frame; + cost is bounded to a few hundred float operations per (series, + model). Models listed in + :data:`decomposition.NON_DECOMPOSABLE_MODELS` skip with + ``reason="unsupported_model"`` so operators see the no-op via + ``forecast_decomposition_skipped_total``. ``group_name`` / + ``query_id`` flow onto the skip counter so persistent skips + can be alerted per-query without log scraping. + """ + if not decomposition_mod.is_decomposable(model_name): + result.decomposition_skipped.append((model_name, "unsupported_model")) + self._exporter.increment_decomposition_skipped( + group_name, query_id, model_name, "unsupported_model" + ) + return + try: + decomposition = decomposition_mod.decompose( + forecast=forecast, + in_sample_values=in_sample_values, + season_length=season_length, + ) + except decomposition_mod.DecompositionError as exc: + result.decomposition_skipped.append((model_name, exc.reason)) + self._exporter.increment_decomposition_skipped( + group_name, query_id, model_name, exc.reason + ) + return + if not decomposition.components: + result.decomposition_skipped.append((model_name, "components_unavailable")) + self._exporter.increment_decomposition_skipped( + group_name, query_id, model_name, "components_unavailable" + ) + return + result.component_points.extend( + factory.component_points( + forecast=forecast, + model_name=model_name, + components=decomposition.components, + selection=selection, + ) + ) + result.component_curve.extend( + factory.component_curve_points( + forecast=forecast, + model_name=model_name, + components=decomposition.components, + selection=selection, + ) + ) + + async def _emit_conditional_forecasts( + self, + *, + df: pd.DataFrame, + query: QueryConfig, + group_name: str, + model_specs: list[ModelSpec], + emitted_inputs: list[_CapacityInput], + actual: float, + horizon: int, + freq: str, + levels: list[int], + season_length: int, + regressors: pd.DataFrame | None, + factory: SeriesPointFactory, + result: _FitResult, + offsets_by_model: dict[str, dict[int, float]], + clip: bool, + rounder: bool, + ) -> None: + """Re-fit each emitted model with held-constant regressors. + + The pass adds one fit per emitted (model, series) on top of the + regular fit, so the cost roughly doubles the forecast budget + for queries that opt in. See ``docs/conditional-forecasts.md``. + + When ``regressors`` is None (no regressors configured), the + held-constant frame is also None and the re-fit produces the + same numbers as the unconditional pass — the family still emits + so dashboards and alerts can target it unconditionally. + + ``offsets_by_model`` reuses the unconditional fit's conformal + offsets so the conditional bands carry the same calibration + widening as the unconditional ones — otherwise an operator + comparing the two would see different empirical-coverage + guarantees and infer something incorrect. + + ``clip`` / ``rounder`` mirror the count-aware safeguards applied + to the unconditional emission, so a count-shaped query sees + consistent rounding/clipping on both families. + + Per-fit failures bump ``forecast_failures_total`` with a + distinct ``conditional_fit_error`` / ``conditional_fit_timeout`` + reason so the operator can tell which fit family is misbehaving + without log scraping. + """ + conditional_regressors = _flatten_regressors_to_last_value(regressors) + spec_by_name = {spec.name: spec for spec in model_specs} + # Two short-circuits below skip the refit when it would be a + # waste of CPU: + # 1. No regressors configured at all -> the held-constant frame + # is identical to the original input, so the conditional fit + # would produce the same numbers. + # 2. The model advertises ``uses_regressors = False`` -> the + # model's ``fit_predict`` ignores the regressors parameter, + # so a refit with held-constant regressors produces the same + # numbers as the unconditional fit. + # In both cases we alias the already-computed unconditional frame + # rather than burning a fresh fit. The family still emits so + # downstream alerts have a stable schema (the docs explicitly + # promise this for queries with no regressors). + no_regressors_configured = regressors is None + for ci in emitted_inputs: + model_name = ci.model_name + # The synthetic ``ensemble`` row in ensemble mode doesn't + # correspond to a registered model, so we can't re-fit it. + # Skip cleanly rather than papering over with a bad fit. + if model_name == "ensemble": + continue + spec = spec_by_name.get(model_name) or ModelSpec(name=model_name) + short_circuit = no_regressors_configured or not model_registry.uses_regressors( + model_name + ) + if short_circuit: + # Alias the unconditional forecast — same numeric result + # the refit would produce, at zero extra CPU. No + # ``fit_duration`` is recorded for an aliased emission + # because no fit ran; the slow-fit counter likewise + # isn't touched. We *do* bump the operational + # ``forecast_conditional_aliased_total`` counter so + # operators can confirm the short-circuit is firing — + # a flat-zero counter means every conditional emission + # is paying the refit cost. + self._exporter.increment_conditional_aliased(group_name, query.id, model_name) + cond_forecast = ci.forecast + else: + t0 = time.monotonic() + try: + cond_forecast = await self._fit_one( + spec=spec, + df=df, + horizon=horizon, + freq=freq, + levels=levels, + season_length=season_length, + regressors=conditional_regressors, + ) + except TimeoutError: + logger.warning( + "conditional_fit_timeout", + group=group_name, + query=query.id, + model=model_name, + ) + result.conditional_failures["conditional_fit_timeout"] = ( + result.conditional_failures.get("conditional_fit_timeout", 0) + 1 + ) + continue + except Exception as exc: + logger.warning( + "conditional_fit_failed", + group=group_name, + query=query.id, + model=model_name, + error=str(exc), + ) + result.conditional_failures["conditional_fit_error"] = ( + result.conditional_failures.get("conditional_fit_error", 0) + 1 + ) + continue + fit_seconds = time.monotonic() - t0 + # Track the conditional fit's duration alongside the + # unconditional one so ``forecast_fit_duration_seconds`` + # reports both. Without this the slow-fit counter would + # tick under ``model="X/conditional"`` but the gauge + # would be silently absent. + result.fit_durations[f"{model_name}/conditional"] = fit_seconds + self._record_slow_fit_if_needed( + result=result, + group=group_name, + query_id=query.id, + model_name=f"{model_name}/conditional", + fit_seconds=fit_seconds, + ) + + # Reuse the unconditional offsets so the conditional bands + # carry the same empirical-coverage guarantee. ``offsets_by_model`` + # may not contain ``model_name`` (no calibration ran or + # ensemble row), in which case the helper is a no-op. + model_offsets = offsets_by_model.get(model_name) + calibrated_df = _apply_offsets_to_threshold_eta_inputs( + cond_forecast, model_offsets, levels + ) + # Apply count-aware safeguards to the dataframe columns so + # the snapshot rows, the sink curve, and the deviation + # gauges all see the same clipped/rounded values. + adjusted_df = _apply_safeguards_to_forecast_frame( + calibrated_df, levels=levels, clip=clip, rounder=rounder + ) + + result.conditional_points.extend( + factory.conditional_forecast_points( + forecast=adjusted_df, + model_name=model_name, + selection=ci.selection, + ) + ) + # Stream the per-step conditional curve into the sink list so + # remote_write users see the same smooth curve they get for + # the unconditional family. The curve uses a parallel metric + # name family (``_forecast_conditional*``) so the two paths + # never collide on the TSDB side. + result.conditional_curve.extend( + factory.conditional_forecast_curve_points( + adjusted_df, model_name, selection=ci.selection + ) + ) + if math.isfinite(actual): + result.conditional_deviation_points.extend( + factory.conditional_deviation_points( + forecast=adjusted_df, + model_name=model_name, + actual=actual, + ) + ) async def _explicit_forecast( self, @@ -2991,6 +3419,17 @@ def _merge_fit_into_ctx(ctx: _RunContext, fit_result: _FitResult, query_id: str) ctx.calibration_offsets.extend(fit_result.calibration_offsets) ctx.threshold_etas.extend(fit_result.threshold_etas) ctx.growth_rates.extend(fit_result.growth_rates) + ctx.will_breach_points.extend(fit_result.will_breach) + ctx.time_to_drain_points.extend(fit_result.time_to_drain) + ctx.error_budget_points.extend(fit_result.error_budget) + ctx.recommended_maintenance_window_points.extend(fit_result.recommended_windows) + ctx.conditional_points.extend(fit_result.conditional_points) + ctx.conditional_deviation_points.extend(fit_result.conditional_deviation_points) + ctx.conditional_curve_points.extend(fit_result.conditional_curve) + ctx.component_points.extend(fit_result.component_points) + ctx.component_curve_points.extend(fit_result.component_curve) + for reason, count in fit_result.conditional_failures.items(): + ctx.failures[reason] = ctx.failures.get(reason, 0) + count for model_name, fit_seconds in fit_result.fit_durations.items(): ctx.fit_durations[(query_id, model_name)] = fit_seconds for model_name in fit_result.slow_fits: @@ -3065,6 +3504,29 @@ def _inherit_prior_for_skipped_queries( # noqa: PLR0912 for gpoint in prior.growth_rates: if _query_id_from_metric(gpoint.metric) not in run_query_ids: ctx.growth_rates.append(gpoint) + for wbpoint in prior.will_breach_points: + if _query_id_from_metric(wbpoint.metric) not in run_query_ids: + ctx.will_breach_points.append(wbpoint) + for drain_point in prior.time_to_drain_points: + if _query_id_from_metric(drain_point.metric) not in run_query_ids: + ctx.time_to_drain_points.append(drain_point) + for ebpoint in prior.error_budget_points: + if _query_id_from_metric(ebpoint.metric) not in run_query_ids: + ctx.error_budget_points.append(ebpoint) + # ``forecast_recommended_maintenance_window`` is a global metric + # name (no per-id prefix); use the ``id`` label to scope inheritance. + for rwpoint in prior.recommended_maintenance_window_points: + if rwpoint.labels.get("id") not in run_query_ids: + ctx.recommended_maintenance_window_points.append(rwpoint) + for cfpoint in prior.conditional_points: + if _query_id_from_metric(cfpoint.metric) not in run_query_ids: + ctx.conditional_points.append(cfpoint) + for cdpoint in prior.conditional_deviation_points: + if cdpoint.labels.get("id") not in run_query_ids: + ctx.conditional_deviation_points.append(cdpoint) + for compoint in prior.component_points: + if _query_id_from_metric(compoint.metric) not in run_query_ids: + ctx.component_points.append(compoint) # Per-(query, model) and per-query dicts that aren't counters. for (q_id, model_name), fit_seconds in prior.fit_durations.items(): if q_id not in run_query_ids: @@ -3094,6 +3556,99 @@ def _inherit_prior_for_skipped_queries( # noqa: PLR0912 ctx.series_count_per_query[q_id] = count +def _apply_safeguards_to_forecast_frame( + forecast: pd.DataFrame, + *, + levels: list[int], + clip: bool, + rounder: bool, +) -> pd.DataFrame: + """Return a copy of ``forecast`` with count-aware safeguards applied. + + Used by the conditional emission path so the snapshot points, the + sink curve, and the deviation gauges built from the same frame all + see the same clipped/rounded values. Counter bumps live with the + unconditional pass — the conditional emission re-uses the same fit + semantics and counting twice would double-charge the per-(query, + kind) total. + + Uses numpy vectorisation so a typical horizon (288 rows by 5 columns + at default 2 levels) is one pass per column instead of 1440 + Python-level float conversions. + """ + if forecast.empty or not (clip or rounder): + return forecast + import numpy as np # noqa: PLC0415 + import pandas as pd # noqa: PLC0415 + + out = forecast.copy() + cols = ["yhat"] + [ + col for level in levels for col in (f"yhat_lower_{level}", f"yhat_upper_{level}") + ] + for col in cols: + if col not in out.columns: + continue + # Coerce to float so clip / round have well-defined semantics + # against pandas extension-array columns. ``errors="coerce"`` + # turns non-numeric entries into NaN, which both numpy ops + # tolerate and propagate — matching the Python-loop version's + # behaviour of passing such entries through unchanged. + arr = pd.to_numeric(out[col], errors="coerce").to_numpy(dtype=float) + # Pre-record the NaN mask so non-finite inputs round-trip + # unchanged: ``np.round`` of NaN is NaN already, but clipping + # would turn NaN into 0.0 under naive ``np.maximum`` semantics — + # we restore NaN at the end so the contract matches the scalar + # ``_classify_emission_correction`` path. + nan_mask = ~np.isfinite(arr) + if clip: + # ``np.maximum`` preserves the input shape and dtype (unlike + # ``np.where`` whose typed return is multidim) — the column + # is 1-D so this is the right primitive here. + arr = np.maximum(arr, 0.0) + if rounder: + arr = np.round(arr) + if nan_mask.any(): + arr[nan_mask] = np.nan + out[col] = arr + return out + + +def _flatten_regressors_to_last_value( + regressors: pd.DataFrame | None, +) -> pd.DataFrame | None: + """Return a copy of ``regressors`` whose every row equals the last row. + + Implements the "held at current value" semantics of the conditional + forecast: each regressor column is replaced with a constant series + equal to its most recent in-sample value. Calendar features (which + legitimately vary per timestamp) are preserved through the same + flattening — the conditional contract is "imagine current state + continued unchanged", which by construction freezes the calendar as + well. Returns ``None`` when no regressors were configured so the + caller skips the re-fit's regressor argument cleanly. + + Uses scalar broadcast (``flat[col] = scalar``) instead of building a + Python list per column so a wide regressor frame doesn't materialise + N copies of the same value through the heap. pandas treats the + scalar assignment as a constant broadcast and writes a single typed + array. + """ + if regressors is None or regressors.empty: + return regressors + import pandas as pd # noqa: PLC0415 + + last_row = regressors.iloc[-1] + flat = pd.DataFrame(index=regressors.index) + for col in regressors.columns: + if col == "ds": + flat[col] = regressors[col].to_numpy() + continue + # Scalar broadcast: pandas writes a constant array of the right + # dtype without allocating a Python list of N references. + flat[col] = last_row[col] + return flat + + def _series_key_from_labels(labels: dict[str, str]) -> str: """Stable per-series key derived from the label set. @@ -3122,7 +3677,16 @@ def _format_band_window(window: timedelta) -> str: _FORECAST_METRIC_SUFFIXES: tuple[str, ...] = ( "_forecast_contribution_ratio", "_forecast_time_to_threshold_seconds", + "_forecast_time_to_drain_seconds", + "_forecast_error_budget_remaining_seconds", + "_forecast_error_budget_exhausted_in_seconds", + "_forecast_error_budget_burn_rate", "_forecast_growth_rate", + "_forecast_will_breach", + "_forecast_conditional_lower", + "_forecast_conditional_upper", + "_forecast_conditional", + "_forecast_component", "_forecast_lower", "_forecast_upper", "_forecast", diff --git a/forecaster/src/promforecast/slo.py b/forecaster/src/promforecast/slo.py new file mode 100644 index 0000000..d63bf33 --- /dev/null +++ b/forecaster/src/promforecast/slo.py @@ -0,0 +1,149 @@ +"""Per-query SLO error-budget forecasting. + +A forecast that represents an error rate (or any rate of bad events) is +the input to three derivations that map the SRE workbook's burn-rate +vocabulary onto first-class Prometheus metrics: + +* ``_forecast_error_budget_remaining_seconds{window}`` — the + total allowed error budget for the configured window. With an SLO + objective of 99.9% over a 30d window this is ``0.001 * 30d = 2592s``. +* ``_forecast_error_budget_burn_rate{window}`` — the predicted + consumption rate divided by the allowed rate. A value of ``1`` means + "burning the budget at exactly the rate that exhausts it in one + window"; values above ``1`` mean the budget is being consumed faster + than sustainable. +* ``_forecast_error_budget_exhausted_in_seconds{window[, level]}`` — + projected time until the cumulative consumption at the forecast's + current rate exceeds the budget. The point row carries no ``level`` + label; band rows use ``yhat_upper`` (the conservative side — a + higher predicted rate exhausts the budget sooner). + +Pure-function implementation so the runner stays orchestration-shaped +and the tests exercise the math directly. Mirrors the structure of +:mod:`promforecast.capacity`. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd + + from .config import SLOConfig + +# Objective is configured as a percentage in (0, 100); divide by this +# to get the fractional allowed-error rate (``0.001`` for a 99.9% SLO). +_PERCENT_FULL = 100.0 + + +@dataclass(frozen=True) +class BudgetSample: + """Per-(window) SLO budget samples derived from one forecast. + + ``exhausted_in_per_level`` is a ``{level: seconds}`` map; ``level`` + is ``None`` for the point sample tracked separately on + ``exhausted_in_point`` (mirrors the ``level=None`` convention on + :class:`promforecast.capacity.ThresholdEta`). All time values are in + seconds; non-positive forecast values produce ``+inf`` (= "budget + never exhausts at this rate") rather than NaN so an alert on + ``... < N`` is well-defined and won't silently match. + """ + + window_seconds: float + remaining_seconds: float + burn_rate: float + exhausted_in_point: float + exhausted_in_per_level: dict[int, float] = field(default_factory=dict) + + +def compute_budget( + *, + forecast: pd.DataFrame, + slo: SLOConfig, + levels: list[int], +) -> list[BudgetSample]: + """Project ``forecast`` onto per-window SLO budget samples. + + The forecast is interpreted as the instantaneous bad-event rate + (e.g. fraction of failing requests, fraction of time in error). The + leading row is "now" — the burn rate and exhausted-in projections + are computed from that snapshot to match how alerts read the + metrics. + + Returns one :class:`BudgetSample` per configured window. Empty input, + or a forecast missing the ``yhat`` column, yields the empty list so + the caller can ``extend`` unconditionally. + """ + if forecast.empty or "yhat" not in forecast.columns: + return [] + if not slo.windows: + return [] + # Pydantic already enforces 0 < objective < 100; guard at runtime too + # so a hand-built config (tests, preview) can't smuggle a degenerate + # value into the math. + if not 0.0 < slo.objective < _PERCENT_FULL: + return [] + allowed_rate = (_PERCENT_FULL - slo.objective) / _PERCENT_FULL + try: + current_yhat = float(forecast["yhat"].iloc[0]) + except (KeyError, ValueError, IndexError): + return [] + out: list[BudgetSample] = [] + for window in slo.windows: + window_seconds = window.total_seconds() + if window_seconds <= 0: + continue + budget_seconds = allowed_rate * window_seconds + # Point burn rate from yhat. ``yhat < 0`` is unusual for an + # error rate but well-defined here: a negative rate means the + # error budget is *recovering*, which the burn rate should + # report as a negative number (operators read it directly). + if math.isfinite(current_yhat) and allowed_rate > 0: + burn_rate = current_yhat / allowed_rate + else: + burn_rate = float("nan") + exhausted_in_point = _exhausted_in(budget_seconds, current_yhat) + exhausted_in_per_level: dict[int, float] = {} + for level in levels: + col = f"yhat_upper_{level}" + if col not in forecast.columns: + continue + try: + upper = float(forecast[col].iloc[0]) + except (KeyError, ValueError, IndexError): + continue + exhausted_in_per_level[level] = _exhausted_in(budget_seconds, upper) + out.append( + BudgetSample( + window_seconds=window_seconds, + remaining_seconds=budget_seconds, + burn_rate=burn_rate, + exhausted_in_point=exhausted_in_point, + exhausted_in_per_level=exhausted_in_per_level, + ) + ) + return out + + +def _exhausted_in(budget_seconds: float, rate: float) -> float: + """Return seconds until ``budget_seconds`` is consumed at ``rate``. + + ``rate <= 0`` (no errors, or an error budget that's recovering) + means the budget never exhausts at this trajectory; we report that + as ``+inf`` rather than NaN so alerts on + ``... > some_threshold`` work the way operators expect — ``inf`` + propagates the "no concern" answer instead of silently dropping + the sample. + + Non-finite ``rate`` (degenerate forecasts) yields ``NaN`` so the + exporter drops the row at render time. Mixing the two would mask + real model breakage behind a comforting "no exhaustion" answer. + """ + if not math.isfinite(rate): + return float("nan") + if rate <= 0: + return float("inf") + return budget_seconds / rate diff --git a/forecaster/src/promforecast/validate.py b/forecaster/src/promforecast/validate.py index be6b65b..3bd631e 100644 --- a/forecaster/src/promforecast/validate.py +++ b/forecaster/src/promforecast/validate.py @@ -59,7 +59,14 @@ class CardinalityEstimate: calibration_lines: int preprocess_lines: int threshold_eta_lines: int + will_breach_lines: int growth_rate_lines: int + # Conditional emission: parallel forecast snapshot + deviation rows + # for queries that opt into ``emission.conditional: true``. + conditional_lines: int + # Component decomposition: per-(series, model, component) rows for + # queries that opt into ``emission.decompose: true``. + decomposition_lines: int # Count-aware emission counter — per (query, kind=negative|rounded) # when a query opts in via ``data_profile: count`` or sets the # explicit emission toggles. @@ -73,6 +80,12 @@ class CardinalityEstimate: # Empirical band coverage: per-(series, model, level, window) rolling # gauge for every query that opts into ``emission.band_coverage``. band_coverage_lines: int + # SLO error-budget snapshot rows (remaining + burn_rate + exhausted_in) + # per (query, window). Only counted for queries that opt into ``slo:``. + slo_lines: int + # Recommended maintenance window rows — bounded by ``count`` per + # opted-in query. + recommended_window_lines: int operational_lines: int @property @@ -86,11 +99,16 @@ def total(self) -> int: + self.calibration_lines + self.preprocess_lines + self.threshold_eta_lines + + self.will_breach_lines + self.growth_rate_lines + + self.conditional_lines + + self.decomposition_lines + self.emission_clipped_lines + self.cold_start_lines + self.drift_lines + self.band_coverage_lines + + self.slo_lines + + self.recommended_window_lines + self.operational_lines ) @@ -100,12 +118,20 @@ def run_validate( *, dry_run: bool = False, datasource_url_override: str | None = None, + strict_cardinality: bool = False, out: IO[str] | None = None, ) -> int: """Validate ``config_path`` and optionally run a sample fit. - Returns 0 on success, 1 on validation failure, 2 on dry-run failure. + Returns 0 on success, 1 on validation failure (including + over-budget cardinality in strict mode), 2 on dry-run failure. Output is plain text by default; ``out`` lets tests capture it. + + ``strict_cardinality`` flips the projected-total overshoot from an + advisory ``WARN:`` line into a hard fail (exit code 1) so a CI job + on a config PR can reject configs whose projected emission exceeds + ``safety.max_total_series`` before the change ships. The knob is + opt-in so existing callers keep their advisory behaviour. """ stream: IO[str] = out if out is not None else sys.stdout @@ -145,18 +171,27 @@ def err(line: str) -> None: write(f" calibration lines {estimate.calibration_lines:>8}") write(f" preprocess lines {estimate.preprocess_lines:>8}") write(f" threshold ETA {estimate.threshold_eta_lines:>8}") + write(f" will-breach lines {estimate.will_breach_lines:>8}") write(f" growth rate lines {estimate.growth_rate_lines:>8}") + write(f" conditional lines {estimate.conditional_lines:>8}") + write(f" decomposition lines {estimate.decomposition_lines:>8}") write(f" emission clipped {estimate.emission_clipped_lines:>8}") write(f" cold start lines {estimate.cold_start_lines:>8}") write(f" drift lines {estimate.drift_lines:>8}") write(f" band coverage {estimate.band_coverage_lines:>8}") + write(f" slo lines {estimate.slo_lines:>8}") + write(f" recommended wins {estimate.recommended_window_lines:>8}") write(f" operational lines {estimate.operational_lines:>8}") write(f" total {estimate.total:>8}") - if cfg.safety.max_total_series and estimate.total > cfg.safety.max_total_series: - write( - f" WARN: total exceeds safety.max_total_series ({cfg.safety.max_total_series}); " - "series_overflow will drop the excess" - ) + rc = _handle_cardinality_overshoot( + cfg=cfg, + estimate=estimate, + strict_cardinality=strict_cardinality, + write=write, + err=err, + ) + if rc != 0: + return rc if not dry_run: return 0 @@ -171,6 +206,44 @@ def err(line: str) -> None: return rc +def _handle_cardinality_overshoot( + *, + cfg: Config, + estimate: CardinalityEstimate, + strict_cardinality: bool, + write: Writer, + err: Writer, +) -> int: + """Apply the per-mode policy when the estimate overshoots the cap. + + Returns ``0`` when the estimate fits inside ``safety.max_total_series`` + *or* when the legacy advisory-only mode is in effect (the caller emits + the WARN line and continues). Returns ``1`` only in + ``strict_cardinality`` mode when the projection exceeds the cap. + + Pulled out of :func:`run_validate` so the runtime statement count + stays under ruff's PLR0915 threshold without hand-waving with a + blanket noqa. + """ + over_budget = bool(cfg.safety.max_total_series and estimate.total > cfg.safety.max_total_series) + if not over_budget: + return 0 + if strict_cardinality: + err( + f"error: projected total ({estimate.total}) exceeds " + f"safety.max_total_series ({cfg.safety.max_total_series}); " + "strict-cardinality mode rejects over-budget configs. " + "Raise the cap, narrow your queries, or disable opt-in emission " + "families. See docs/operations/cardinality.md." + ) + return 1 + write( + f" WARN: total exceeds safety.max_total_series ({cfg.safety.max_total_series}); " + "series_overflow will drop the excess" + ) + return 0 + + def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, PLR0915 """Worst-case forecast/accuracy/deviation/quality/operational line counts. @@ -240,6 +313,10 @@ def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, # band ETA for each threshold. Counted only against queries that # actually configure thresholds — most don't. threshold_eta_lines = 0 + # Predictive will-breach gauge: same shape as the ETA family above + # but only for thresholds that opt into ``alert_within`` + ``severity``. + # ``N_thresholds_with_alert * (N_levels + 1)`` per (series, model). + will_breach_lines = 0 for group in cfg.groups: for q in group.queries: if not q.thresholds: @@ -248,16 +325,40 @@ def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, threshold_eta_lines += ( series_cap * emitted_models * len(q.thresholds) * per_threshold_emissions ) + opted_in = sum( + 1 for t in q.thresholds if t.alert_within is not None and t.severity is not None + ) + if opted_in: + will_breach_lines += ( + series_cap * emitted_models * opted_in * per_threshold_emissions + ) # Growth-rate: per (series, model, window) gauge. Counted only against # queries that opt into ``emission.growth_rate.enabled: true``. growth_rate_lines = 0 + # Conditional forecast: parallel emission family + per-(series, model, + # level) deviation gauges. Charged only against queries that opt + # into ``emission.conditional: true``. + conditional_lines = 0 + # Decomposition: per-(series, model, component=trend|seasonal|level|residual) + # — bounded to 4 lines per (series, model) per opted-in query. + decomposition_lines = 0 + n_components = 4 for group in cfg.groups: for q in group.queries: - if not q.emission.growth_rate.enabled: - continue - n_windows = max(1, len(q.emission.growth_rate.windows)) - growth_rate_lines += series_cap * emitted_models * n_windows + if q.emission.growth_rate.enabled: + n_windows = max(1, len(q.emission.growth_rate.windows)) + growth_rate_lines += series_cap * emitted_models * n_windows + if q.emission.conditional: + # Per-series snapshot: central + 2 * levels rows per model. + # Conditional deviation: 2 lines per (series, model, level) + # (ratio + outside_band). + conditional_lines += ( + series_cap * emitted_models * (1 + 2 * n_levels) + + series_cap * emitted_models * n_levels * 2 + ) + if q.emission.decompose: + decomposition_lines += series_cap * emitted_models * n_components # Preprocessing emits per-(group, query) telemetry. The label sets # are all bounded: # * max_gap_seconds : 1 line per query @@ -331,6 +432,29 @@ def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, n_windows = max(1, len(q.emission.band_coverage.windows)) band_coverage_lines += series_cap * emitted_models * n_levels * n_windows + # SLO error-budget projections. Three sibling metrics — remaining, + # burn_rate, exhausted_in — per (series, model, window). The + # exhausted_in row has an additional fan-out by level (central row + + # one per level), so we charge ``2 + 1 + N_levels = 3 + N_levels`` + # rows per (series, model, window). Bounded; only counted for + # queries that opt into ``slo:``. + slo_lines = 0 + for group in cfg.groups: + for q in group.queries: + if q.slo is None: + continue + n_windows = max(1, len(q.slo.windows)) + slo_lines += series_cap * emitted_models * n_windows * (3 + n_levels) + + # Recommended maintenance windows — ``count`` rows per (series, + # model) for queries that opt in. Bounded by construction. + recommended_window_lines = 0 + for group in cfg.groups: + for q in group.queries: + if not q.recommended_windows.enabled: + continue + recommended_window_lines += series_cap * emitted_models * q.recommended_windows.count + # Operational: 7 gauges + N reasons per group; we ballpark 7 per group. # The runner also emits per-(group, query) rows of # ``forecast_query_run_duration_seconds`` and @@ -352,11 +476,16 @@ def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, calibration_lines=calibration_lines, preprocess_lines=preprocess_lines, threshold_eta_lines=threshold_eta_lines, + will_breach_lines=will_breach_lines, growth_rate_lines=growth_rate_lines, + conditional_lines=conditional_lines, + decomposition_lines=decomposition_lines, emission_clipped_lines=emission_clipped_lines, cold_start_lines=cold_start_lines, drift_lines=drift_lines, band_coverage_lines=band_coverage_lines, + slo_lines=slo_lines, + recommended_window_lines=recommended_window_lines, operational_lines=operational_lines, ) diff --git a/forecaster/tests/test_capacity.py b/forecaster/tests/test_capacity.py index 4d53e90..de8ec96 100644 --- a/forecaster/tests/test_capacity.py +++ b/forecaster/tests/test_capacity.py @@ -10,6 +10,7 @@ from promforecast.capacity import ( compute_growth_rates, compute_threshold_etas, + compute_will_breach, ) from promforecast.config import ThresholdConfig @@ -162,3 +163,188 @@ def test_growth_rate_skips_no_windows() -> None: df = _forecast_frame([1.0, 2.0]) rates = compute_growth_rates(forecast=df, windows_seconds=[], units="units_per_second") assert rates == [] + + +def test_will_breach_fires_when_eta_below_window() -> None: + """A finite ETA below ``alert_within`` flips the gauge to 1.""" + # values 0, 50, 100 at t=0,60,120 → threshold 75 crosses at 90s. + df = _forecast_frame([0.0, 50.0, 100.0]) + threshold = ThresholdConfig( + name="warn", + value=75.0, + comparator="gt", + alert_within=timedelta(seconds=120), + severity="critical", + ) + etas = compute_threshold_etas(forecast=df, thresholds=[threshold], levels=[]) + breaches = compute_will_breach(etas=etas, thresholds=[threshold]) + assert len(breaches) == 1 + assert breaches[0].value == 1.0 + assert breaches[0].severity == "critical" + assert breaches[0].level is None + + +def test_will_breach_quiet_when_eta_above_window() -> None: + """A finite ETA above ``alert_within`` emits a 0 (zero-baseline series).""" + df = _forecast_frame([0.0, 50.0, 100.0]) + threshold = ThresholdConfig( + name="warn", + value=75.0, + comparator="gt", + alert_within=timedelta(seconds=30), # crossing at 90s is well above 30s + severity="warning", + ) + etas = compute_threshold_etas(forecast=df, thresholds=[threshold], levels=[]) + breaches = compute_will_breach(etas=etas, thresholds=[threshold]) + assert len(breaches) == 1 + assert breaches[0].value == 0.0 + + +def test_will_breach_emits_zero_when_no_crossing() -> None: + """NaN ETA = no crossing in horizon — emit 0 so the series exists.""" + df = _forecast_frame([10.0, 20.0, 30.0]) + threshold = ThresholdConfig( + name="warn", + value=100.0, + comparator="gt", + alert_within=timedelta(seconds=120), + severity="warning", + ) + etas = compute_threshold_etas(forecast=df, thresholds=[threshold], levels=[]) + breaches = compute_will_breach(etas=etas, thresholds=[threshold]) + assert len(breaches) == 1 + assert breaches[0].value == 0.0 + + +def test_will_breach_skips_thresholds_without_alert_within() -> None: + """Thresholds without ``alert_within`` produce no will-breach samples.""" + df = _forecast_frame([0.0, 50.0, 100.0]) + threshold = ThresholdConfig(name="warn", value=75.0, comparator="gt") + etas = compute_threshold_etas(forecast=df, thresholds=[threshold], levels=[]) + breaches = compute_will_breach(etas=etas, thresholds=[threshold]) + assert breaches == [] + + +def test_drain_eta_returns_inf_on_no_crossing() -> None: + """``le_drain`` returns +Inf — operators alert on ``... == +Inf for 30m``. + + The distinction from ``lt`` matters: ``lt`` returns NaN (dropped at + render time) so an alert like ``time_to_threshold < N`` doesn't + silently match every series; ``le_drain``'s NaN-equivalent has to + round-trip through Prometheus and stay queryable as ``+Inf``. + """ + # Forecast stays well above 0 the whole horizon → no drain crossing. + df = _forecast_frame([10.0, 12.0, 15.0]) + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + levels=[], + ) + assert len(out) == 1 + assert math.isinf(out[0].seconds) + + +def test_drain_eta_emits_finite_when_curve_crosses_from_above() -> None: + """A draining queue → finite ETA on the first crossing-from-above.""" + # 100 → 50 → 0 → -10 at 60s step. Threshold value=0 crossed between + # step 1 (50) and step 2 (0). Linear interp: at exactly the boundary. + df = _forecast_frame([100.0, 50.0, 0.0, -10.0]) + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + levels=[], + ) + assert math.isfinite(out[0].seconds) + assert out[0].seconds > 0 + + +def test_drain_band_uses_yhat_upper() -> None: + """The conservative drain band is ``yhat_upper`` — the *latest* drain. + + Alerts ask "queue still not draining" — the conservative answer is + the latest predicted drain. ``yhat_upper`` is a larger queue depth + that crosses zero later than ``yhat`` does. + """ + # Central drains by step 2; yhat_upper still positive at step 2. + df = _forecast_frame([100.0, 50.0, 0.0, -10.0]) + df["yhat_upper_95"] = [120.0, 80.0, 20.0, -5.0] + df["yhat_lower_95"] = [80.0, 20.0, -10.0, -15.0] + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + levels=[95], + ) + point = next(e for e in out if e.level is None) + band = next(e for e in out if e.level == 95) + # yhat_upper drains later than yhat → band ETA strictly greater. + assert band.seconds > point.seconds + + +def test_drain_eta_zero_when_already_drained() -> None: + """A forecast that starts below the threshold reads as already-drained (ETA=0).""" + df = _forecast_frame([-5.0, -10.0, -15.0]) + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + levels=[], + ) + assert out[0].seconds == 0.0 + + +def test_drain_eta_zero_when_forecast_exactly_at_threshold() -> None: + """A forecast that *equals* the drain value is drained (predicate is ``<=``). + + Regression guard: a queue forecast that touches zero exactly used to + report ``+Inf`` "not draining" because the predicate was strict + ``<``. The fix is to use ``<=`` for ``le_drain`` so the alert + doesn't fire on a healthy zero-queue. + """ + df = _forecast_frame([0.0, 0.0, 0.0]) + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + levels=[], + ) + assert out[0].seconds == 0.0 + assert not math.isinf(out[0].seconds) + + +def test_drain_eta_still_inf_for_lt_predicate_at_threshold() -> None: + """``lt`` keeps strict ``<`` semantics — at-threshold is *not* a crossing. + + Pairs with ``test_drain_eta_zero_when_forecast_exactly_at_threshold`` + to assert the predicate split: ``le_drain`` is ``<=``, ``lt`` stays + ``<``. Without this guard a refactor that unifies the two predicates + would silently change ``lt`` semantics. + """ + df = _forecast_frame([0.0, 0.0, 0.0]) + out = compute_threshold_etas( + forecast=df, + thresholds=[ThresholdConfig(name="floor", value=0.0, comparator="lt")], + levels=[], + ) + assert math.isnan(out[0].seconds) + + +def test_will_breach_emits_per_level_row() -> None: + """Per-level ETAs produce per-level will-breach rows distinguished by level.""" + df = _forecast_frame([10.0, 50.0, 60.0]) + df["yhat_upper_80"] = [20.0, 70.0, 100.0] + df["yhat_lower_80"] = [0.0, 30.0, 20.0] + threshold = ThresholdConfig( + name="warn", + value=55.0, + comparator="gt", + # Point ETA ~90s; band ETA (yhat_upper) ~42s — pick a window that + # excludes the point but includes the band, so the two rows + # disagree and we can prove the level distinction. + alert_within=timedelta(seconds=60), + severity="critical", + ) + etas = compute_threshold_etas(forecast=df, thresholds=[threshold], levels=[80]) + breaches = compute_will_breach(etas=etas, thresholds=[threshold]) + assert len(breaches) == 2 + point = next(b for b in breaches if b.level is None) + band = next(b for b in breaches if b.level == 80) + assert point.value == 0.0 + assert band.value == 1.0 diff --git a/forecaster/tests/test_chart_servicemonitor.py b/forecaster/tests/test_chart_servicemonitor.py index 3b9b890..cbd2daa 100644 --- a/forecaster/tests/test_chart_servicemonitor.py +++ b/forecaster/tests/test_chart_servicemonitor.py @@ -10,7 +10,7 @@ off, no relabeling is rendered. A literal ``true`` / ``false`` pins the behaviour irrespective of the sink. The umbrella chart pre-sets both the sink and ServiceMonitor toggles so a fresh install lands on -sink+VM with the snapshot drop applied (the v1.5 default). +sink+VM with the snapshot drop applied by default. These tests shell out to ``helm template`` because the alternative (implementing a Go template renderer in Python) would diverge from the @@ -265,6 +265,44 @@ def test_umbrella_datasources_default_to_victoriametrics() -> None: assert vm["uid"] == "victoriametrics" +def test_standalone_validate_hook_pod_renders_by_default() -> None: + """The ``helm test`` validate-config Pod renders with the strict flag. + + The hook lets an operator chain ``helm test `` into a + GitOps PostSync gate that rejects configs whose projected + cardinality exceeds ``safety.max_total_series``. CI does not run + ``helm test`` (no kind cluster), but ``helm template`` + + ``kubeconform`` validate that the asset renders to a schema-valid + Pod and carries the expected ``--strict-cardinality`` argument so + the gate is actually wired up. + """ + docs = _helm_template(STANDALONE_CHART) + pod = _find(docs, "Pod", name_contains="test-validate") + assert pod is not None, "standalone chart must render the validate-config test pod by default" + annotations = pod["metadata"]["annotations"] + assert annotations.get("helm.sh/hook") == "test" + # Inspect the container's args so a regression that drops the + # strict flag (turning the hook back into an advisory smoke test) + # surfaces in CI. + containers = pod["spec"]["containers"] + assert containers, "validate-config test pod must declare a container" + args = containers[0].get("args") or [] + assert "validate" in args + assert "--strict-cardinality" in args + assert "--config=/etc/promforecast/config.yaml" in args + + +def test_standalone_validate_hook_can_be_disabled() -> None: + """``tests.validateConfig.enabled=false`` suppresses the hook entirely. + + Useful for managed-cluster environments that forbid Helm hook + Pod creation. + """ + docs = _helm_template(STANDALONE_CHART, "tests.validateConfig.enabled=false") + pod = _find(docs, "Pod", name_contains="test-validate") + assert pod is None + + def test_umbrella_helm_test_pod_renders() -> None: """The labelset-uniqueness ``helm test`` hook is present by default. diff --git a/forecaster/tests/test_config.py b/forecaster/tests/test_config.py index 539d199..1e36733 100644 --- a/forecaster/tests/test_config.py +++ b/forecaster/tests/test_config.py @@ -680,6 +680,102 @@ def test_load_rejects_duplicate_threshold_name(tmp_path: Path) -> None: config_module.load(path) +def test_load_accepts_threshold_with_alert_within_and_severity(tmp_path: Path) -> None: + """A threshold with ``alert_within`` + ``severity`` round-trips through the loader.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: disk_used_pct + promql: up + thresholds: + - name: warn + value: 80 + alert_within: 6h + severity: warning +""", + ) + cfg = config_module.load(path) + threshold = cfg.groups[0].queries[0].thresholds[0] + assert threshold.alert_within == timedelta(hours=6) + assert threshold.severity == "warning" + + +def test_load_rejects_threshold_with_severity_but_no_alert_within(tmp_path: Path) -> None: + """``severity`` without ``alert_within`` is the half-configured-shape error.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + thresholds: + - name: warn + value: 80 + severity: warning +""", + ) + with pytest.raises(ValueError, match="alert_within and severity must be set together"): + config_module.load(path) + + +def test_load_rejects_threshold_with_alert_within_but_no_severity(tmp_path: Path) -> None: + """``alert_within`` without ``severity`` is the inverse half-configured error.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + thresholds: + - name: warn + value: 80 + alert_within: 6h +""", + ) + with pytest.raises(ValueError, match="alert_within and severity must be set together"): + config_module.load(path) + + +def test_load_rejects_zero_alert_within(tmp_path: Path) -> None: + """``alert_within: 0s`` is rejected — every finite ETA satisfies < 0 is False.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + thresholds: + - name: warn + value: 80 + alert_within: 0s + severity: warning +""", + ) + with pytest.raises(ValueError, match="alert_within must be a positive duration"): + config_module.load(path) + + def test_load_rejects_growth_rate_enabled_without_windows(tmp_path: Path) -> None: """``growth_rate.enabled: true`` with empty windows must fail at load time.""" path = _write( @@ -843,3 +939,155 @@ def test_load_accepts_damped_models(tmp_path: Path) -> None: assert "AutoARIMADamped" in names ets = next(s for s in cfg.defaults.models if s.name == "AutoETSDamped") assert ets.params == {"damping_parameter": 0.95} + + +def test_load_rejects_le_drain_with_alert_within(tmp_path: Path) -> None: + """``alert_within`` + ``le_drain`` is rejected — will_breach would invert. + + The will_breach gauge fires when the ETA is small; for a drain threshold + that means "queue will drain soon" — the opposite of a predictive + breach. The combination must fail at load time so operators don't + silently ship the inverted alert. + """ + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: kafka_consumer_lag + promql: up + thresholds: + - name: drained + value: 0 + comparator: le_drain + alert_within: 4h + severity: warning +""", + ) + with pytest.raises(ValueError, match="alert_within \\+ severity are not supported on le_drain"): + config_module.load(path) + + +def test_load_accepts_slo_block(tmp_path: Path) -> None: + """A query with ``slo:`` and multiple windows loads cleanly.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: http_error_ratio + promql: up + slo: + objective: 99.9 + windows: [5m, 1h, 30d] +""", + ) + cfg = config_module.load(path) + slo = cfg.groups[0].queries[0].slo + assert slo is not None + assert slo.objective == 99.9 + assert slo.windows == [timedelta(minutes=5), timedelta(hours=1), timedelta(days=30)] + + +def test_load_rejects_slo_objective_at_bounds(tmp_path: Path) -> None: + """``objective`` must be strictly inside ``(0, 100)`` — pydantic enforces.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + slo: + objective: 100 +""", + ) + with pytest.raises(ValueError, match="objective"): + config_module.load(path) + + +def test_load_rejects_slo_empty_windows(tmp_path: Path) -> None: + """An explicitly empty ``windows: []`` must fail (default is non-empty).""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + slo: + objective: 99.9 + windows: [] +""", + ) + with pytest.raises(ValueError, match="at least one window is required"): + config_module.load(path) + + +def test_load_accepts_recommended_windows_block(tmp_path: Path) -> None: + """A query with ``recommended_windows.enabled: true`` round-trips cleanly.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: node_cpu_busy_pct + promql: up + recommended_windows: + enabled: true + duration: 15m + lookahead: 24h + count: 5 +""", + ) + cfg = config_module.load(path) + rw = cfg.groups[0].queries[0].recommended_windows + assert rw.enabled is True + assert rw.duration == timedelta(minutes=15) + assert rw.lookahead == timedelta(hours=24) + assert rw.count == 5 + + +def test_load_rejects_recommended_windows_duration_larger_than_lookahead( + tmp_path: Path, +) -> None: + """A window longer than the lookahead can never be recommended — reject.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +groups: + - name: g + queries: + - id: m + promql: up + recommended_windows: + enabled: true + duration: 2h + lookahead: 1h +""", + ) + with pytest.raises(ValueError, match="must be <= lookahead"): + config_module.load(path) diff --git a/forecaster/tests/test_decomposition.py b/forecaster/tests/test_decomposition.py new file mode 100644 index 0000000..ea3935e --- /dev/null +++ b/forecaster/tests/test_decomposition.py @@ -0,0 +1,166 @@ +"""Tests for the forecast component-decomposition primitive.""" + +from __future__ import annotations + +import math +from datetime import UTC, datetime, timedelta + +import pandas as pd +import pytest + +from promforecast.decomposition import ( + COMPONENT_NAMES, + NON_DECOMPOSABLE_MODELS, + DecompositionError, + decompose, + is_decomposable, + reconstruct, +) + + +def _forecast_frame(values: list[float], *, step_seconds: int = 60) -> pd.DataFrame: + base = datetime(2026, 5, 21, tzinfo=UTC) + timestamps = [base + timedelta(seconds=i * step_seconds) for i in range(len(values))] + return pd.DataFrame({"unique_id": ["s"] * len(values), "ds": timestamps, "yhat": values}) + + +def test_decompose_returns_all_four_components() -> None: + """Every emission produces the configured four component lists.""" + df = _forecast_frame([10.0, 12.0, 14.0, 16.0]) + result = decompose(forecast=df, in_sample_values=[8.0, 9.0, 10.0], season_length=2) + assert set(result.components.keys()) == set(COMPONENT_NAMES) + for values in result.components.values(): + assert len(values) == len(df) + + +def test_decompose_sums_back_to_yhat() -> None: + """The contract: trend + seasonal + level + residual ≈ yhat row by row. + + A failure here would silently bias dashboards that stack components. + """ + df = _forecast_frame([10.0, 12.0, 14.0, 13.0, 15.0, 17.0, 16.0, 18.0]) + result = decompose( + forecast=df, + in_sample_values=[5.0, 7.0, 9.0, 11.0], + season_length=4, + ) + summed = reconstruct(result.components) + yhat = df["yhat"].tolist() + assert len(summed) == len(yhat) + for s, y in zip(summed, yhat, strict=True): + assert math.isclose(s, y, abs_tol=1e-9) + + +def test_decompose_zero_season_length_collapses_seasonal_to_zero() -> None: + """``season_length=0`` is a valid degenerate case (no cycle to fold).""" + df = _forecast_frame([1.0, 2.0, 3.0]) + result = decompose(forecast=df, in_sample_values=[1.0, 2.0], season_length=0) + assert all(v == 0.0 for v in result.components["seasonal"]) + # Even with no seasonal split, the sum-back invariant holds. + summed = reconstruct(result.components) + for s, y in zip(summed, df["yhat"].tolist(), strict=True): + assert math.isclose(s, y, abs_tol=1e-9) + + +def test_decompose_rejects_empty_frame() -> None: + """An empty forecast frame raises with ``components_unavailable``.""" + empty = pd.DataFrame({"unique_id": [], "ds": [], "yhat": []}) + with pytest.raises(DecompositionError) as exc: + decompose(forecast=empty, in_sample_values=[1.0], season_length=2) + assert exc.value.reason == "components_unavailable" + + +def test_decompose_rejects_frame_missing_yhat() -> None: + """A frame without ``yhat`` is unprocessable — that's the central output.""" + df = pd.DataFrame({"unique_id": ["s"], "ds": [datetime(2026, 5, 21, tzinfo=UTC)]}) + with pytest.raises(DecompositionError) as exc: + decompose(forecast=df, in_sample_values=[1.0], season_length=2) + assert exc.value.reason == "components_unavailable" + + +def test_is_decomposable_skips_intermittent_models() -> None: + """Intermittent-demand model families don't admit a trend/season split.""" + assert "Croston" in NON_DECOMPOSABLE_MODELS + assert not is_decomposable("Croston") + assert is_decomposable("AutoARIMA") + + +def test_decompose_raises_invariant_violation_when_summing_off() -> None: + """The sum-to-yhat invariant must be enforced at fit time, not only in tests. + + A future change to the math (e.g. dropping the residual term) would + silently bias the stacked-area panel; the guard turns that into a + skip with ``reason="invariant_violation"`` so the operator sees it. + """ + from promforecast.decomposition import _enforce_sum_invariant # noqa: PLC0415 + + with pytest.raises(DecompositionError) as exc: + _enforce_sum_invariant( + yhat=[10.0, 20.0], + components={ + "trend": [0.0, 0.0], + "seasonal": [0.0, 0.0], + "level": [5.0, 5.0], # sums to 5 vs yhat 10 — violation. + "residual": [0.0, 0.0], + }, + ) + assert exc.value.reason == "invariant_violation" + + +def test_decompose_sum_invariant_tolerates_large_magnitude_noise() -> None: + """The invariant must use a relative tolerance for large-magnitude metrics. + + A pure absolute floor of 1e-6 would falsely flag harmless float + accumulation on, e.g., disk-bytes forecasts (~10^12). The + component sum should clear the check at any magnitude where the + drift is float-noise relative to ``|yhat|``, not raise + ``invariant_violation``. + """ + from promforecast.decomposition import _enforce_sum_invariant # noqa: PLC0415 + + yhat = 1.0e12 + # A drift of 1e-3 absolute is float noise next to 10^12 (ratio + # 1e-15) and must pass; the old absolute-only check would have + # rejected it. + _enforce_sum_invariant( + yhat=[yhat], + components={ + "trend": [0.0], + "seasonal": [0.0], + "level": [yhat], + "residual": [-1.0e-3], # not zero — exercises the tolerance + }, + ) + + +def test_decompose_sum_invariant_still_catches_real_drift() -> None: + """Relative tolerance still rejects drift that isn't float noise. + + A 10% drift on a 100-unit yhat row is a logic bug, not float + accumulation; the check must still raise so the runner emits the + skip counter rather than ship a misleading panel. + """ + from promforecast.decomposition import _enforce_sum_invariant # noqa: PLC0415 + + with pytest.raises(DecompositionError) as exc: + _enforce_sum_invariant( + yhat=[100.0], + components={ + "trend": [0.0], + "seasonal": [0.0], + "level": [90.0], + "residual": [0.0], + }, + ) + assert exc.value.reason == "invariant_violation" + + +def test_decompose_trend_captures_slope() -> None: + """A pure linear ramp should land its slope on the ``trend`` component.""" + # values rise by 5/step from 0; trend slope = 5 + df = _forecast_frame([0.0, 5.0, 10.0, 15.0]) + result = decompose(forecast=df, in_sample_values=[0.0], season_length=1) + trend = result.components["trend"] + # Step 0 is the anchor (0), step k is k * slope. + assert math.isclose(trend[0], 0.0, abs_tol=1e-9) + assert math.isclose(trend[3] - trend[0], 15.0, abs_tol=1e-9) diff --git a/forecaster/tests/test_example_alerts_labelset.py b/forecaster/tests/test_example_alerts_labelset.py index e4655be..d069765 100644 --- a/forecaster/tests/test_example_alerts_labelset.py +++ b/forecaster/tests/test_example_alerts_labelset.py @@ -1,5 +1,5 @@ """Audit ``examples/alerts/promforecast-rules.yaml`` against the sink+VM -labelset that the v1.5 defaults emit. +labelset that the sink-first defaults emit. The forecaster ships two write paths for forecast metrics: the ``/metrics`` snapshot scraped by Prometheus (carries the forecaster's own @@ -223,6 +223,75 @@ def _expand_levels(base: list[dict[str, str]], levels: tuple[str, ...]) -> list[ ) for w in ("1d", "7d") ], + "node_filesystem_avail_bytes_forecast_will_breach": [ + _with( + _SNAPSHOT_OP_LABELS, + id="node_filesystem_avail_bytes", + group="node_storage", + model="AutoARIMA", + instance="node-exporter:9100", + mountpoint="/var", + name=name, + severity=sev, + level=lvl, + ) + for name in THRESHOLD_NAMES + for sev in ("warning", "critical") + for lvl in LEVELS + ], + "kafka_consumer_lag_forecast_time_to_drain_seconds": [ + _with( + _SNAPSHOT_OP_LABELS, + id="kafka_consumer_lag", + group="kafka", + model="AutoARIMA", + topic="orders", + name="drained", + ), + ], + "http_request_error_ratio_forecast_error_budget_exhausted_in_seconds": [ + _with( + _SNAPSHOT_OP_LABELS, + id="http_request_error_ratio", + group="http_slo", + model="AutoARIMA", + service="checkout", + window="30d", + ), + ], + "http_request_error_ratio_forecast_error_budget_burn_rate": [ + _with( + _SNAPSHOT_OP_LABELS, + id="http_request_error_ratio", + group="http_slo", + model="AutoARIMA", + service="checkout", + window="30d", + ), + ], + "http_request_error_ratio_forecast_error_budget_remaining_seconds": [ + _with( + _SNAPSHOT_OP_LABELS, + id="http_request_error_ratio", + group="http_slo", + model="AutoARIMA", + service="checkout", + window="30d", + ), + ], + "forecast_recommended_maintenance_window": [ + _with( + _SNAPSHOT_OP_LABELS, + id="node_cpu_busy_pct", + group="node_capacity", + model="AutoARIMA", + instance="node-exporter:9100", + rank=str(r), + duration_seconds="1800", + start_timestamp_seconds="1700000000", + ) + for r in (1, 2, 3) + ], } # --------------------------------------------------------------------------- @@ -320,7 +389,7 @@ def _labelset_matches(labels: dict[str, str], matchers: list[tuple[str, str, str def test_rule_metric_names_avoid_reserved_colons(group: str, alert: str, expr: str) -> None: """Forecaster output metric names must never use colons. - Design principle 6 in ``CLAUDE.md`` reserves ``:`` for Prometheus + The project's naming convention reserves ``:`` for Prometheus recording rules — exporters and direct instrumentation never use it. ``exporter.py`` rejects colons at runtime, but bundled alerts could drift independently (a recording-rule expression copy-pasted here @@ -331,7 +400,7 @@ def test_rule_metric_names_avoid_reserved_colons(group: str, alert: str, expr: s raise AssertionError( f"{group}/{alert}: rule references metric {metric!r} which uses the " "reserved ``:`` character. Forecaster output metrics never use colons " - "(see CLAUDE.md design principle 6). Did a recording-rule expression " + "(forecaster naming convention). Did a recording-rule expression " "slip into the bundled alerts?" ) diff --git a/forecaster/tests/test_maintenance.py b/forecaster/tests/test_maintenance.py new file mode 100644 index 0000000..062e979 --- /dev/null +++ b/forecaster/tests/test_maintenance.py @@ -0,0 +1,130 @@ +"""Tests for the maintenance-window recommender.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pandas as pd + +from promforecast.config import RecommendedWindowsConfig +from promforecast.maintenance import recommend_windows + + +def _forecast_frame(values: list[float], *, step_seconds: int = 300) -> pd.DataFrame: + """Forecast frame with monotone timestamps at the configured step.""" + base = datetime(2026, 5, 21, 12, 0, tzinfo=UTC) + timestamps = [base + timedelta(seconds=i * step_seconds) for i in range(len(values))] + return pd.DataFrame({"unique_id": ["s"] * len(values), "ds": timestamps, "yhat": values}) + + +def test_returns_lowest_load_window_first() -> None: + """Rank 1 corresponds to the lowest-mean-load window in the lookahead.""" + # 5 steps of 5m each → 25 minutes of horizon. + # Values: [10, 5, 1, 1, 10] — the cheapest two-step window starts at index 2. + df = _forecast_frame([10.0, 5.0, 1.0, 1.0, 10.0]) + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=10), # 2 steps at 5m + lookahead=timedelta(hours=1), + count=1, + ) + out = recommend_windows(forecast=df, config=config) + assert len(out) == 1 + assert out[0].rank == 1 + assert out[0].predicted_load == 1.0 + + +def test_returns_top_n_windows_in_ascending_load_order() -> None: + """``count=3`` returns the three lowest-load windows, ranked ascending.""" + df = _forecast_frame([10.0, 5.0, 1.0, 1.0, 10.0, 8.0, 2.0]) + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=5), # 1 step + lookahead=timedelta(hours=1), + count=3, + ) + out = recommend_windows(forecast=df, config=config) + assert len(out) == 3 + assert [w.rank for w in out] == [1, 2, 3] + assert [w.predicted_load for w in out] == sorted(w.predicted_load for w in out) + + +def test_window_overflowing_horizon_is_skipped() -> None: + """A window that runs past the forecast's last sample isn't recommended. + + Operators read the panel as "during this period, load will be N"; + half-overlapping windows would advertise a load they can't see the + full curve of. + """ + df = _forecast_frame([1.0, 1.0, 1.0]) # 15 minutes of horizon + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(hours=1), # 60m duration > 15m horizon + lookahead=timedelta(hours=2), + count=3, + ) + out = recommend_windows(forecast=df, config=config) + assert out == [] + + +def test_ties_break_by_start_timestamp() -> None: + """Equal-load windows rank by earliest start so successive runs are stable.""" + df = _forecast_frame([1.0, 1.0, 1.0, 1.0, 1.0]) + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=5), + lookahead=timedelta(hours=1), + count=3, + ) + out = recommend_windows(forecast=df, config=config) + assert len(out) == 3 + timestamps = [w.start_timestamp for w in out] + assert timestamps == sorted(timestamps) + + +def test_lookahead_limits_candidate_window_set() -> None: + """Anchors past the lookahead don't enter the candidate pool.""" + df = _forecast_frame([10.0, 10.0, 10.0, 1.0, 1.0, 1.0]) + # Lookahead is 10m → only the first three anchors (t=0, t=5m, t=10m) qualify + # — all of which have load 10. The cheap (load=1) windows past the + # lookahead must not be picked. + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=5), + lookahead=timedelta(minutes=10), + count=1, + ) + out = recommend_windows(forecast=df, config=config) + assert len(out) == 1 + assert out[0].predicted_load == 10.0 + + +def test_empty_forecast_yields_empty() -> None: + df = pd.DataFrame({"ds": [], "yhat": []}) + config = RecommendedWindowsConfig(enabled=True) + assert recommend_windows(forecast=df, config=config) == [] + + +def test_disabled_count_zero_emits_nothing() -> None: + """``count=0`` is rejected by pydantic; verify the runtime also short-circuits.""" + # pydantic enforces count >= 1, so we exercise the runtime guard directly + # against a hand-constructed config. + df = _forecast_frame([1.0, 1.0]) + # Set count via internal field manipulation since pydantic refuses 0 + config = RecommendedWindowsConfig(enabled=True, duration=timedelta(minutes=5)) + object.__setattr__(config, "count", 0) + assert recommend_windows(forecast=df, config=config) == [] + + +def test_start_timestamp_anchored_to_forecast_first_row() -> None: + """The reported ``start_timestamp`` is the Unix seconds of the window's first row.""" + df = _forecast_frame([1.0, 1.0, 1.0, 1.0]) + config = RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=5), + lookahead=timedelta(hours=1), + count=1, + ) + out = recommend_windows(forecast=df, config=config) + expected_first_row_unix = df["ds"].iloc[0].timestamp() + assert out[0].start_timestamp == expected_first_row_unix diff --git a/forecaster/tests/test_models_registry.py b/forecaster/tests/test_models_registry.py index 9303e8c..682089c 100644 --- a/forecaster/tests/test_models_registry.py +++ b/forecaster/tests/test_models_registry.py @@ -108,6 +108,42 @@ def test_unknown_model_error_lists_alternatives() -> None: model_registry.build("AutoArimaa", season_length=288) +@pytest.mark.parametrize( + "name", + [ + "AutoARIMA", + "AutoARIMADamped", + "AutoETS", + "AutoETSDamped", + "MSTL", + "SeasonalNaive", + "Croston", + "CrostonSBA", + "ADIDA", + "IMAPA", + ], +) +def test_builtin_models_advertise_uses_regressors_false(name: str) -> None: + """Every built-in wrapper must advertise ``uses_regressors = False``. + + The conditional-forecast emission path reads this flag to decide + whether to refit the model with held-constant regressors. A built-in + that silently flipped to True would double the per-series fit cost + for every operator who enables ``emission.conditional``. + """ + assert model_registry.uses_regressors(name) is False + + +def test_uses_regressors_returns_true_for_unknown_model() -> None: + """Unknown models must be treated as regressor-aware. + + The conditional path uses ``uses_regressors`` to *skip* refits; + silently dropping refits for an unrecognised model would change + correctness, so the default for unknown names is the safe one. + """ + assert model_registry.uses_regressors("__definitely_not_a_real_model__") is True + + class _FakeExternalModel: """Stand-in for an external plug-in registered via entry points. diff --git a/forecaster/tests/test_runner_capacity.py b/forecaster/tests/test_runner_capacity.py index 7a72c0a..e08c01e 100644 --- a/forecaster/tests/test_runner_capacity.py +++ b/forecaster/tests/test_runner_capacity.py @@ -19,10 +19,13 @@ DefaultsConfig, GroupConfig, GrowthRateEmissionConfig, + ModelSpec, QueryConfig, QueryEmissionConfig, + RecommendedWindowsConfig, SafetyConfig, ServerConfig, + SLOConfig, ThresholdConfig, ) from promforecast.exporter import Exporter @@ -210,6 +213,96 @@ async def test_runner_eta_band_reflects_conformal_calibration( assert cal_band.seconds < raw_band.seconds +@pytest.mark.asyncio +async def test_runner_emits_will_breach_family(monkeypatch: pytest.MonkeyPatch) -> None: + """A threshold with ``alert_within`` + ``severity`` emits the boolean. + + The RampForecastModel's central forecast crosses 60 mid-horizon and 70 + near the end. We pick ``alert_within`` so the warn threshold fires + (1.0) and critical does not (0.0). Per-level rows are produced for + each emitted band. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="disk_used_pct", + promql="up", + thresholds=[ + ThresholdConfig( + name="warn", + value=60.0, + comparator="gt", + alert_within=timedelta(hours=2), + severity="warning", + ), + ThresholdConfig( + name="critical", + value=70.0, + comparator="gt", + alert_within=timedelta(minutes=1), + severity="critical", + ), + ], + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "disk_used_pct_forecast_will_breach" in body + assert 'severity="warning"' in body + assert 'severity="critical"' in body + # warn alert_within=2h easily contains the ~11min ETA → 1. + warn_lines = [ + line + for line in body.splitlines() + if line.startswith("disk_used_pct_forecast_will_breach{") + and 'name="warn"' in line + and 'level="80"' not in line + ] + assert warn_lines and warn_lines[0].endswith(" 1.0") + # critical alert_within=1min cannot contain the ~21min ETA → 0. + crit_lines = [ + line + for line in body.splitlines() + if line.startswith("disk_used_pct_forecast_will_breach{") + and 'name="critical"' in line + and 'level="80"' not in line + ] + assert crit_lines and crit_lines[0].endswith(" 0.0") + + +@pytest.mark.asyncio +async def test_runner_omits_will_breach_when_threshold_opts_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A threshold without ``alert_within`` does not emit the boolean.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="quiet_metric", + promql="up", + thresholds=[ThresholdConfig(name="warn", value=60.0, comparator="gt")], + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + body = exporter.render().decode() + # ETA family exists, will_breach does not — the operator opted into + # one and not the other. + assert "quiet_metric_forecast_time_to_threshold_seconds" in body + assert "quiet_metric_forecast_will_breach" not in body + + @pytest.mark.asyncio async def test_runner_omits_capacity_families_when_unconfigured( monkeypatch: pytest.MonkeyPatch, @@ -228,3 +321,271 @@ async def test_runner_omits_capacity_families_when_unconfigured( body = exporter.render().decode() assert "quiet_forecast_time_to_threshold_seconds" not in body assert "quiet_forecast_growth_rate" not in body + assert "quiet_forecast_time_to_drain_seconds" not in body + assert "quiet_forecast_error_budget" not in body + assert "forecast_recommended_maintenance_window" not in body + + +class _DrainForecastModel: + """Emits a strictly-decreasing forecast so a drain ETA is well-defined.""" + + def __init__(self, season_length: int = 288) -> None: + self.name = "DrainModel" + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + idx = pd.date_range(df["ds"].iloc[-1], periods=horizon + 1, freq=freq)[1:] + # Start at 100, drop by 5/step so a 24-step horizon crosses 0 around + # step 20 — well within the horizon. + values = [100.0 - 5.0 * i for i in range(horizon)] + out = pd.DataFrame({"unique_id": ["s"] * horizon, "ds": idx, "yhat": values}) + for level in levels: + out[f"yhat_lower_{level}"] = [v - 5.0 for v in values] + out[f"yhat_upper_{level}"] = [v + 5.0 for v in values] + return out + + +@pytest.mark.asyncio +async def test_runner_emits_drain_eta_family(monkeypatch: pytest.MonkeyPatch) -> None: + """``le_drain`` thresholds materialise the drain-ETA family on /metrics.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _DrainForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="kafka_consumer_lag", + promql="up", + thresholds=[ + ThresholdConfig(name="drained", value=0.0, comparator="le_drain"), + ], + ) + ) + cfg.defaults.models = [ModelSpec(name="DrainModel")] + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "kafka_consumer_lag_forecast_time_to_drain_seconds" in body + # The standard ETA metric must NOT appear — le_drain is routed to the + # drain-only family. A regression that emits both would create a + # confusing parallel ``..._time_to_threshold_seconds`` sample. + assert "kafka_consumer_lag_forecast_time_to_threshold_seconds" not in body + + +@pytest.mark.asyncio +async def test_runner_drain_eta_emits_plus_inf_when_not_draining( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A queue that never crosses 0 must emit ``+Inf`` (so the alert can match).""" + from promforecast import models as model_registry # noqa: PLC0415 + + class _RisingModel: + name = "RisingModel" + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + idx = pd.date_range(df["ds"].iloc[-1], periods=horizon + 1, freq=freq)[1:] + values = [100.0 + i for i in range(horizon)] + out = pd.DataFrame({"unique_id": ["s"] * horizon, "ds": idx, "yhat": values}) + for level in levels: + out[f"yhat_lower_{level}"] = [v - 5.0 for v in values] + out[f"yhat_upper_{level}"] = [v + 5.0 for v in values] + return out + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RisingModel()) + + cfg = _config_with_query( + QueryConfig( + id="growing_queue", + promql="up", + thresholds=[ThresholdConfig(name="drained", value=0.0, comparator="le_drain")], + ) + ) + cfg.defaults.models = [ModelSpec(name="RisingModel")] + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # The exporter renders +Inf as "+Inf" in Prometheus exposition format. + inf_lines = [ + line + for line in body.splitlines() + if line.startswith("growing_queue_forecast_time_to_drain_seconds{") + and 'level="80"' not in line + ] + assert inf_lines + assert "+Inf" in inf_lines[0] + + +@pytest.mark.asyncio +async def test_runner_emits_slo_error_budget_family( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A query with ``slo:`` emits the three error-budget metric families.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr( + model_registry, + "build", + lambda *a, **kw: _RampForecastModel(start=0.001, per_step=0.0), + ) + + cfg = _config_with_query( + QueryConfig( + id="http_error_ratio", + promql="up", + slo=SLOConfig(objective=99.9, windows=[timedelta(days=30)]), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "http_error_ratio_forecast_error_budget_remaining_seconds" in body + assert "http_error_ratio_forecast_error_budget_burn_rate" in body + assert "http_error_ratio_forecast_error_budget_exhausted_in_seconds" in body + assert 'window="30d"' in body + + +@pytest.mark.asyncio +async def test_runner_inherits_new_emission_families_on_skipped_query( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Per-query refresh: SLO + drain + recommended-window rows for a skipped + query must survive the next partial tick, mirroring the behaviour of + the existing forecast / capacity families. + + Without inheritance plumbing the snapshot would drop every other + query's last-known SLO / drain / recommendation rows the next time + ``run_group`` runs a subset of the group's queries. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr( + model_registry, + "build", + lambda *a, **kw: _RampForecastModel(start=0.001, per_step=0.0), + ) + + cfg = Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(), + defaults=DefaultsConfig( + min_points=5, + models=[ModelSpec(name="RampModel")], + confidence_levels=[80], + horizon=timedelta(hours=2), + step=timedelta(minutes=5), + accuracy=AccuracyConfig(evaluate=False), + ), + groups=[ + GroupConfig( + name="g", + queries=[ + QueryConfig( + id="active_query", + promql="qa", + ), + QueryConfig( + id="skipped_query", + promql="qb", + slo=SLOConfig(objective=99.9, windows=[timedelta(days=30)]), + recommended_windows=RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=10), + lookahead=timedelta(hours=1), + count=2, + ), + thresholds=[ + ThresholdConfig( + name="drained", + value=-1.0, + comparator="le_drain", + ), + ], + ), + ], + ) + ], + ) + source = _FakeSource({"qa": [_frame(20)], "qb": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + initial = exporter.render().decode() + # On the initial full run all three new families must be present + # for the opted-in query. + assert "skipped_query_forecast_error_budget_remaining_seconds" in initial + assert "skipped_query_forecast_time_to_drain_seconds" in initial + assert 'id="skipped_query"' in initial + + # Partial run touches only ``active_query``. The skipped query's + # SLO / drain / recommendation rows must survive. + await runner.run_group(cfg.groups[0], queries=[cfg.groups[0].queries[0]]) + partial = exporter.render().decode() + assert "skipped_query_forecast_error_budget_remaining_seconds" in partial + assert "skipped_query_forecast_time_to_drain_seconds" in partial + # Recommended windows are a global metric family — verify the + # ``id="skipped_query"`` row survives. + rec_lines = [ + line + for line in partial.splitlines() + if line.startswith("forecast_recommended_maintenance_window{") + and 'id="skipped_query"' in line + ] + assert rec_lines + + +@pytest.mark.asyncio +async def test_runner_emits_recommended_maintenance_window_family( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A query with ``recommended_windows.enabled`` emits the global family.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="cpu_busy", + promql="up", + recommended_windows=RecommendedWindowsConfig( + enabled=True, + duration=timedelta(minutes=10), + lookahead=timedelta(hours=1), + count=2, + ), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "forecast_recommended_maintenance_window" in body + # The id is a label here, not in the metric name (unlike the + # capacity ETA families). + assert 'id="cpu_busy"' in body + assert 'rank="1"' in body + assert 'rank="2"' in body diff --git a/forecaster/tests/test_runner_conditional_decompose.py b/forecaster/tests/test_runner_conditional_decompose.py new file mode 100644 index 0000000..22096ab --- /dev/null +++ b/forecaster/tests/test_runner_conditional_decompose.py @@ -0,0 +1,770 @@ +"""Runner-level integration tests for the conditional + decomposition families.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pandas as pd +import pytest + +from promforecast.config import ( + AccuracyConfig, + Config, + DatasourceConfig, + DefaultsConfig, + GroupConfig, + QueryConfig, + QueryEmissionConfig, + SafetyConfig, + ServerConfig, +) +from promforecast.exporter import Exporter +from promforecast.runner import Runner +from promforecast.source import PromSource, SeriesFrame + + +class _FakeSource(PromSource): + def __init__(self, by_query: dict[str, list[SeriesFrame]]) -> None: + self._by_query = by_query + + async def query_range( + self, + promql: str, + start: datetime, + end: datetime, + step_seconds: int, + ) -> list[SeriesFrame]: + return self._by_query.get(promql, []) + + async def aclose(self) -> None: + return None + + +class _RampForecastModel: + """Stable ramp model; ignores regressors so conditional == unconditional.""" + + def __init__( + self, + season_length: int = 288, + *, + start: float = 50.0, + per_step: float = 1.0, + ) -> None: + self.name = "RampModel" + self._start = start + self._per_step = per_step + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + idx = pd.date_range(df["ds"].iloc[-1], periods=horizon + 1, freq=freq)[1:] + values = [self._start + i * self._per_step for i in range(horizon)] + out = pd.DataFrame({"unique_id": ["s"] * horizon, "ds": idx, "yhat": values}) + for level in levels: + out[f"yhat_lower_{level}"] = [v - 1.0 for v in values] + out[f"yhat_upper_{level}"] = [v + 1.0 for v in values] + return out + + +def _frame(n: int) -> SeriesFrame: + base = datetime(2026, 5, 21, tzinfo=UTC) + timestamps = [base + timedelta(minutes=5 * i) for i in range(n)] + return SeriesFrame(labels={"instance": "host-a"}, timestamps=timestamps, values=[1.0] * n) + + +def _config_with_query(query: QueryConfig) -> Config: + return Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(), + defaults=DefaultsConfig( + min_points=5, + models=["RampModel"], + confidence_levels=[80], + horizon=timedelta(hours=2), + step=timedelta(minutes=5), + accuracy=AccuracyConfig(evaluate=False), + ), + groups=[GroupConfig(name="g", queries=[query])], + ) + + +@pytest.mark.asyncio +async def test_conditional_emission_off_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + """No conditional metrics until ``emission.conditional`` is flipped on.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query(QueryConfig(id="m", promql="up")) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "m_forecast_conditional" not in body + assert "forecast_conditional_deviation_outside_band" not in body + + +@pytest.mark.asyncio +async def test_conditional_emission_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + """Enabling the toggle adds the parallel conditional family + deviation.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="http_requests_rate", + promql="up", + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # Conditional snapshot family is emitted (one row per (model, level) for + # upper / lower bands plus the central point). + assert "http_requests_rate_forecast_conditional" in body + assert "http_requests_rate_forecast_conditional_lower" in body + assert "http_requests_rate_forecast_conditional_upper" in body + # Conditional deviation operates against the trailing actual; the + # outside-band gauge is emitted regardless of the ratio's finiteness. + assert "forecast_conditional_deviation_outside_band" in body + + +@pytest.mark.asyncio +async def test_conditional_emission_applies_count_safeguards( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count-aware clipping must apply to the conditional family too. + + A query with ``data_profile: count`` would otherwise emit clipped + unconditional values alongside un-clipped conditional values — the + same operator alert evaluated against both would behave inconsistently. + The model below emits negative band values; with safeguards on, the + conditional output should never go below zero. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + class _NegativeBandModel(_RampForecastModel): + name = "RampModel" + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + idx = pd.date_range(df["ds"].iloc[-1], periods=horizon + 1, freq=freq)[1:] + values = [0.5 + i * 0.1 for i in range(horizon)] + out = pd.DataFrame({"unique_id": ["s"] * horizon, "ds": idx, "yhat": values}) + for level in levels: + # Band is wide enough that the lower edge goes negative. + out[f"yhat_lower_{level}"] = [v - 5.0 for v in values] + out[f"yhat_upper_{level}"] = [v + 5.0 for v in values] + return out + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _NegativeBandModel()) + + cfg = _config_with_query( + QueryConfig( + id="count_metric", + promql="up", + data_profile="count", + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # No negative conditional band value should make it out — the + # count-aware clip must apply. + negative_lower = [ + line + for line in body.splitlines() + if line.startswith("count_metric_forecast_conditional_lower{") + and line.rsplit(" ", 1)[-1].startswith("-") + ] + assert negative_lower == [], f"unexpected negative conditional bands: {negative_lower}" + + +@pytest.mark.asyncio +async def test_conditional_fit_failure_bumps_failures_counter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failing conditional fit must surface via ``forecast_failures_total``. + + The unconditional fit succeeds (so the forecast emits and the + snapshot has data), and the conditional re-fit then raises — the + operator should see ``reason="conditional_fit_error"`` rise without + needing to scrape logs. + + Needs the refit path to be exercised: that requires (a) regressors + actually configured on the query so the held-constant frame is + non-empty *and* (b) ``model_registry.uses_regressors`` returning + True so the runner doesn't short-circuit to the alias path. We + patch the registry helper for clarity rather than relying on the + unknown-model default. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.config import RegressorConfig # noqa: PLC0415 + + call_count = {"n": 0} + + class _ConditionalFails(_RampForecastModel): + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + call_count["n"] += 1 + # First call: succeed (the unconditional pass). + # Subsequent: fail (the conditional re-fit). + if call_count["n"] == 1: + return super().fit_predict(df, horizon, freq, levels, regressors) + raise RuntimeError("conditional fit blew up") + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _ConditionalFails()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # Counter line appears with reason="conditional_fit_error". + failure_lines = [ + line + for line in body.splitlines() + if "forecast_failures_total" in line + and 'reason="conditional_fit_error"' in line + and line.rsplit(" ", 1)[-1].strip() != "0.0" + ] + assert failure_lines, f"conditional_fit_error counter never bumped; body=\n{body[:500]}" + + +@pytest.mark.asyncio +async def test_conditional_short_circuits_refit_for_non_regressor_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A model that ignores regressors must not be refit for conditional emission. + + The built-in models all advertise ``uses_regressors = False``. When + ``emission.conditional: true`` is on but the configured model is in + that set, the conditional family should emit at zero extra fit cost + — the runner aliases the unconditional forecast frame instead of + refitting. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + call_count = {"n": 0} + + class _CountingRampModel(_RampForecastModel): + # Inherits ``uses_regressors`` from _RampForecastModel below; + # set it explicitly here so the test fails loudly if the + # attribute disappears from the protocol. + uses_regressors = False + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + call_count["n"] += 1 + return super().fit_predict(df, horizon, freq, levels, regressors) + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _CountingRampModel()) + # The runner reads the class-level marker via the registry's + # ``uses_regressors`` helper, which looks the class up by name. + # Patch the helper directly so the test doesn't depend on a registered + # ``RampModel`` entry point. + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: False) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + # Exactly one fit: the unconditional pass. No second fit was issued + # for the conditional emission because the model advertises + # uses_regressors=False, and the runner aliased the existing + # forecast frame. + assert call_count["n"] == 1 + # Conditional family still emitted on /metrics — the short-circuit + # must not skip the *emission*, only the redundant refit. + body = exporter.render().decode() + assert "m_forecast_conditional" in body + + +@pytest.mark.asyncio +async def test_conditional_aliased_counter_bumps_on_short_circuit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``forecast_conditional_aliased_total`` must tick every time the + runner skips the refit and aliases the unconditional frame. Without + this counter, operators have no way to confirm the short-circuit + is firing in their deployment — a regression that silently disables + it would only show up as a CPU regression, not an alertable signal. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: False) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # The counter family is labelled with (group, query, model). The + # counter must be > 0 because the unconditional fit's emitted model + # (``RampModel``) advertises no regressor use, so the conditional + # pass aliased. + aliased_lines = [ + line + for line in body.splitlines() + if line.startswith("forecast_conditional_aliased_total{") + and 'group="g"' in line + and 'query="m"' in line + and 'model="RampModel"' in line + and line.rsplit(" ", 1)[-1].strip() != "0.0" + ] + assert aliased_lines, ( + "expected a forecast_conditional_aliased_total > 0 row for " + f"(g, m, RampModel); body:\n{body[:600]}" + ) + + +@pytest.mark.asyncio +async def test_conditional_aliased_counter_quiet_when_refitting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Conversely: when the runner *did* refit, the alias counter must NOT + show a positive sample — otherwise an operator alert on + ``rate(forecast_conditional_aliased_total[5m]) == 0`` (the "is the + short-circuit firing" check) would silently always pass and miss the + case where every conditional emission is paying the refit cost. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.config import RegressorConfig # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # When no alias happens the counter family must stay quiet — the + # exporter only emits ``forecast_conditional_aliased_total`` after + # at least one alias has occurred. + assert "forecast_conditional_aliased_total" not in body + + +@pytest.mark.asyncio +async def test_conditional_short_circuits_when_model_ignores_configured_regressors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The `uses_regressors=False` branch must fire even when regressors are configured. + + This is the realistic built-in case: the operator wires a calendar + regressor onto a query AND turns on ``emission.conditional``, but + the configured model (every built-in wrapper) ignores the + regressors parameter. Without this short-circuit, every query of + this shape pays a wasted refit per series per model. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.config import RegressorConfig # noqa: PLC0415 + + call_count = {"n": 0} + + class _CountingRampModel(_RampForecastModel): + uses_regressors = False + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + call_count["n"] += 1 + return super().fit_predict(df, horizon, freq, levels, regressors) + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _CountingRampModel()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: False) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + # Exactly one fit — even though regressors were configured, the + # model marker said it doesn't use them, so the runner aliased the + # unconditional frame. + assert call_count["n"] == 1 + # And the conditional family still landed on /metrics. + assert "m_forecast_conditional" in exporter.render().decode() + + +@pytest.mark.asyncio +async def test_conditional_curve_factory_emits_under_conditional_metric_family( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The new ``conditional_forecast_curve_points`` factory method must + emit curves under the ``_forecast_conditional*`` metric family — + not the unconditional ``_forecast*`` family, which would collide + with the unconditional sink push under the same labelset. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.sink import ForecastCurvePoint, ForecastSink # noqa: PLC0415 + + captured: list[ForecastCurvePoint] = [] + + class _CapturingSink(ForecastSink): + async def write( + self, + points: list[ForecastCurvePoint], + *, + on_retry: object = None, + ) -> None: + captured.extend(points) + + async def aclose(self) -> None: + return None + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="cpu_busy", + promql="up", + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner( + config=cfg, + source=source, + exporter=Exporter(), + sink=_CapturingSink(), + ) + await runner.run_group(cfg.groups[0]) + + # The sink received curve points; the conditional ones must be + # under the ``_forecast_conditional*`` family (not the rewritten + # ``_forecast`` family that the old post-hoc renamer used to leak). + conditional_metrics = {p.metric for p in captured if "_forecast_conditional" in p.metric} + assert "cpu_busy_forecast_conditional" in conditional_metrics + # Lower / upper band rows present too at the configured 80% level. + assert "cpu_busy_forecast_conditional_lower" in conditional_metrics + assert "cpu_busy_forecast_conditional_upper" in conditional_metrics + + +@pytest.mark.asyncio +async def test_conditional_refits_when_model_uses_regressors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A regressor-aware model must still be refit for the conditional pass. + + The Prophet plug-in (and any future regressor-aware model) reads the + held-constant regressor frame at fit time, so the conditional + output legitimately differs from the unconditional. The runner must + not alias the unconditional frame in that case. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + call_count = {"n": 0} + + class _RegressorAwareRamp(_RampForecastModel): + uses_regressors = True + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + call_count["n"] += 1 + return super().fit_predict(df, horizon, freq, levels, regressors) + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorAwareRamp()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + # ``conditional`` is on AND a regressor is configured — the runner + # must run an extra fit per emitted model for the held-constant + # refit. + from promforecast.config import RegressorConfig # noqa: PLC0415 + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=Exporter()) + await runner.run_group(cfg.groups[0]) + + # 2 fits: unconditional + conditional refit. + assert call_count["n"] == 2 + + +@pytest.mark.asyncio +async def test_decomposition_emission_off_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + """No component family until ``emission.decompose`` is flipped on.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query(QueryConfig(id="m", promql="up")) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "m_forecast_component" not in body + assert "forecast_decomposition_skipped_total" not in body + + +@pytest.mark.asyncio +async def test_decomposition_emission_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + """Enabling the toggle materialises one row per component on /metrics.""" + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="cpu_busy", + promql="up", + emission=QueryEmissionConfig(decompose=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "cpu_busy_forecast_component" in body + # All four components show up exactly once per (series, model) at the + # end-of-horizon row. + for component in ("trend", "seasonal", "level", "residual"): + assert f'component="{component}"' in body + + +@pytest.mark.asyncio +async def test_decomposition_curve_pushed_to_sink(monkeypatch: pytest.MonkeyPatch) -> None: + """Per-step component curves must reach the remote_write sink. + + Without the sink push, dashboards built on the per-step decomposition + family have no data to render — only the end-of-horizon snapshot is + available, which is a single point, not a stacked-area panel input. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.sink import ForecastCurvePoint, ForecastSink # noqa: PLC0415 + + captured: list[ForecastCurvePoint] = [] + + class _CapturingSink(ForecastSink): + async def write( + self, + points: list[ForecastCurvePoint], + *, + on_retry: object = None, + ) -> None: + captured.extend(points) + + async def aclose(self) -> None: + return None + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="cpu_busy", + promql="up", + emission=QueryEmissionConfig(decompose=True), + ) + ) + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner( + config=cfg, + source=source, + exporter=Exporter(), + sink=_CapturingSink(), + ) + await runner.run_group(cfg.groups[0]) + + component_curves = [p for p in captured if p.metric == "cpu_busy_forecast_component"] + assert component_curves, "no component curve points reached the sink" + # Each step contributes 4 components (trend/seasonal/level/residual). + seen_components = {p.labels.get("component") for p in component_curves} + assert seen_components == {"trend", "seasonal", "level", "residual"} + + +@pytest.mark.asyncio +async def test_ensemble_row_is_decomposed_alongside_component_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The synthetic ``ensemble`` row must produce its own decomposition. + + ``ensemble`` is deliberately *not* in + :data:`decomposition.NON_DECOMPOSABLE_MODELS` because the averaged + curve is still a valid forecast shape. Operators looking at the + ensemble forecast benefit from seeing trend/seasonal/level/residual + on the *ensemble* curve alongside the same split on each component + model. Lock the behaviour in here so a future tweak to the registry + can't silently drop the ensemble decomposition. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + + cfg = _config_with_query( + QueryConfig( + id="cpu_busy", + promql="up", + emission=QueryEmissionConfig(decompose=True), + ) + ) + # Ensemble mode: emit every configured model PLUS the synthetic + # ``ensemble`` row. + cfg.defaults.accuracy.auto_select = "ensemble" + cfg.defaults.accuracy.evaluate = True # backtests power the weights + cfg.defaults.models = [ + cfg.defaults.models[0], + type(cfg.defaults.models[0])(name="RampModelTwo"), + ] + + source = _FakeSource({"up": [_frame(40)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # The ensemble row's decomposition lands on /metrics alongside the + # component models'. We assert specifically on the ``ensemble`` + # model label rather than just the metric name so a regression that + # drops the ensemble decomposition silently is caught. + component_lines = [ + line for line in body.splitlines() if line.startswith("cpu_busy_forecast_component{") + ] + ensemble_decomp_lines = [line for line in component_lines if 'model="ensemble"' in line] + assert ensemble_decomp_lines, ( + 'ensemble row produced no decomposition; expected `model="ensemble"` rows ' + "on cpu_busy_forecast_component. body sample:\n" + "\n".join(component_lines[:5]) + ) + # And no skip counter was bumped for the ensemble row — it must be + # treated as decomposable, not as an unsupported model. + skip_lines = [ + line + for line in body.splitlines() + if line.startswith("forecast_decomposition_skipped_total{") + and 'model="ensemble"' in line + and line.rsplit(" ", 1)[-1].strip() != "0.0" + ] + assert skip_lines == [], f"ensemble row was unexpectedly skipped: {skip_lines}" + + +@pytest.mark.asyncio +async def test_decomposition_skip_counter_bumps_on_unsupported_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A non-decomposable model name skips emission and bumps the counter. + + The ``Croston`` family is registered as non-decomposable; we point the + registry stub at a Croston-named model so the runner takes the skip + path. The forecast itself still emits normally. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + class _CrostonStub(_RampForecastModel): + name = "Croston" + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _CrostonStub()) + + cfg = _config_with_query( + QueryConfig( + id="m", + promql="up", + emission=QueryEmissionConfig(decompose=True), + ) + ) + # Inject the unsupported model name into the defaults so the snapshot + # forecast points label themselves with ``model="Croston"``. + cfg.defaults.models[0].name = "Croston" + + source = _FakeSource({"up": [_frame(20)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "forecast_decomposition_skipped_total" in body + assert 'model="Croston"' in body + assert 'reason="unsupported_model"' in body + # The skip counter now carries (group, query) labels so operators + # can alert per-query on persistent skip-path tripping. + assert 'group="g"' in body + assert 'query="m"' in body + # Component family is *not* emitted when every emitted model skipped. + assert "m_forecast_component" not in body diff --git a/forecaster/tests/test_slo.py b/forecaster/tests/test_slo.py new file mode 100644 index 0000000..f61d145 --- /dev/null +++ b/forecaster/tests/test_slo.py @@ -0,0 +1,134 @@ +"""Tests for the SLO error-budget primitives.""" + +from __future__ import annotations + +import math +from datetime import UTC, datetime, timedelta + +import pandas as pd + +from promforecast.config import SLOConfig +from promforecast.slo import compute_budget + + +def _forecast_frame(values: list[float], *, step_seconds: int = 60) -> pd.DataFrame: + base = datetime(2026, 5, 21, tzinfo=UTC) + timestamps = [base + timedelta(seconds=i * step_seconds) for i in range(len(values))] + return pd.DataFrame({"unique_id": ["s"] * len(values), "ds": timestamps, "yhat": values}) + + +def test_remaining_seconds_matches_objective_times_window() -> None: + """``remaining_seconds`` is the static budget — ``(1 - SLO/100) * window``.""" + # 99.9% SLO over 30d → allowed_rate = 0.001 → budget = 0.001 * 30d = 2592s + df = _forecast_frame([0.001, 0.001, 0.001]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert len(out) == 1 + assert math.isclose(out[0].remaining_seconds, 0.001 * 30 * 86400, rel_tol=1e-6) + + +def test_burn_rate_one_when_consuming_at_allowed_rate() -> None: + """Burn rate of ``1`` means consuming the budget at the rate that exhausts it.""" + # SLO 99.9% → allowed_rate 0.001. yhat 0.001 → burn_rate 1.0. + df = _forecast_frame([0.001, 0.001, 0.001]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert math.isclose(out[0].burn_rate, 1.0) + + +def test_burn_rate_scales_with_predicted_rate() -> None: + """Burn rate of 30 means consuming the budget 30x faster than sustainable.""" + df = _forecast_frame([0.03, 0.03, 0.03]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert math.isclose(out[0].burn_rate, 30.0) + + +def test_exhausted_in_inverse_of_burn_rate_times_window() -> None: + """``exhausted_in = window / burn_rate`` for the central forecast.""" + # burn_rate=30 → exhaust at 30x the rate → window / 30 seconds. + df = _forecast_frame([0.03, 0.03, 0.03]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + expected = 30 * 86400 / 30 # window / burn_rate + assert math.isclose(out[0].exhausted_in_point, expected, rel_tol=1e-6) + + +def test_exhausted_in_is_infinite_on_zero_forecast() -> None: + """A predicted error rate of zero means the budget never exhausts → +Inf.""" + df = _forecast_frame([0.0, 0.0, 0.0]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert math.isinf(out[0].exhausted_in_point) + + +def test_exhausted_in_is_infinite_on_negative_forecast() -> None: + """A negative forecast (budget recovering) also gives +Inf exhausted_in.""" + df = _forecast_frame([-0.001, -0.001, -0.001]) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert math.isinf(out[0].exhausted_in_point) + + +def test_per_level_exhausted_in_uses_upper_band() -> None: + """Per-level row uses ``yhat_upper`` — the more pessimistic (faster-burn) side.""" + df = _forecast_frame([0.01, 0.01, 0.01]) + df["yhat_upper_95"] = [0.03, 0.03, 0.03] + df["yhat_lower_95"] = [0.005, 0.005, 0.005] + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + out = compute_budget(forecast=df, slo=slo, levels=[95]) + sample = out[0] + # Point uses yhat=0.01 → burn_rate=10 → exhaust at 30d/10 + # Band uses yhat_upper=0.03 → burn_rate=30 → exhaust at 30d/30 (sooner) + assert sample.exhausted_in_per_level[95] < sample.exhausted_in_point + + +def test_no_emission_for_empty_forecast() -> None: + """An empty frame yields no samples (caller can ``extend`` unconditionally).""" + df = pd.DataFrame({"ds": [], "yhat": []}) + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + assert compute_budget(forecast=df, slo=slo, levels=[]) == [] + + +def test_multiple_windows_emit_one_sample_each() -> None: + """Each configured window produces its own row.""" + df = _forecast_frame([0.001, 0.001]) + slo = SLOConfig( + objective=99.9, + windows=[timedelta(minutes=5), timedelta(hours=1), timedelta(days=30)], + ) + out = compute_budget(forecast=df, slo=slo, levels=[]) + assert len(out) == 3 + window_seconds = sorted(s.window_seconds for s in out) + assert window_seconds == [300.0, 3600.0, 30 * 86400.0] + + +def test_no_windows_emits_nothing() -> None: + slo = SLOConfig(objective=99.9, windows=[]) + df = _forecast_frame([0.001, 0.001]) + assert compute_budget(forecast=df, slo=slo, levels=[]) == [] + + +def test_exhausted_in_responds_to_calibrated_band() -> None: + """A wider ``yhat_upper`` (after conformal calibration) shortens exhausted_in. + + Operators alert on the per-level row; the band shown in /metrics + must agree with the budget projection at the same labelset. This + test exercises the contract on the pure helper — the runner-level + fix that threads calibration into ``compute_budget`` is verified + by the runner test suite. + """ + df = _forecast_frame([0.005, 0.005, 0.005]) + # Raw band: yhat_upper exhausted_in at level=95 corresponds to a + # particular budget consumption rate; widening it should make the + # budget exhaust sooner. + df["yhat_upper_95"] = [0.005, 0.005, 0.005] + df["yhat_lower_95"] = [0.005, 0.005, 0.005] + slo = SLOConfig(objective=99.9, windows=[timedelta(days=30)]) + raw = compute_budget(forecast=df, slo=slo, levels=[95]) + raw_level = raw[0].exhausted_in_per_level[95] + + df["yhat_upper_95"] = [0.05, 0.05, 0.05] # calibrated wider + cal = compute_budget(forecast=df, slo=slo, levels=[95]) + cal_level = cal[0].exhausted_in_per_level[95] + assert cal_level < raw_level diff --git a/forecaster/tests/test_validate_cli.py b/forecaster/tests/test_validate_cli.py index 9261cf2..c44de8c 100644 --- a/forecaster/tests/test_validate_cli.py +++ b/forecaster/tests/test_validate_cli.py @@ -25,6 +25,14 @@ def _write(tmp_path: Path, body: str) -> Path: apiVersion: promforecast.io/v1 datasource: url: http://vm:8428/ +safety: + # Default emission set (accuracy + deviation + quality + drift + + # band_coverage at default windows) on two 500-series queries + # projects ~21k lines — comfortably over the 5000-default cap. Bump + # the cap so the strict-mode happy path is exercised without + # disabling every default. Configs that genuinely fit under the + # default cap are exercised separately. + max_total_series: 100000 defaults: models: [SeasonalNaive] confidence_levels: [80, 95] @@ -191,6 +199,68 @@ def test_cardinality_estimate_counts_threshold_and_growth_rate_lines() -> None: assert est.growth_rate_lines == 100 * 1 * 2 +def test_validate_strict_cardinality_passes_when_under_budget(tmp_path: Path) -> None: + """A config whose estimated total fits inside the cap exits 0 in strict mode.""" + path = _write(tmp_path, _GOOD_CONFIG) + out = io.StringIO() + rc = run_validate(path, strict_cardinality=True, out=out) + assert rc == 0 + + +def test_validate_strict_cardinality_fails_when_over_budget(tmp_path: Path) -> None: + """Strict mode flips the over-budget WARN into a non-zero exit code. + + Tiny ``max_total_series`` against the default emission set guarantees + the estimate overshoots, so the test is deterministic without + materialising a contrived large config. + """ + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +safety: + max_total_series: 1 +defaults: + models: [SeasonalNaive] +groups: + - name: g + queries: + - id: m + promql: up +""", + ) + out = io.StringIO() + rc = run_validate(path, strict_cardinality=True, out=out) + assert rc == 1 + + +def test_validate_advisory_mode_returns_zero_when_over_budget(tmp_path: Path) -> None: + """Default (non-strict) mode keeps the historical advisory-only behaviour.""" + path = _write( + tmp_path, + """ +apiVersion: promforecast.io/v1 +datasource: + url: http://vm:8428/ +safety: + max_total_series: 1 +defaults: + models: [SeasonalNaive] +groups: + - name: g + queries: + - id: m + promql: up +""", + ) + out = io.StringIO() + rc = run_validate(path, out=out) + assert rc == 0 + assert "WARN: total exceeds safety.max_total_series" in out.getvalue() + + def test_validate_dry_run_reports_failure_when_datasource_unreachable(tmp_path: Path) -> None: """Dry-run failures must not crash the CLI; they return rc=2.""" path = _write(