diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml new file mode 100644 index 0000000..1cfb41b --- /dev/null +++ b/.github/workflows/nightly-bench.yml @@ -0,0 +1,48 @@ +name: Nightly benchmarks + +# Micro-benchmarks gated out of the per-PR ``pytest`` run because shared +# CI throughput fluctuates. The runner's hot-path suffix lookup is the +# only benchmark today (see ``forecaster/tests/perf/`` and the +# ``benchmark`` pytest marker); this workflow opts in so a regression +# like "lru_cache silently dropped" is caught within 24 hours instead +# of waiting for someone to run the marker locally. +# +# Failure here does NOT block any merge — it surfaces as a job-level +# failure in the Actions tab. Treat it as a heads-up signal. + +on: + schedule: + # 03:17 UTC — picked off the hour so the run doesn't pile onto the + # GitHub-wide top-of-hour scheduling spike. Daily cadence keeps the + # signal fresh without burning CI minutes. + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + bench: + name: Microbenchmarks (opt-in) + runs-on: ubuntu-latest + defaults: + run: + working-directory: forecaster + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "forecaster/pyproject.toml" + + - run: uv sync --all-extras + + # ``-m benchmark`` flips the marker filter set in + # ``forecaster/pyproject.toml`` (which defaults to + # ``-m 'not benchmark'``) so only the perf suite runs. Keeping the + # marker name in lockstep with the test module is intentional — + # adding a benchmark elsewhere requires the same marker. + - name: pytest -m benchmark + run: uv run pytest -m benchmark tests/perf -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d56de4e..b886b9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,7 @@ make chart-lint # helm lint + kubeconform changes and require a deprecation period. - `mypy --strict` and `ruff` must be clean. - Adding a model? Update three places: `forecaster/src/promforecast/models/`, - `pyproject.toml` entry points, and `docs/models.md`. + `pyproject.toml` entry points, and `docs/reference/models.md`. - Adding a Helm value? Regenerate `values.schema.json` (`make chart-schema`). ## Commit style diff --git a/README.md b/README.md index cc5068e..83978a7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ VM with native future timestamps, and the bundled ServiceMonitor drops the redundant `*_forecast` / `*_forecast_lower` / `*_forecast_upper` snapshot families from the scrape so dashboards plot a smooth forward curve rather than averaging the sink push against a 5-minute snapshot -staircase. See [`docs/architecture/emission-paths.md`](docs/architecture/emission-paths.md) +staircase. See [`docs/reference/emission-paths.md`](docs/reference/emission-paths.md) for what each path emits and the dual-source trap that motivates the defaults. Existing snapshot-only installs upgrading to the new defaults should follow [`docs/how-to/migrate-to-sink-first.md`](docs/how-to/migrate-to-sink-first.md) @@ -49,7 +49,7 @@ Editing the YAML config does **not** require a pod restart: the forecaster watches the mounted ConfigMap, validates the new config in-place, and rolls back on validation failure (`kubectl apply` to the ConfigMap is enough). `POST /-/reload` and `SIGHUP` trigger the same -path. See [`docs/cli.md`](docs/cli.md) for `promforecast validate`, +path. See [`docs/reference/cli.md`](docs/reference/cli.md) for `promforecast validate`, which CI users can run against a config file before merging. See [`docs/`](docs/) for configuration, model selection, and operational diff --git a/charts/promforecast-stack/README.md b/charts/promforecast-stack/README.md index 00d764a..acd9d73 100644 --- a/charts/promforecast-stack/README.md +++ b/charts/promforecast-stack/README.md @@ -42,7 +42,7 @@ on demand: helm test ``` -See [`docs/architecture/emission-paths.md`](../../docs/architecture/emission-paths.md) +See [`docs/reference/emission-paths.md`](../../docs/reference/emission-paths.md) for what the two paths emit, why running both unguarded is a silent trap, and the `metric_relabel_configs` snippet to paste into an externally-managed Prometheus. diff --git a/charts/promforecast-stack/templates/datasources.yaml b/charts/promforecast-stack/templates/datasources.yaml index 4921d5b..4da0c12 100644 --- a/charts/promforecast-stack/templates/datasources.yaml +++ b/charts/promforecast-stack/templates/datasources.yaml @@ -5,7 +5,7 @@ up automatically. VictoriaMetrics is registered as the default so the bundled dashboards land on the sink-fed copy of the forecast metrics (smooth curves) rather than the scraped snapshot (5-min staircase). - See ``docs/architecture/emission-paths.md`` for why the default matters. + See ``docs/reference/emission-paths.md`` for why the default matters. */}} {{- $vm := .Values.datasources.victoriaMetrics }} {{- $prom := .Values.datasources.prometheus }} diff --git a/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml b/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml index b975b4f..9592c01 100644 --- a/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml +++ b/charts/promforecast-stack/templates/tests/labelset-uniqueness.yaml @@ -110,7 +110,7 @@ spec: echo "Remediation: promforecast.serviceMonitor.enabled=true (the umbrella default)" >&2 echo " so the standalone chart drops the snapshot families. For an" >&2 echo " externally-managed Prometheus, add the metric_relabel_configs" >&2 - echo " snippet from docs/architecture/emission-paths.md." >&2 + echo " snippet from docs/reference/emission-paths.md." >&2 echo "$dup_out" >&2 exit 1 fi diff --git a/charts/promforecast-stack/values.yaml b/charts/promforecast-stack/values.yaml index 1a3bc59..3a9e667 100644 --- a/charts/promforecast-stack/values.yaml +++ b/charts/promforecast-stack/values.yaml @@ -64,7 +64,7 @@ promforecast: # fail with ``no matches for kind ServiceMonitor`` on a cluster without # the CRD. For installs that scrape with a plain Prometheus (no # operator), flip this off and paste the ``metric_relabel_configs`` - # snippet from ``docs/architecture/emission-paths.md`` into your scrape + # snippet from ``docs/reference/emission-paths.md`` into your scrape # config — the snapshot families must be dropped somewhere or the # dual-source trap reappears. serviceMonitor: diff --git a/charts/promforecast/README.md b/charts/promforecast/README.md index de8bbb0..9729ae5 100644 --- a/charts/promforecast/README.md +++ b/charts/promforecast/README.md @@ -74,7 +74,7 @@ release-name-agnostically: > forecaster pod's `instance` need to be updated to the new stable > values before upgrading. -See [`docs/architecture/emission-paths.md`](../../docs/architecture/emission-paths.md) +See [`docs/reference/emission-paths.md`](../../docs/reference/emission-paths.md) for what each emission path emits and the dual-source trap the drop prevents. diff --git a/charts/promforecast/templates/servicemonitor.yaml b/charts/promforecast/templates/servicemonitor.yaml index 720d9d6..fead904 100644 --- a/charts/promforecast/templates/servicemonitor.yaml +++ b/charts/promforecast/templates/servicemonitor.yaml @@ -15,7 +15,7 @@ */}} {{- $explicit := .Values.serviceMonitor.dropForecastSnapshots }} {{- if and .Values.existingConfigMap (not (kindIs "bool" $explicit)) }} -{{- fail "serviceMonitor.dropForecastSnapshots must be set explicitly (true/false) when existingConfigMap is set — the chart cannot read the sink toggle from an externally-managed ConfigMap. See docs/architecture/emission-paths.md for the dual-source labelset trap." }} +{{- fail "serviceMonitor.dropForecastSnapshots must be set explicitly (true/false) when existingConfigMap is set — the chart cannot read the sink toggle from an externally-managed ConfigMap. See docs/reference/emission-paths.md for the dual-source labelset trap." }} {{- end }} {{- $sinkOn := dig "sink" "remote_write" "enabled" false .Values.config }} {{- $drop := ternary $explicit $sinkOn (kindIs "bool" $explicit) }} @@ -56,7 +56,7 @@ spec: # making the example alerts in ``examples/alerts/promforecast-rules.yaml`` # (which interpolate ``{{`{{ $labels.instance }}`}}`` to identify the # node-exporter target) report the wrong host. Matches the - # ``honor_labels: true`` snippet in ``docs/architecture/emission-paths.md``. + # ``honor_labels: true`` snippet in ``docs/reference/emission-paths.md``. honorLabels: true interval: {{ .Values.serviceMonitor.interval }} scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} diff --git a/charts/promforecast/values.yaml b/charts/promforecast/values.yaml index 2859bce..708138c 100644 --- a/charts/promforecast/values.yaml +++ b/charts/promforecast/values.yaml @@ -27,7 +27,7 @@ config: # Schema version. The forecaster routes on this string and warns when # it is missing. ``promforecast.io/v1`` is the stable schema; # ``promforecast.io/v1alpha1`` still loads (with a DeprecationWarning) - # for configs predating the stable cut-over. See docs/config-schema.md + # for configs predating the stable cut-over. See docs/reference/config-schema.md # for the deprecation policy. apiVersion: promforecast.io/v1 diff --git a/docker/prometheus.yml b/docker/prometheus.yml index 3d007ba..5547d23 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -62,7 +62,7 @@ scrape_configs: # Operational metrics (``forecast_failures_total``, ``forecast_quality_score``, # ``forecast_run_duration_seconds``, etc.) stay scraped — they only # exist on /metrics regardless of sink configuration. See - # docs/architecture/emission-paths.md for the dual-source trap and the + # docs/reference/emission-paths.md for the dual-source trap and the # cross-metric-join labelset asymmetry this drop preserves. metric_relabel_configs: - source_labels: [__name__] diff --git a/docs/README.md b/docs/README.md index 28f6c3c..99a5b1a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,38 +1,68 @@ # Documentation -The docs follow the [Diataxis](https://diataxis.fr/) framework: +Organised by pipeline position — input → model → output — then operations, +guides, and reference around the edges. If you're not sure where a topic +lives, the section descriptions below should point you in the right direction. -- **Tutorials** — learning-oriented, end-to-end walkthroughs. -- **How-tos** — task-oriented, "how do I X?" recipes. -- **Reference** — information-oriented, exhaustive details. -- **Explanation** — understanding-oriented, design and trade-offs. +## Reference -## Index +Information-oriented, exhaustive details. Look here when you know the name +of the thing and want its full specification. + +- [Config schema](reference/config-schema.md) — `apiVersion`, stability guarantees, and the `v1alpha1` → `v1` migration +- [CLI](reference/cli.md) — `promforecast run`, `promforecast validate`, `promforecast diagnose` +- [Models](reference/models.md) — built-in forecasters, auto-select, accuracy metrics +- [Emission paths (snapshot vs. sink + VM)](reference/emission-paths.md) — what each path emits, the dual-source trap, and why the official charts default to sink + VM + +## Inputs + +Everything that shapes the data flowing into the forecasting model. + +- [Preprocessing](inputs/preprocessing.md) — gap imputation and outlier scrubbing +- [Change-points](inputs/change-points.md) — input-side break detection and lookback adaptation +- [Seasonality](inputs/seasonality.md) — periodicity detection +- [Cold-start](inputs/cold-start.md) — sparse-history behaviour and peer-sibling borrowing +- [Regressors](inputs/regressors.md) — calendar, custom PromQL, and holiday regressors, plus conditional forecasts (baseline given the current regressor state) +- [Series discovery](inputs/discovery.md) — template-driven auto-fan-out over label values + +## Outputs + +Everything the forecaster emits beyond raw forecast curves — derived +metrics, calibration, explainability, and ad-hoc what-if endpoints. + +- [Capacity planning](outputs/capacity.md) — damped trend, time-to-threshold ETAs, growth rate, will-breach predictive boolean, queue burn-down (drain) forecasting +- [SLO error-budget forecasting](outputs/slo.md) — burn rate, exhaustion projection, multi-window alerting +- [Maintenance window recommendations](outputs/maintenance-windows.md) — top-N lowest-load windows for scheduling reboots, rollouts, and cert rotations +- [Conformal calibration](outputs/conformal.md) — empirical-coverage band correction +- [Explainability (decomposition)](outputs/explainability.md) — trend / seasonal / level / residual split of the forecast curve +- [Forecast drift](outputs/drift.md) — prediction-vs-prediction shift telemetry +- [Forecast preview](outputs/preview.md) — opt-in ad-hoc what-if endpoint +- [Canary forecasts](outputs/canary.md) + +## Operations + +Running the forecaster in production — diagnosis, capacity safeguards, +scheduling, and serving telemetry. -- [Emission paths (snapshot vs. sink + VM)](architecture/emission-paths.md) — what each path emits, the dual-source trap, and why the official charts default to sink + VM -- [Config schema](config-schema.md) — `apiVersion`, stability guarantees, and the `v1alpha1` → `v1` migration -- [Models](models.md) — built-in forecasters, auto-select, accuracy metrics -- [CLI](cli.md) — `promforecast run`, `promforecast validate`, `promforecast diagnose` - [Reload verification](operations/reload.md) — confirming a config reload actually picked up the expected queries - [Duplicate-labelset diagnosis](operations/diagnose.md) — detecting and remediating the scrape+sink dual-source state - [Cardinality budgeting](operations/cardinality.md) — per-family multipliers and the `--strict-cardinality` CI gate -- [High availability](ha.md) — Lease-based leader election, Redis snapshot cache -- [Query cache](cache.md) — in-memory LRU + optional Redis dedup for overlapping PromQL -- [Series discovery](discovery.md) — template-driven auto-fan-out over label values -- [Preprocessing](preprocessing.md) — gap imputation and outlier scrubbing -- [Change-points](change-points.md) — input-side break detection and lookback adaptation -- [Conformal calibration](conformal.md) — empirical-coverage band correction -- [Capacity planning](capacity.md) — damped trend, time-to-threshold ETAs, growth rate, will-breach predictive boolean, queue burn-down (drain) forecasting -- [SLO error-budget forecasting](slo.md) — burn rate, exhaustion projection, multi-window alerting -- [Maintenance window recommendations](operations/maintenance-windows.md) — top-N lowest-load windows for scheduling reboots, rollouts, and cert rotations -- [Conditional forecasts](conditional-forecasts.md) — baseline given the current regressor state, for "unusual *for current conditions*" alerts -- [Explainability (decomposition)](explainability.md) — trend / seasonal / level / residual split of the forecast curve -- [Forecast drift](drift.md) — prediction-vs-prediction shift telemetry -- [Reading the numbers](reading-the-numbers.md) — how to interpret quality and accuracy in practice (mean vs p95, horizon paradox, "wait three cycles", when to actually worry) -- [Scheduling](scheduling.md) — per-query refresh intervals, max-concurrent-fits cap -- [Forecast preview](preview.md) — opt-in ad-hoc what-if endpoint -- [Regressors](regressors.md) — calendar, custom PromQL, and holiday regressors -- [Development](development.md) — local toolchain, CI overview, adding models +- [Scheduling](operations/scheduling.md) — per-query refresh intervals, max-concurrent-fits cap +- [High availability](operations/ha.md) — Lease-based leader election, Redis snapshot cache +- [Query cache](operations/cache.md) — in-memory LRU + optional Redis dedup for overlapping PromQL +- [Telemetry](operations/telemetry.md) — internal metrics catalogue + +## Guides + +Cross-cutting explanations that don't fit cleanly under one feature. + +- [Reading the numbers](guides/reading-the-numbers.md) — how to interpret quality and accuracy in practice (mean vs p95, horizon paradox, "wait three cycles", when to actually worry) + +## Contributing + +- [Development](contributing/development.md) — local toolchain, CI overview, adding models + +--- For a copy-and-edit reference deployment (multi-group config, HA values, ArgoCD/Flux manifests, architecture diagram), see diff --git a/docs/conditional-forecasts.md b/docs/conditional-forecasts.md deleted file mode 100644 index cc96ff9..0000000 --- a/docs/conditional-forecasts.md +++ /dev/null @@ -1,66 +0,0 @@ -# Conditional forecasts - -The default forecast answers: “Is the actual inside the historic envelope?” -That works for most cases, but breaks when a known regressor legitimately shifts the metric (e.g. a deploy that *should* increase request rate). - -**Conditional forecasts** answer: “Given the *current* state of the regressors, what should the metric look like now?” -Use the conditional deviation when you want to detect anomalies *after accounting for known external drivers*. - -## Configuration - -Per-query, off by default: - -```yaml -groups: - - name: http - queries: - - id: http_requests_rate - promql: sum(rate(http_requests_total[1m])) by (endpoint) - regressors: - - id: deploys - promql: changes(kube_deployment_status_observed_generation[1h]) - emission: - conditional: true -``` - -The runner re-fits each emitted model while holding the most recent regressor values constant. - -## Emitted metrics - -| Metric | Description | -|---------------------------------------------|-----------| -| `_forecast_conditional` | Point forecast at horizon | -| `_forecast_conditional_lower{level}` | Lower band per level | -| `_forecast_conditional_upper{level}` | Upper band per level | -| `forecast_conditional_deviation_ratio` | Signed deviation of latest actual | -| `forecast_conditional_deviation_outside_band` | 1 if actual is outside the conditional band | - -These carry the same labels as the unconditional deviation family, so you can alert on them directly: - -```promql -# Anomaly after accounting for known regressors -forecast_conditional_deviation_outside_band{level="95"} == 1 - and on (id, group, model) forecast_quality_score > 0.6 -``` - -## Model behaviour - -Built-in models (`AutoARIMA`, `AutoETS`, `MSTL`, `SeasonalNaive`, etc.) do **not** use regressors. -They advertise `uses_regressors = False`, so the runner **short-circuits** the conditional refit and aliases the unconditional forecast. - -You still get a stable schema (conditional metrics always exist), but at **zero extra CPU cost**. - -Regressor-aware models (e.g. Prophet plug-in) perform a real second fit with held-constant regressors. - -## Cardinality - -Roughly doubles the forecast snapshot footprint for opted-in queries. -`promforecast validate` includes the extra lines in its estimate. - -See [operations/cardinality.md](operations/cardinality.md) for the full multiplier table. - -## When not to use it - -- No regressors on the query (conditional becomes a pure duplicate). -- No model in the set uses regressors (wastes cardinality). -- You want classic anomaly detection that treats regressor shifts as real deviations. diff --git a/docs/development.md b/docs/contributing/development.md similarity index 98% rename from docs/development.md rename to docs/contributing/development.md index 7405946..98a63b2 100644 --- a/docs/development.md +++ b/docs/contributing/development.md @@ -56,7 +56,7 @@ Three files must stay in sync: 1. `forecaster/src/promforecast/models/.py` (model wrapper). 2. `forecaster/pyproject.toml` (entry-point registration). -3. `docs/models.md` (user docs). +3. `docs/reference/models.md` (user docs). CI automatically tests the full registered model set. diff --git a/docs/reading-the-numbers.md b/docs/guides/reading-the-numbers.md similarity index 91% rename from docs/reading-the-numbers.md rename to docs/guides/reading-the-numbers.md index 421a6c5..4e64bb4 100644 --- a/docs/reading-the-numbers.md +++ b/docs/guides/reading-the-numbers.md @@ -25,7 +25,7 @@ There is **no universal “good MAPE”** — it depends heavily on the signal. | Bursty | `50–100 %` | Disk I/O with occasional spikes | | Sparse / near-zero | `100 % +` (or NaN) | Rare error rates | -For sparse/near-zero series, MAPE becomes unreliable. Set `data_profile: intermittent` on the query to switch to MASE scoring (see [models.md](models.md)). +For sparse/near-zero series, MAPE becomes unreliable. Set `data_profile: intermittent` on the query to switch to MASE scoring (see [`../reference/models.md`](../reference/models.md)). ## Mean quality vs. p95 MAPE — why they can disagree @@ -85,7 +85,7 @@ For per-series drill-down (quality, drift, deviation, durations, errors) switch ## See also -- [telemetry.md](telemetry.md) — full list of emitted metrics and labels -- [conformal.md](conformal.md) — how bands are empirically calibrated -- [drift.md](drift.md) — `forecast_drift_score` semantics -- [scheduling.md](scheduling.md) — how `refresh_interval` and per-query overrides work +- [`../operations/telemetry.md`](../operations/telemetry.md) — full list of emitted metrics and labels +- [`../outputs/conformal.md`](../outputs/conformal.md) — how bands are empirically calibrated +- [`../outputs/drift.md`](../outputs/drift.md) — `forecast_drift_score` semantics +- [`../operations/scheduling.md`](../operations/scheduling.md) — how `refresh_interval` and per-query overrides work diff --git a/docs/change-points.md b/docs/inputs/change-points.md similarity index 97% rename from docs/change-points.md rename to docs/inputs/change-points.md index 77eb58b..80f95be 100644 --- a/docs/change-points.md +++ b/docs/inputs/change-points.md @@ -51,5 +51,5 @@ Dashboard tip: stack the static `defaults.lookback` against `forecast_input_effe ## See also -- [Models](models.md) — how the adjusted lookback feeds the fit. +- [Models](../reference/models.md) — how the adjusted lookback feeds the fit. - [Input preprocessing](preprocessing.md) — outlier scrubbing runs *before* change-point detection so a single spike cannot masquerade as a regime shift. diff --git a/docs/cold-start.md b/docs/inputs/cold-start.md similarity index 100% rename from docs/cold-start.md rename to docs/inputs/cold-start.md diff --git a/docs/discovery.md b/docs/inputs/discovery.md similarity index 100% rename from docs/discovery.md rename to docs/inputs/discovery.md diff --git a/docs/preprocessing.md b/docs/inputs/preprocessing.md similarity index 96% rename from docs/preprocessing.md rename to docs/inputs/preprocessing.md index 02e5acb..aa218cf 100644 --- a/docs/preprocessing.md +++ b/docs/inputs/preprocessing.md @@ -82,5 +82,5 @@ With `outliers.enabled: true, method: hampel, sensitivity: medium` the spike is ## Interaction with conformal calibration -Outlier scrubbing runs **before** conformal calibration (see [`conformal.md`](conformal.md)). +Outlier scrubbing runs **before** conformal calibration (see [`../outputs/conformal.md`](../outputs/conformal.md)). A clean hold-out window produces tighter, more honest quantile bands. diff --git a/docs/inputs/regressors.md b/docs/inputs/regressors.md new file mode 100644 index 0000000..1f3767f --- /dev/null +++ b/docs/inputs/regressors.md @@ -0,0 +1,179 @@ +# Exogenous regressors + +A regressor is an extra input fed to the forecasting model alongside the primary series. Three flavours are supported: + +- **`calendar`** — deterministic features derived from timestamps (`hour_of_day`, `day_of_week`, `is_weekend`). +- **`custom`** — any PromQL query whose result is aligned to the primary series. +- **`holidays`** — public-holiday calendar from the [`holidays`](https://pypi.org/project/holidays/) package (install with `pip install 'promforecast[holidays]'`). + +Regressors are declared per query under `regressors:`. Built-in models ignore them today; plug-in models (e.g. `promforecast-prophet`) receive them via the `regressors` argument to `fit_predict`. + +## Calendar regressors + +```yaml +queries: + - id: requests_per_second + promql: rate(http_requests_total[5m]) + regressors: + - id: hour_of_day + type: calendar + - id: day_of_week + type: calendar + - id: is_weekend + type: calendar +``` + +The `id` must be one of the three supported names — the schema rejects anything else at boot. + +## Custom (PromQL) regressors + +```yaml +regressors: + - id: deploys + type: custom + promql: changes(kube_deployment_status_observed_generation[1h]) + required: false # default +``` + +- The PromQL is executed once per primary query and aligned via nearest-asof join. +- `required: true` makes a failure fatal for the whole series. +- Failures are counted in `forecast_regressor_failures_total{group, query, regressor, reason}`. + +## Holiday regressors + +Requires the `holidays` extra. + +### Binary mode (default) + +```yaml +regressors: + - id: is_holiday + type: holidays + country: US # ISO-3166-1 alpha-2 + regions: [] # country-wide by default + timezone: America/New_York # optional, defaults to UTC + expand: false # default +``` + +Emits a single binary column `is_holiday` (1.0 on any holiday, 0.0 otherwise). + +### One-hot mode + +```yaml +regressors: + - id: holiday + type: holidays + country: US + regions: [CA, NY] # union of calendars + expand: true +``` + +Emits one column per observed holiday name (`holiday__thanksgiving`, `holiday__independence_day`, …). +If the lookback window contains no holidays, falls back to a single binary column so the feature schema stays stable. + +### Timezone & regions + +- `timezone` (IANA, e.g. `America/New_York`) converts UTC timestamps to local dates. Defaults to `UTC`. +- `regions` takes a list of subdivisions (states/provinces); the runner unions their calendars. + +Supported countries/regions are whatever the installed `holidays>=0.50` package provides (≥150 countries + most major subdivisions). + +## Failure handling + +| Failure type | Reason label | `required: true` behaviour | +|-------------------------------------|-------------------|-------------------------------------| +| Invalid calendar id | `invalid_type` | Series skipped | +| Custom PromQL error / timeout | `query_error` / `query_timeout` | Entire series skipped | +| Custom PromQL returns no data | `empty_result` | Entire series skipped | +| No datasource (dry-run / test) | `no_source` | Entire series skipped | +| `holidays` package missing | `invalid_type` | Series skipped | +| Unknown country / region | `invalid_type` | Series skipped | + +Optional regressors (default) only increment the failure counter; the forecast still emits without that feature. + +## Contribution metric + +When any regressor contributes signal, the runner emits a normalised explainability share: + +``` +_forecast_contribution_ratio{regressor, ...} # values in [0, 1], sum to 1 per series +``` + +Disable via `defaults.emission.contribution: false` (see [`../reference/config-schema.md`](../reference/config-schema.md)). + +## Conditional forecasts + +The default forecast answers: "Is the actual inside the historic envelope?" +That works for most cases, but breaks when a known regressor legitimately shifts the metric (e.g. a deploy that *should* increase request rate). + +**Conditional forecasts** answer: "Given the *current* state of the regressors, what should the metric look like now?" +Use the conditional deviation when you want to detect anomalies *after accounting for known external drivers*. + +### Configuration + +Per-query, off by default: + +```yaml +groups: + - name: http + queries: + - id: http_requests_rate + promql: sum(rate(http_requests_total[1m])) by (endpoint) + regressors: + - id: deploys + promql: changes(kube_deployment_status_observed_generation[1h]) + emission: + conditional: true +``` + +The runner re-fits each emitted model while holding the most recent regressor values constant. + +### Emitted metrics + +| Metric | Description | +|---------------------------------------------|-----------| +| `_forecast_conditional` | Point forecast at horizon | +| `_forecast_conditional_lower{level}` | Lower band per level | +| `_forecast_conditional_upper{level}` | Upper band per level | +| `forecast_conditional_deviation_ratio` | Signed deviation of latest actual | +| `forecast_conditional_deviation_outside_band` | 1 if actual is outside the conditional band | + +These carry the same labels as the unconditional deviation family, so you can alert on them directly: + +```promql +# Anomaly after accounting for known regressors +forecast_conditional_deviation_outside_band{level="95"} == 1 + and on (id, group, model) forecast_quality_score > 0.6 +``` + +### Model behaviour + +Built-in models (`AutoARIMA`, `AutoETS`, `MSTL`, `SeasonalNaive`, etc.) do **not** use regressors. +They advertise `uses_regressors = False`, so the runner **short-circuits** the conditional refit and aliases the unconditional forecast. + +You still get a stable schema (conditional metrics always exist), but at **zero extra CPU cost**. + +Regressor-aware models (e.g. Prophet plug-in) perform a real second fit with held-constant regressors. + +### Conditional-band conformal calibration + +When `accuracy.conformal.enabled: true` *and* `emission.conditional: true` are both on for a query, the conditional band carries its own empirical-coverage guarantee — derived from a dedicated calibration fit where regressors are held flat at the final pre-holdout value, matching the production-time conditional inference contract. + +The per-(model, level) widening applied to the conditional band lands on `/metrics` as `forecast_conditional_band_calibration_offset{id, group, model, level}` (gauge, same units as the underlying metric), sibling of the unconditional `forecast_band_calibration_offset`. Comparing the two reveals whether the conditional band needs different widening than the unconditional one — typically yes, because the unconditional residuals were scored on a holdout where regressors *did* vary. + +Per-(model) calibration failures (degenerate holdout, fit timeout, fit error) fall back to the inherited unconditional offsets and bump `forecast_conditional_calibration_failures_total{group, query, reason}` with `reason` one of `degenerate_holdout`, `fit_timeout`, `fit_error`. The fallback path is no worse than the inherited-offset baseline, but a sustained tick rate signals fresh calibration is silently not running. + +Models that advertise `uses_regressors = False` skip the extra calibration fit entirely — the conditional fit would produce the same numbers as the unconditional fit, so emitting two parallel offset gauges with identical values would just be noise. This means the gauge family appears only when at least one emitted model is regressor-aware. + +### Cardinality + +Roughly doubles the forecast snapshot footprint for opted-in queries. +`promforecast validate` includes the extra lines in its estimate. + +See [`../operations/cardinality.md`](../operations/cardinality.md) for the full multiplier table. + +### When not to use it + +- No regressors on the query (conditional becomes a pure duplicate). +- No model in the set uses regressors (wastes cardinality). +- You want classic anomaly detection that treats regressor shifts as real deviations. diff --git a/docs/seasonality.md b/docs/inputs/seasonality.md similarity index 100% rename from docs/seasonality.md rename to docs/inputs/seasonality.md diff --git a/docs/cache.md b/docs/operations/cache.md similarity index 100% rename from docs/cache.md rename to docs/operations/cache.md diff --git a/docs/operations/cardinality.md b/docs/operations/cardinality.md index 7688858..db34cfc 100644 --- a/docs/operations/cardinality.md +++ b/docs/operations/cardinality.md @@ -45,6 +45,7 @@ Notation: | Drain ETA (`le_drain`) | `S × M × N_drain_thresholds × (1 + L)` | Charged separately from the standard ETA | | Growth rate | `S × M × N_windows` | `emission.growth_rate.enabled: true` | | Conditional forecast families | `S × M × (1 + 2L)` | `emission.conditional: true` (may add extra fit) | +| `forecast_conditional_band_calibration_offset` | `S × M × L` | `emission.conditional: true` *and* `accuracy.conformal.enabled: true` | | SLO error budget | `S × M × N_windows × (3 + L)` | `slo:` configured (3 sibling families × N_windows, plus level fan-out on exhausted_in) | | Recommended maintenance windows | `S × M × count` | `recommended_windows.enabled: true` | | Operational + preprocessing families | ~7 per group + ~6 per query | Mostly always on | @@ -71,7 +72,7 @@ With default `safety.max_total_series: 5000` this would fail `--strict-cardinali ## See also -- [Architecture — emission paths](../architecture/emission-paths.md) -- [Capacity planning](../capacity.md) -- [Conditional forecasts](../conditional-forecasts.md) -- [Explainability](../explainability.md) +- [Emission paths](../reference/emission-paths.md) +- [Capacity planning](../outputs/capacity.md) +- [Conditional forecasts](../inputs/regressors.md#conditional-forecasts) +- [Explainability](../outputs/explainability.md) diff --git a/docs/operations/diagnose.md b/docs/operations/diagnose.md index b70f4bf..d65683c 100644 --- a/docs/operations/diagnose.md +++ b/docs/operations/diagnose.md @@ -138,5 +138,4 @@ kubectl run promforecast-diagnose --rm -it --image=ghcr.io/... \ ## See also -- [Emission paths](../architecture/emission-paths.md) — why dual-source is a problem. -- [Migrate to sink-first](../how-to/migrate-to-sink-first.md) — recommended remediation. +- [Emission paths](../reference/emission-paths.md) — why dual-source is a problem. diff --git a/docs/ha.md b/docs/operations/ha.md similarity index 100% rename from docs/ha.md rename to docs/operations/ha.md diff --git a/docs/scheduling.md b/docs/operations/scheduling.md similarity index 100% rename from docs/scheduling.md rename to docs/operations/scheduling.md diff --git a/docs/telemetry.md b/docs/operations/telemetry.md similarity index 100% rename from docs/telemetry.md rename to docs/operations/telemetry.md diff --git a/docs/canary.md b/docs/outputs/canary.md similarity index 100% rename from docs/canary.md rename to docs/outputs/canary.md diff --git a/docs/capacity.md b/docs/outputs/capacity.md similarity index 100% rename from docs/capacity.md rename to docs/outputs/capacity.md diff --git a/docs/conformal.md b/docs/outputs/conformal.md similarity index 84% rename from docs/conformal.md rename to docs/outputs/conformal.md index 34baee1..e6d8008 100644 --- a/docs/conformal.md +++ b/docs/outputs/conformal.md @@ -41,6 +41,16 @@ Additive offset (in metric units) applied to each side of the band. Zero means the original parametric band already met its nominal coverage. Emitted bands also carry the label `calibrated="true"` on `_forecast_lower` / `_forecast_upper` so dashboards can filter or color them separately. +### Conditional band + +When `emission.conditional: true` is also on for the query, the conditional band carries its own per-fit calibration (regressors held flat at the final pre-holdout value, matching the production-time conditional inference contract). The per-(model, level) widening lands as: + +``` +forecast_conditional_band_calibration_offset{id, group, model, level} +``` + +Per-(model) calibration failures fall back to the inherited unconditional offsets and bump `forecast_conditional_calibration_failures_total{group, query, reason}` (`reason` ∈ `degenerate_holdout` | `fit_timeout` | `fit_error`). See [`../inputs/regressors.md`](../inputs/regressors.md#conditional-band-conformal-calibration). + ## When conformal can’t help - **Insufficient lookback** — fewer than five points on each side of the split. @@ -119,6 +129,6 @@ comfortably bounded. ## See also -- [Models](models.md) — backtest accuracy feeds the calibrator. -- [Input preprocessing](preprocessing.md) — outlier scrubbing runs *before* calibration to keep the hold-out clean. +- [Models](../reference/models.md) — backtest accuracy feeds the calibrator. +- [Input preprocessing](../inputs/preprocessing.md) — outlier scrubbing runs *before* calibration to keep the hold-out clean. - [Forecast drift telemetry](drift.md) — complements coverage by surfacing prediction-vs-prediction shifts. diff --git a/docs/drift.md b/docs/outputs/drift.md similarity index 100% rename from docs/drift.md rename to docs/outputs/drift.md diff --git a/docs/explainability.md b/docs/outputs/explainability.md similarity index 97% rename from docs/explainability.md rename to docs/outputs/explainability.md index 970ad8c..794f0c8 100644 --- a/docs/explainability.md +++ b/docs/outputs/explainability.md @@ -2,7 +2,7 @@ Two related questions get two different metrics: -* "Which **regressor** is moving the forecast?" → [`_forecast_contribution_ratio`](regressors.md) +* "Which **regressor** is moving the forecast?" → [`_forecast_contribution_ratio`](../inputs/regressors.md) * "Which **model component** is driving the curve?" → the decomposition family below The decomposition projects the forecast onto `trend / seasonal / level / residual` so you can read “disk fills in 14 days because the trend steepened last week” directly from a stacked-area Grafana panel. diff --git a/docs/operations/maintenance-windows.md b/docs/outputs/maintenance-windows.md similarity index 100% rename from docs/operations/maintenance-windows.md rename to docs/outputs/maintenance-windows.md diff --git a/docs/preview.md b/docs/outputs/preview.md similarity index 100% rename from docs/preview.md rename to docs/outputs/preview.md diff --git a/docs/slo.md b/docs/outputs/slo.md similarity index 100% rename from docs/slo.md rename to docs/outputs/slo.md diff --git a/docs/cli.md b/docs/reference/cli.md similarity index 97% rename from docs/cli.md rename to docs/reference/cli.md index dde691b..ad09051 100644 --- a/docs/cli.md +++ b/docs/reference/cli.md @@ -101,7 +101,7 @@ promforecast diagnose duplicates \ Scans the configured TSDB for forecast metrics (`*_forecast` / `*_forecast_lower` / `*_forecast_upper`) that appear under more than one labelset family — the dual-source state described -in [the architecture page](architecture/emission-paths.md). Exit codes +in [the emission paths reference](emission-paths.md). Exit codes are CI-friendly: `0` for clean, `1` for duplicates detected, `2` for datasource unreachable. @@ -122,6 +122,6 @@ over the window exceeds `--max-samples`. Covers the same-labelset case writing to the **same** series produce a count higher than either path alone would. -See [operations/diagnose.md](operations/diagnose.md) for the full +See [`../operations/diagnose.md`](../operations/diagnose.md) for the full reference, authentication recipes, in-cluster invocation (`kubectl exec`), and CI-integration patterns. diff --git a/docs/config-schema.md b/docs/reference/config-schema.md similarity index 100% rename from docs/config-schema.md rename to docs/reference/config-schema.md diff --git a/docs/architecture/emission-paths.md b/docs/reference/emission-paths.md similarity index 100% rename from docs/architecture/emission-paths.md rename to docs/reference/emission-paths.md diff --git a/docs/models.md b/docs/reference/models.md similarity index 96% rename from docs/models.md rename to docs/reference/models.md index 5005369..b11c906 100644 --- a/docs/models.md +++ b/docs/reference/models.md @@ -42,7 +42,7 @@ External models can be added via the `promforecast.models` entry-point group (se | `0.95` | 0.95 | 7.62 | 13.45 | 18.99 | 19.00 | 19 | | `0.90` | 0.90 | 5.86 | 8.28 | 9.00 | 9.00 | 9 | -See [`capacity.md`](capacity.md) for recommended defaults and decision flow. +See [`../outputs/capacity.md`](../outputs/capacity.md) for recommended defaults and decision flow. ### When your series is sparse — intermittent-demand models @@ -87,7 +87,7 @@ Applies to both `/metrics` and `remote_write`. ## Cold-start seasonality borrowing -See [`cold-start.md`](cold-start.md). Short version: when history < `min_points`, set `cold_start.borrow_from.promql` to synthesise a `SeasonalNaive`-style forecast from peer siblings. Emits with `model="ColdStart"` and `cold_start="true"`. Quality score is capped (default 0.5). +See [`../inputs/cold-start.md`](../inputs/cold-start.md). Short version: when history < `min_points`, set `cold_start.borrow_from.promql` to synthesise a `SeasonalNaive`-style forecast from peer siblings. Emits with `model="ColdStart"` and `cold_start="true"`. Quality score is capped (default 0.5). ## Multi-horizon accuracy diff --git a/docs/regressors.md b/docs/regressors.md deleted file mode 100644 index cffe048..0000000 --- a/docs/regressors.md +++ /dev/null @@ -1,102 +0,0 @@ -# Exogenous regressors - -A regressor is an extra input fed to the forecasting model alongside the primary series. Three flavours are supported: - -- **`calendar`** — deterministic features derived from timestamps (`hour_of_day`, `day_of_week`, `is_weekend`). -- **`custom`** — any PromQL query whose result is aligned to the primary series. -- **`holidays`** — public-holiday calendar from the [`holidays`](https://pypi.org/project/holidays/) package (install with `pip install 'promforecast[holidays]'`). - -Regressors are declared per query under `regressors:`. Built-in models ignore them today; plug-in models (e.g. `promforecast-prophet`) receive them via the `regressors` argument to `fit_predict`. - -## Calendar regressors - -```yaml -queries: - - id: requests_per_second - promql: rate(http_requests_total[5m]) - regressors: - - id: hour_of_day - type: calendar - - id: day_of_week - type: calendar - - id: is_weekend - type: calendar -``` - -The `id` must be one of the three supported names — the schema rejects anything else at boot. - -## Custom (PromQL) regressors - -```yaml -regressors: - - id: deploys - type: custom - promql: changes(kube_deployment_status_observed_generation[1h]) - required: false # default -``` - -- The PromQL is executed once per primary query and aligned via nearest-asof join. -- `required: true` makes a failure fatal for the whole series. -- Failures are counted in `forecast_regressor_failures_total{group, query, regressor, reason}`. - -## Holiday regressors - -Requires the `holidays` extra. - -### Binary mode (default) - -```yaml -regressors: - - id: is_holiday - type: holidays - country: US # ISO-3166-1 alpha-2 - regions: [] # country-wide by default - timezone: America/New_York # optional, defaults to UTC - expand: false # default -``` - -Emits a single binary column `is_holiday` (1.0 on any holiday, 0.0 otherwise). - -### One-hot mode - -```yaml -regressors: - - id: holiday - type: holidays - country: US - regions: [CA, NY] # union of calendars - expand: true -``` - -Emits one column per observed holiday name (`holiday__thanksgiving`, `holiday__independence_day`, …). -If the lookback window contains no holidays, falls back to a single binary column so the feature schema stays stable. - -### Timezone & regions - -- `timezone` (IANA, e.g. `America/New_York`) converts UTC timestamps to local dates. Defaults to `UTC`. -- `regions` takes a list of subdivisions (states/provinces); the runner unions their calendars. - -Supported countries/regions are whatever the installed `holidays>=0.50` package provides (≥150 countries + most major subdivisions). - -## Failure handling - -| Failure type | Reason label | `required: true` behaviour | -|-------------------------------------|-------------------|-------------------------------------| -| Invalid calendar id | `invalid_type` | Series skipped | -| Custom PromQL error / timeout | `query_error` / `query_timeout` | Entire series skipped | -| Custom PromQL returns no data | `empty_result` | Entire series skipped | -| No datasource (dry-run / test) | `no_source` | Entire series skipped | -| `holidays` package missing | `invalid_type` | Series skipped | -| Unknown country / region | `invalid_type` | Series skipped | - -Optional regressors (default) only increment the failure counter; the forecast still emits without that feature. - -## Contribution metric - -When any regressor contributes signal, the runner emits a normalised explainability share: - -``` -_forecast_contribution_ratio{regressor, ...} # values in [0, 1], sum to 1 per series -``` - -Disable via `defaults.emission.contribution: false` (see `config-schema.md`). diff --git a/examples/alerts/promforecast-rules.yaml b/examples/alerts/promforecast-rules.yaml index 5348df1..bcfd15d 100644 --- a/examples/alerts/promforecast-rules.yaml +++ b/examples/alerts/promforecast-rules.yaml @@ -7,7 +7,7 @@ # source labels (``instance``, ``device``, etc.) — no # ``instance=":9091"`` and no ``job="promforecast"`` — # so the expressions below filter purely on source-side labels. See -# ``docs/architecture/emission-paths.md`` for the two-path picture and +# ``docs/reference/emission-paths.md`` for the two-path picture and # the scrape-drop snippet for an externally-managed Prometheus. apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule @@ -207,7 +207,7 @@ spec: # template — do not deploy as-is without tuning. The # ``_forecast`` snapshot value is the *end-of-horizon* # forecast (carrying ``horizon="next"``, which means "the next - # snapshot value to be served") — see ``docs/models.md`` for + # snapshot value to be served") — see ``docs/reference/models.md`` for # the convention. If ``defaults.horizon: 24h`` is the configured # forecast horizon then this alert reads "the disk is forecast # to be empty 24h from now". diff --git a/examples/production/README.md b/examples/production/README.md index b1a2e3a..43ebdb1 100644 --- a/examples/production/README.md +++ b/examples/production/README.md @@ -61,7 +61,7 @@ If you're an order of magnitude smaller, drop: * `resources.limits.cpu` to `"2"` and `memory` to `2Gi` If you're an order of magnitude larger, see -[`docs/ha.md`](../../docs/ha.md) for sharding guidance. +[`docs/operations/ha.md`](../../docs/operations/ha.md) for sharding guidance. ## A note on duplication diff --git a/forecaster/pyproject.toml b/forecaster/pyproject.toml index 68de4d8..7b50885 100644 --- a/forecaster/pyproject.toml +++ b/forecaster/pyproject.toml @@ -180,6 +180,9 @@ module = ["holidays", "holidays.*"] ignore_missing_imports = true [tool.pytest.ini_options] -addopts = "-ra -q --strict-markers --strict-config" +addopts = "-ra -q --strict-markers --strict-config -m 'not benchmark'" testpaths = ["tests"] asyncio_mode = "auto" +markers = [ + "benchmark: micro-benchmarks gated out of the default run; opt in with `-m benchmark`", +] diff --git a/forecaster/src/promforecast/config.py b/forecaster/src/promforecast/config.py index e474769..66fa4e1 100644 --- a/forecaster/src/promforecast/config.py +++ b/forecaster/src/promforecast/config.py @@ -36,7 +36,7 @@ # with a one-shot ``DeprecationWarning`` nudging operators to retag. Any # future breaking change will ship under a fresh ``apiVersion`` with both # the old and new schemas accepted during a documented deprecation window -# — see ``docs/config-schema.md`` for the migration policy. +# — see ``docs/reference/config-schema.md`` for the migration policy. CURRENT_API_VERSION = "promforecast.io/v1" DEPRECATED_API_VERSIONS: frozenset[str] = frozenset({"promforecast.io/v1alpha1"}) SUPPORTED_API_VERSIONS: frozenset[str] = frozenset({CURRENT_API_VERSION}) | DEPRECATED_API_VERSIONS @@ -1149,7 +1149,7 @@ def _check_api_version(raw: dict[str, Any], *, source: str) -> dict[str, Any]: * ``promforecast.io/v1`` -> the stable schema; loads silently. * ``promforecast.io/v1alpha1`` -> deprecated alias of v1 (the schemas are byte-identical). Loads with a one-shot ``DeprecationWarning`` - pointing at the migration recipe in ``docs/config-schema.md``. + pointing at the migration recipe in ``docs/reference/config-schema.md``. * Unknown ``apiVersion`` -> raise :class:`UnsupportedApiVersionError` with the offending value and the supported set so a typo fails clearly rather than bottoming out in a generic pydantic ``Literal`` @@ -1180,7 +1180,7 @@ def _check_api_version(raw: dict[str, Any], *, source: str) -> dict[str, Any]: f"config {source}: apiVersion {declared!r} is deprecated and will be " f"removed in a future major release; the v1alpha1 schema is identical " f"to {CURRENT_API_VERSION!r}, so changing the apiVersion field is the " - "only required edit. See docs/config-schema.md for the migration recipe.", + "only required edit. See docs/reference/config-schema.md for the migration recipe.", DeprecationWarning, stacklevel=3, ) diff --git a/forecaster/src/promforecast/diagnose.py b/forecaster/src/promforecast/diagnose.py index 30760bb..81c722d 100644 --- a/forecaster/src/promforecast/diagnose.py +++ b/forecaster/src/promforecast/diagnose.py @@ -3,7 +3,7 @@ Two subcommands today: * ``duplicates`` — scan for the dual-source forecast labelset state described - in ``docs/architecture/emission-paths.md``. Catches the case where the + in ``docs/reference/emission-paths.md``. Catches the case where the scraped and sink-pushed copies are *distinguishable* by labelset shape (different external_labels, ``honor_labels: false`` accidentally set, source signals missing a ``job`` label). @@ -63,7 +63,7 @@ # ``*_time_to_threshold_seconds`` derived ETA metrics) are *not* in scope — # they live on /metrics only and intentionally carry source-side labels # like ``job="node-exporter"`` per design principle 8. See the operational- -# metrics section of ``docs/architecture/emission-paths.md``. +# metrics section of ``docs/reference/emission-paths.md``. FORECAST_METRIC_REGEX = r".+_forecast(_lower|_upper)?" # Scrape-only marker labels. A labelset family carrying *any* of these is diff --git a/forecaster/src/promforecast/exporter.py b/forecaster/src/promforecast/exporter.py index 289cced..4b560c7 100644 --- a/forecaster/src/promforecast/exporter.py +++ b/forecaster/src/promforecast/exporter.py @@ -34,6 +34,7 @@ "BandCoveragePoint", "CalibrationOffsetPoint", "ComponentDecompositionPoint", + "ConditionalCalibrationOffsetPoint", "ConditionalDeviationPoint", "ConditionalForecastPoint", "ContributionPoint", @@ -323,6 +324,22 @@ class CalibrationOffsetPoint: offset: float +@dataclass(frozen=True) +class ConditionalCalibrationOffsetPoint: + """Conditional-band variant of :class:`CalibrationOffsetPoint`. + + Rendered as + ``forecast_conditional_band_calibration_offset{id, group, model, level}`` + so an operator alerting on ``forecast_conditional_deviation_outside_band`` + can see exactly how much widening the conditional band received from its + own calibration pass (not the unconditional pass). Emitted only when + both ``accuracy.conformal.enabled`` and ``emission.conditional`` are on. + """ + + labels: dict[str, str] + offset: float + + @dataclass class PreprocessStats: """Per-(group, query) preprocessing telemetry. @@ -437,6 +454,15 @@ class GroupSnapshot: # /metrics; series that disappear from the config disappear from the # exposition until they reappear. calibration_offsets: list[CalibrationOffsetPoint] = field(default_factory=list) + # Per-(series, model, level) conditional-band conformal offsets. + # Same shape as ``calibration_offsets`` but derived from a calibration + # fit whose regressors are held flat at their final pre-holdout value + # — matching the production-time conditional inference contract. The + # parallel family lets operators eyeball whether the conditional band's + # widening differs from the unconditional's at the same (model, level). + conditional_calibration_offsets: list[ConditionalCalibrationOffsetPoint] = field( + default_factory=list + ) # Per-(query) preprocessing statistics for the most recent run. # Counters (gaps_filled, outliers_replaced, change_points) are merged # with the previous snapshot so the exposed counter never decreases; @@ -539,6 +565,15 @@ def __init__(self) -> None: # their deployment; a flat-zero counter means every conditional # emission is paying the refit cost. self._conditional_aliased: dict[tuple[str, str, str], int] = {} + # Per-(group, query, reason) conditional-calibration failure counter. + # Bumped whenever the fresh per-fit conformal calibration pass for + # the conditional band fails (degenerate holdout, fit timeout, fit + # error) and the runner falls back to the inherited unconditional + # offsets. The fallback path is no worse than the pre-v1.6.5 + # behaviour, but operators want to know when fresh calibration + # *didn't* run — a sustained tick rate signals the conditional + # band is silently relying on inherited offsets. + self._conditional_calibration_failures: dict[tuple[str, str, str], int] = {} # Per-(group, query, source) season_length auto-detection counter. # ``source`` is bounded to {detected, snapped, fallback, too_short} # so cardinality is small and predictable. Useful for alerting on @@ -699,6 +734,25 @@ def increment_conditional_aliased(self, group: str, query: str, model: str) -> N key = (group, query, model) self._conditional_aliased[key] = self._conditional_aliased.get(key, 0) + 1 + def increment_conditional_calibration_failure( + self, + group: str, + query: str, + reason: str, + ) -> None: + """Bump the per-(group, query, reason) conditional-calibration failure counter. + + ``reason`` is one of ``degenerate_holdout``, ``fit_timeout``, + ``fit_error`` — the bounded set produced by + :meth:`Runner._compute_conditional_conformal_offsets`. Any other + token would inflate cardinality; callers must use the bounded set. + """ + with self._lock: + key = (group, query, reason) + self._conditional_calibration_failures[key] = ( + self._conditional_calibration_failures.get(key, 0) + 1 + ) + def increment_decomposition_skipped( self, group: str, @@ -791,6 +845,12 @@ def _conditional_aliased_state(self) -> dict[tuple[str, str, str], int]: with self._lock: return dict(self._conditional_aliased) + def _conditional_calibration_failures_state( + self, + ) -> dict[tuple[str, str, str], int]: + with self._lock: + return dict(self._conditional_calibration_failures) + def _season_length_state(self) -> dict[tuple[str, str, str], int]: with self._lock: return dict(self._season_length_detections) @@ -815,6 +875,9 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: season_length_state = self._exporter._season_length_state() decomposition_skipped_state = self._exporter._decomposition_skipped_state() conditional_aliased_state = self._exporter._conditional_aliased_state() + conditional_calibration_failures_state = ( + self._exporter._conditional_calibration_failures_state() + ) yield from _forecast_families(groups) yield from _conditional_forecast_families(groups) yield from _conditional_deviation_families(groups) @@ -831,6 +894,10 @@ def collect(self) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: yield from _recommended_window_families(groups) yield from _growth_rate_families(groups) yield from _calibration_families(groups) + yield from _conditional_calibration_families(groups) + yield from _conditional_calibration_failures_families( + conditional_calibration_failures_state + ) yield from _preprocess_families(groups) yield from _emission_clipped_families(groups) yield from _cold_start_siblings_families(groups) @@ -1687,6 +1754,58 @@ def _calibration_families( ) +def _conditional_calibration_families( + groups: list[GroupSnapshot], +) -> Iterator[GaugeMetricFamily]: + """Render ``forecast_conditional_band_calibration_offset`` gauges. + + Mirrors :func:`_calibration_families` but for the conditional emission + family. Emitted only when at least one series produced a non-empty + conditional offset list — a deployment with no query opting into both + ``accuracy.conformal.enabled`` and ``emission.conditional`` produces + no samples and the family is omitted entirely. + """ + points = [p for snap in groups for p in snap.conditional_calibration_offsets] + yield from _emit_gauge_family( + points, + "forecast_conditional_band_calibration_offset", + "Additive correction applied to the conditional forecast band by " + "split-conformal calibration on a holdout where regressors are held " + "flat at their final pre-holdout value. Positive values widen each " + "side of the conditional band; 0 means the parametric band already " + "empirically covered its nominal level under conditional inference.", + value_fn=lambda p: p.offset, + skip_fn=lambda p: not math.isfinite(p.offset), + ) + + +def _conditional_calibration_failures_families( + state: dict[tuple[str, str, str], int], +) -> Iterator[CounterMetricFamily]: + """Render ``forecast_conditional_calibration_failures_total{group, query, reason}``. + + Bumped every time the runner falls back from a fresh per-fit + conditional-band calibration to the inherited unconditional offsets. + ``reason`` is bounded to the set produced by + :meth:`Runner._compute_conditional_conformal_offsets` + (``degenerate_holdout``, ``fit_timeout``, ``fit_error``). + """ + if not state: + return + counter = CounterMetricFamily( + "forecast_conditional_calibration_failures", + "Number of times the conditional-band conformal calibration fit " + "could not produce fresh offsets and the runner fell back to the " + "inherited unconditional offsets. The fallback path is no worse " + "than the pre-v1.6.5 behaviour, but a sustained tick rate means " + "fresh calibration is silently not running.", + labels=["group", "query", "reason"], + ) + for (group, query, reason), count in state.items(): + counter.add_metric([group, query, reason], float(count)) + yield counter + + def _preprocess_families( groups: list[GroupSnapshot], ) -> Iterator[GaugeMetricFamily | CounterMetricFamily]: diff --git a/forecaster/src/promforecast/main.py b/forecaster/src/promforecast/main.py index e7ae778..ff7d391 100644 --- a/forecaster/src/promforecast/main.py +++ b/forecaster/src/promforecast/main.py @@ -672,7 +672,7 @@ async def preview_endpoint(req: Request) -> Response: Stateless: nothing about this request touches ``/metrics``, the snapshot, or scheduled fits. Same network-level - access-control model as ``/-/reload``. See ``docs/preview.md`` + access-control model as ``/-/reload``. See ``docs/outputs/preview.md`` for the request schema + worked examples. """ try: diff --git a/forecaster/src/promforecast/models/damping.py b/forecaster/src/promforecast/models/damping.py index a69a59d..9364969 100644 --- a/forecaster/src/promforecast/models/damping.py +++ b/forecaster/src/promforecast/models/damping.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: import pandas as pd -# 0.98 is the capacity-planning rule of thumb cited in ``docs/models.md``: +# 0.98 is the capacity-planning rule of thumb cited in ``docs/reference/models.md``: # at horizon=288 (24h at 5m step) it caps the extrapolated trend at # roughly the asymptote ``phi / (1 - phi) = 49`` trend units — well # under the undamped 288 — while leaving short-horizon forecasts within diff --git a/forecaster/src/promforecast/point_factory.py b/forecaster/src/promforecast/point_factory.py index 8ab3e17..7b0961c 100644 --- a/forecaster/src/promforecast/point_factory.py +++ b/forecaster/src/promforecast/point_factory.py @@ -16,7 +16,7 @@ import math from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from . import deviation as deviation_mod from . import quality as quality_mod @@ -26,6 +26,7 @@ AccuracyPoint, CalibrationOffsetPoint, ComponentDecompositionPoint, + ConditionalCalibrationOffsetPoint, ConditionalDeviationPoint, ConditionalForecastPoint, ContributionPoint, @@ -192,12 +193,23 @@ def conditional_forecast_points( forecast: pd.DataFrame, model_name: str, selection: SelectionResult | None, + *, + calibrated: bool = False, ) -> Iterable[ConditionalForecastPoint]: """Mirror of :meth:`forecast_points` for the conditional family. Same end-of-horizon convention — one snapshot value per (series, model, level). The metric names carry the ``_forecast_conditional`` suffix family instead of ``_forecast``. + + ``calibrated`` is True when the conditional band that + ``forecast`` carries was widened by a fresh per-fit conformal + calibration pass; in that case the band rows pick up a + ``calibrated="true"`` label matching the unconditional family's + post-hoc labelling convention so dashboards can distinguish + widened vs. raw conditional bands within a single emission + family. The yhat row never carries the label (calibration is a + band adjustment, not a point-forecast adjustment). """ if forecast.empty: return @@ -208,19 +220,20 @@ def conditional_forecast_points( labels=common, value=float(last["yhat"]), ) + band_extra = {"calibrated": "true"} if calibrated else {} for level in self._levels: lo_col = f"yhat_lower_{level}" hi_col = f"yhat_upper_{level}" if lo_col in forecast.columns: yield ConditionalForecastPoint( metric=f"{self._metric_id}_forecast_conditional_lower", - labels={**common, "level": str(level)}, + labels={**common, "level": str(level), **band_extra}, value=float(last[lo_col]), ) if hi_col in forecast.columns: yield ConditionalForecastPoint( metric=f"{self._metric_id}_forecast_conditional_upper", - labels={**common, "level": str(level)}, + labels={**common, "level": str(level), **band_extra}, value=float(last[hi_col]), ) @@ -229,6 +242,8 @@ def conditional_forecast_curve_points( forecast: pd.DataFrame, model_name: str, selection: SelectionResult | None, + *, + calibrated: bool = False, ) -> Iterable[ForecastCurvePoint]: """Yield every horizon step as a push point for the conditional sink curve. @@ -237,11 +252,24 @@ def conditional_forecast_curve_points( the conditional curves in a parallel TSDB family. Caller-side renaming is no longer needed once this method is used. + ``calibrated`` is True when the conditional band rows were + widened by a fresh per-fit conformal pass; the resulting band + curve points pick up a ``calibrated="true"`` label so the + snapshot and the sink push agree on labelling (the unconditional + path applies the same label post-hoc via + :func:`~promforecast.runner._apply_offsets_to_curve_points`). + Rows with non-finite timestamps or values are dropped — TSDBs reject NaN samples, and emitting them would just generate noise in the sink's failure counter. """ - yield from self._curve_points(forecast, model_name, selection, suffix_family="_conditional") + yield from self._curve_points( + forecast, + model_name, + selection, + suffix_family="_conditional", + calibrated_band=calibrated, + ) def conditional_deviation_points( self, @@ -307,26 +335,39 @@ def component_curve_points( reject NaN samples and emitting them just generates noise in the sink's failure counter. """ - if forecast.empty or not components: + if forecast.empty or not components or "ds" not in forecast.columns: return + import pandas as pd # noqa: PLC0415 + rows = len(forecast) common = self._build_common_labels(model_name, selection) metric_name = f"{self._metric_id}_forecast_component" + # Vectorise the ``ds`` -> ms conversion up-front so the inner loop + # only does integer indexing. Pre-filter the components dict to + # the ones whose value list matches the frame length, so we don't + # repeat the same length check for every row. + ds_normalised = pd.to_datetime(forecast["ds"], errors="coerce") + valid_ts_mask = ds_normalised.notna().to_numpy() + ts_ms_arr = ds_normalised.astype("int64").to_numpy() // 1_000_000 + component_labels: dict[str, dict[str, str]] = { + name: {**common, "component": name} + for name, values in components.items() + if len(values) == rows + } + if not component_labels: + return + usable_components = [(name, components[name]) for name in component_labels] for i in range(rows): - try: - ts = forecast["ds"].iloc[i] - ts_ms = int(float(ts.timestamp()) * 1000) - except (KeyError, AttributeError, TypeError, ValueError): + if not valid_ts_mask[i]: continue - for component_name, values in components.items(): - if len(values) != rows: - continue + ts_ms = int(ts_ms_arr[i]) + for component_name, values in usable_components: raw = values[i] if not math.isfinite(raw): continue yield ForecastCurvePoint( metric=metric_name, - labels={**common, "component": component_name}, + labels=component_labels[component_name], value=float(raw), timestamp_ms=ts_ms, ) @@ -398,6 +439,30 @@ def calibration_offset_points( offset=float(offset), ) + def conditional_calibration_offset_points( + self, + model_name: str, + offsets: dict[int, float], + ) -> Iterable[ConditionalCalibrationOffsetPoint]: + """One ``forecast_conditional_band_calibration_offset`` per (model, level). + + Mirrors :meth:`calibration_offset_points` but emits under the + conditional family so an operator alerting on + ``forecast_conditional_deviation_outside_band`` can read the + per-(model, level) widening that was actually applied to that + band. Emitted only when both ``accuracy.conformal.enabled`` and + ``emission.conditional`` are on and the fresh calibration fit + succeeded. + """ + common = self._common_with_id() + for level, offset in offsets.items(): + if not math.isfinite(offset): + continue + yield ConditionalCalibrationOffsetPoint( + labels={**common, "model": model_name, "level": str(level)}, + offset=float(offset), + ) + def forecast_curve_points( self, forecast: pd.DataFrame, @@ -423,6 +488,7 @@ def _curve_points( selection: SelectionResult | None, *, suffix_family: str, + calibrated_band: bool = False, ) -> Iterable[ForecastCurvePoint]: """Internal helper shared by every per-step curve emission method. @@ -433,51 +499,79 @@ def _curve_points( family. Keeping the rendering logic in one place stops the two public callers from drifting on band-handling or NaN-skip semantics as new options arrive. + + ``calibrated_band`` adds the ``calibrated="true"`` label to + band rows (lower/upper only — never to the yhat row, which is + not a band quantity). The unconditional curve path applies the + same label post-hoc via + :func:`~promforecast.runner._apply_offsets_to_curve_points`; the + conditional path applies the offsets to the DataFrame *before* + invoking the factory, so the in-factory flag is the symmetric + place to attach the label. """ - if forecast.empty: + if forecast.empty or "ds" not in forecast.columns: return + import pandas as pd # noqa: PLC0415 + common = self._build_common_labels(model_name, selection) base_metric = f"{self._metric_id}_forecast{suffix_family}" lower_metric = f"{self._metric_id}_forecast{suffix_family}_lower" upper_metric = f"{self._metric_id}_forecast{suffix_family}_upper" - for _, row in forecast.iterrows(): - ts = row.get("ds") - if ts is None: - continue - try: - # ``Timestamp.timestamp()`` returns seconds; we want ms. - ts_ms = int(float(ts.timestamp()) * 1000) - except (AttributeError, TypeError, ValueError): - continue - yhat_raw = row.get("yhat") - if yhat_raw is not None and math.isfinite(float(yhat_raw)): - yield ForecastCurvePoint( - metric=base_metric, - labels=common, - value=float(yhat_raw), - timestamp_ms=ts_ms, + band_extra = {"calibrated": "true"} if calibrated_band else {} + # Column-oriented iteration: ``iterrows`` materialises one Series + # per row (~10x slower than indexing numpy arrays). For a typical + # 288-step horizon with two confidence levels the helper yields + # ~1440 points per series-model; the conditional emission family + # added in v1.6.5 doubles that, so the per-step assembly cost is + # worth keeping out of Python's Series-construction path. + ds_normalised = pd.to_datetime(forecast["ds"], errors="coerce") + ds_ns = ds_normalised.astype("int64").to_numpy() + # ``NaT`` becomes the int64 sentinel ``-9223372036854775808``; mask + # those rows so we don't emit a bogus 1677-era timestamp. + valid_ts_mask = ds_normalised.notna().to_numpy() + ts_ms_arr = ds_ns // 1_000_000 + yhat_arr = forecast["yhat"].to_numpy(dtype=float) if "yhat" in forecast.columns else None + # Pre-resolve band columns once. Each entry carries the metric + # name, the (immutable-by-convention) labels dict to attach to + # every row of this band, and the value array as float64. + band_cols: list[tuple[str, dict[str, str], Any]] = [] + for level in self._levels: + level_str = str(level) + lower_labels = {**common, "level": level_str, **band_extra} + upper_labels = {**common, "level": level_str, **band_extra} + lo_col = f"yhat_lower_{level}" + if lo_col in forecast.columns: + band_cols.append( + (lower_metric, lower_labels, forecast[lo_col].to_numpy(dtype=float)) ) - for level in self._levels: - lo_col = f"yhat_lower_{level}" - hi_col = f"yhat_upper_{level}" - if lo_col in forecast.columns: - lo = row[lo_col] - if math.isfinite(float(lo)): - yield ForecastCurvePoint( - metric=lower_metric, - labels={**common, "level": str(level)}, - value=float(lo), - timestamp_ms=ts_ms, - ) - if hi_col in forecast.columns: - hi = row[hi_col] - if math.isfinite(float(hi)): - yield ForecastCurvePoint( - metric=upper_metric, - labels={**common, "level": str(level)}, - value=float(hi), - timestamp_ms=ts_ms, - ) + hi_col = f"yhat_upper_{level}" + if hi_col in forecast.columns: + band_cols.append( + (upper_metric, upper_labels, forecast[hi_col].to_numpy(dtype=float)) + ) + n = len(forecast) + for i in range(n): + if not valid_ts_mask[i]: + continue + ts_ms = int(ts_ms_arr[i]) + if yhat_arr is not None: + v = yhat_arr[i] + if math.isfinite(v): + yield ForecastCurvePoint( + metric=base_metric, + labels=common, + value=float(v), + timestamp_ms=ts_ms, + ) + for metric_name, band_labels, arr in band_cols: + v = arr[i] + if math.isfinite(v): + yield ForecastCurvePoint( + metric=metric_name, + labels=band_labels, + value=float(v), + timestamp_ms=ts_ms, + ) def ensemble_weight_points( self, diff --git a/forecaster/src/promforecast/runner/__init__.py b/forecaster/src/promforecast/runner/__init__.py new file mode 100644 index 0000000..4e68f0b --- /dev/null +++ b/forecaster/src/promforecast/runner/__init__.py @@ -0,0 +1,140 @@ +"""Per-group execution and scheduling. + +This package replaces what used to be a single 3700+-line ``runner.py``. +The split keeps the public surface — ``Runner``, ``ReloadResult``, and a +handful of test-private helpers — at the package root so existing +``from promforecast.runner import ...`` imports continue to work +unchanged. + +Layout: + +* :mod:`._scheduler` — the :class:`Runner` orchestration class. +* :mod:`._state` — per-fit / per-group state dataclasses, the + fit-into-ctx merge helper, and the registry-driven inheritance loop + for queries skipped on a tick. +* :mod:`._helpers` — pure helpers that don't depend on a live + ``Runner`` (suffix lookup, backtest summarisation, etc.). +* :mod:`._budgets` — series-budget allocation and per-query overflow / + canary sampling. +* :mod:`.emission.calibration` — post-fit pure helpers that widen + forecast bands by per-(model, level) conformal offsets. +* :mod:`.emission.safeguards` — count-aware emission safeguards + (clip-from-zero, round-to-integer). +* :mod:`.emission.conditional` — pure helpers for conditional forecast + emission. + +Architectural intent: per-family code that does *not* need a live +``Runner`` lives next to other code about the same family; the +``Runner`` class stays in one place because its methods share an +executor, a config, and an exporter, and splitting them across modules +would just trade one long file for a tangle of cross-module references. + +Tests monkey-patch ``promforecast.runner.SLOW_FIT_RATIO`` and +``promforecast.runner._summarise_backtest`` directly on this package's +attributes; the scheduler reads both via ``import promforecast.runner +as _pkg`` and ``_pkg.`` so the patched value is picked up on the +next call. Do not bind those names locally inside the scheduler — that +would silently break the monkey-patch contract. +""" + +from __future__ import annotations + +# A fit is considered "slow" once its wall-clock crosses this fraction of +# ``safety.fit_timeout``. We pick 0.5 (half) so the warning fires comfortably +# before the timeout actually kills the fit - long enough that operators +# can observe a degraded model before alerts go red, short enough that +# noise-induced 5-10% wobble doesn't trip it. +# +# Defined here (rather than in :mod:`._scheduler`) so test monkey-patches +# of ``promforecast.runner.SLOW_FIT_RATIO`` land on the same attribute the +# scheduler reads via ``_pkg.SLOW_FIT_RATIO``. +SLOW_FIT_RATIO: float = 0.5 + +# ---- Re-exports ----------------------------------------------------------- +# Order matters: ``_scheduler`` does ``import promforecast.runner as _pkg`` +# so the package must already exist in ``sys.modules`` (it does, because +# we are mid-``__init__``) and ``SLOW_FIT_RATIO`` above must be set +# before any Runner method runs. The scheduler import is deliberately +# *last* so the package's other re-exports are already in place when the +# scheduler module body executes. + +from ._budgets import _apply_canary, _apply_overflow, _compute_global_budgets # noqa: E402 +from ._helpers import ( # noqa: E402 + _FORECAST_METRIC_SUFFIXES, + _combine_ensemble, + _format_band_window, + _pandas_freq, + _query_id_from_metric, + _resolve_spec, + _run_backtest, + _run_fit, + _series_key_from_labels, + _summarise_backtest, +) +from ._scheduler import Runner # noqa: E402 +from ._state import ( # noqa: E402 + _INHERITANCE_REGISTRY, + ReloadResult, + _BacktestNeeds, + _CapacityInput, + _FitResult, + _inherit_prior_for_skipped_queries, + _InheritanceSpec, + _merge_fit_into_ctx, + _RunContext, +) +from .emission.calibration import ( # noqa: E402 + _apply_offsets_to_curve_points, + _apply_offsets_to_snapshot_points, + _apply_offsets_to_threshold_eta_inputs, + _calibrated_band_value, +) +from .emission.conditional import _flatten_regressors_to_last_value # noqa: E402 +from .emission.safeguards import ( # noqa: E402 + _ROUND_COUNTER_TRIGGER, + _apply_safeguards_to_curve_points, + _apply_safeguards_to_forecast_frame, + _apply_safeguards_to_snapshot_points, + _classify_emission_correction, +) + +# ``__all__`` covers the test-private helpers as well as the public surface +# so re-exports aren't flagged unused; tests reach into these via +# ``from promforecast.runner import _foo`` patterns. Adding a private here +# is a deliberate choice to keep the test contract stable. +__all__ = [ + "SLOW_FIT_RATIO", + "_FORECAST_METRIC_SUFFIXES", + "_INHERITANCE_REGISTRY", + "_ROUND_COUNTER_TRIGGER", + "ReloadResult", + "Runner", + "_BacktestNeeds", + "_CapacityInput", + "_FitResult", + "_InheritanceSpec", + "_RunContext", + "_apply_canary", + "_apply_offsets_to_curve_points", + "_apply_offsets_to_snapshot_points", + "_apply_offsets_to_threshold_eta_inputs", + "_apply_overflow", + "_apply_safeguards_to_curve_points", + "_apply_safeguards_to_forecast_frame", + "_apply_safeguards_to_snapshot_points", + "_calibrated_band_value", + "_classify_emission_correction", + "_combine_ensemble", + "_compute_global_budgets", + "_flatten_regressors_to_last_value", + "_format_band_window", + "_inherit_prior_for_skipped_queries", + "_merge_fit_into_ctx", + "_pandas_freq", + "_query_id_from_metric", + "_resolve_spec", + "_run_backtest", + "_run_fit", + "_series_key_from_labels", + "_summarise_backtest", +] diff --git a/forecaster/src/promforecast/runner/_budgets.py b/forecaster/src/promforecast/runner/_budgets.py new file mode 100644 index 0000000..87e9381 --- /dev/null +++ b/forecaster/src/promforecast/runner/_budgets.py @@ -0,0 +1,83 @@ +"""Series-budget allocation and per-query overflow / canary sampling. + +Pulled out of the runner monolith because none of these helpers touch +runner instance state; they're pure data shaping. The runner reaches +for them once per group run to decide which series survive the +per-group / total / canary caps. +""" + +from __future__ import annotations + +import math +import random +from typing import Literal + +from ..config import CanaryConfig, GroupConfig +from ..source import SeriesFrame + +OverflowStrategy = Literal["drop_lowest_priority", "reject", "sample"] + + +def _compute_global_budgets(groups: list[GroupConfig], max_total_series: int) -> dict[str, int]: + """Static per-group share of ``max_total_series``, weighted by priority. + + A static split avoids cross-group coordination at runtime: groups run on + independent timers and may overlap. With weights ``priority/sum(priorities)`` + the lowest-priority groups give up budget first, which matches the intent + of ``series_overflow: drop_lowest_priority``. + """ + if max_total_series <= 0 or not groups: + return {g.name: 0 for g in groups} + weights = {g.name: max(1, g.priority) for g in groups} + total_weight = sum(weights.values()) + return { + name: max(0, (max_total_series * weight) // total_weight) + for name, weight in weights.items() + } + + +def _apply_overflow( + frames: list[SeriesFrame], cap: int, strategy: OverflowStrategy +) -> tuple[list[SeriesFrame], int]: + """Reduce ``frames`` to fit ``cap`` according to ``strategy``. + + Returns ``(allowed, dropped_count)``. With ``cap <= 0`` everything is + dropped; ``reject`` drops the whole query rather than half-emitting it. + """ + if cap <= 0: + return [], len(frames) + if len(frames) <= cap: + return frames, 0 + if strategy == "reject": + return [], len(frames) + if strategy == "sample": + return random.sample(frames, cap), len(frames) - cap + return frames[:cap], len(frames) - cap + + +def _apply_canary( + frames: list[SeriesFrame], + canary: CanaryConfig, +) -> tuple[list[SeriesFrame], int]: + """Random-sample ``ceil(weight * N)`` frames when canary mode is on. + + Returns ``(kept, dropped)``. ``ceil`` (not ``floor``) so a tiny + population doesn't round down to zero — ``weight=0.1`` on a 5-series + query still fits one series, which is the useful canary case. The + caller emits the drop count as + ``forecast_series_dropped_total{reason="canary"}`` so the sampling + is observable from /metrics. + + Sampling uses :func:`random.sample` (no fixed seed) so successive runs + rotate through the population. A fixed seed would bias the canary to + the same N series forever and defeat coverage over time. + """ + if not canary.enabled or not frames: + return frames, 0 + if canary.weight >= 1.0: + return frames, 0 + keep = max(1, math.ceil(len(frames) * canary.weight)) + if keep >= len(frames): + return frames, 0 + sampled = random.sample(frames, keep) + return sampled, len(frames) - keep diff --git a/forecaster/src/promforecast/runner/_helpers.py b/forecaster/src/promforecast/runner/_helpers.py new file mode 100644 index 0000000..1ad6fef --- /dev/null +++ b/forecaster/src/promforecast/runner/_helpers.py @@ -0,0 +1,247 @@ +"""Misc pure helpers extracted from the runner monolith. + +Each function here either reshapes a pandas frame, formats a label +value, or wraps an upstream helper to keep the runner's call-sites +short. Nothing here depends on a live :class:`~promforecast.runner.Runner` +instance. +""" + +from __future__ import annotations + +import functools +from datetime import timedelta +from typing import TYPE_CHECKING, Any + +from .. import point_factory as point_factory_mod +from ..accuracy import backtest +from ..config import ModelSpec +from ..point_factory import BandSummary + +if TYPE_CHECKING: + import numpy as np + import pandas as pd + + from ..models import ForecastModel + + +def _series_key_from_labels(labels: dict[str, str]) -> str: + """Stable per-series key derived from the label set. + + Used to address per-series state in the drift cache and band-coverage + tracker. Skips ``__name__`` (Prometheus's reserved metric-name + pseudo-label) so two queries that return the same labelset on a + different metric share state appropriately — both forecast the same + underlying signal. + + The key is ``"k1=v1,k2=v2"`` with sorted keys so equal label + dictionaries produce the same key regardless of iteration order. + """ + items = sorted((k, v) for k, v in labels.items() if k != "__name__") + return ",".join(f"{k}={v}" for k, v in items) + + +def _format_band_window(window: timedelta) -> str: + """Render a ``timedelta`` window as a Prometheus-style label.""" + return point_factory_mod.format_window(window.total_seconds()) + + +# Suffixes that the runner / point_factory emits on forecast-derived +# metrics. ``_query_id_from_metric`` matches the longest one first so an +# id like ``api_forecast_count`` (a legal Prometheus identifier) is not +# mis-routed by an internal ``_forecast`` substring. +_FORECAST_METRIC_SUFFIXES: tuple[str, ...] = ( + "_forecast_contribution_ratio", + "_forecast_time_to_threshold_seconds", + "_forecast_time_to_drain_seconds", + "_forecast_error_budget_remaining_seconds", + "_forecast_error_budget_exhausted_in_seconds", + "_forecast_error_budget_burn_rate", + "_forecast_growth_rate", + "_forecast_will_breach", + "_forecast_conditional_lower", + "_forecast_conditional_upper", + "_forecast_conditional", + "_forecast_component", + "_forecast_lower", + "_forecast_upper", + "_forecast", +) + + +# Suffixes indexed by length (longest first). Each ``length`` maps to a +# ``frozenset`` so the longest-suffix probe in ``_query_id_from_metric`` is an +# O(1) hash lookup per distinct length rather than an O(N_suffixes) linear +# scan. With today's 15-suffix set there are 13 distinct lengths, so a +# no-match call costs at most 13 hash lookups instead of 15 ``endswith`` +# scans — but the realistic call shape from +# ``_inherit_prior_for_skipped_queries`` calls the helper thousands of times +# per refresh on a tiny set of distinct metric names, so the +# ``functools.lru_cache`` wrapper below collapses the repeat-call cost to a +# single cache hit. Cache size is bounded by ``N_queries x N_suffixes`` +# (≤ 300 in any realistic config), well under the ``maxsize`` cap. +_SUFFIXES_BY_LENGTH: dict[int, frozenset[str]] = {} +for _suffix in _FORECAST_METRIC_SUFFIXES: + _SUFFIXES_BY_LENGTH[len(_suffix)] = _SUFFIXES_BY_LENGTH.get(len(_suffix), frozenset()) | { + _suffix + } +_SUFFIX_LENGTHS_DESC: tuple[int, ...] = tuple(sorted(_SUFFIXES_BY_LENGTH.keys(), reverse=True)) + + +@functools.lru_cache(maxsize=2048) +def _query_id_from_metric(metric_name: str) -> str: + """Pull the query ``id`` back out of a forecast metric name. + + Matches the known emission suffix set rather than ``rfind("_forecast")`` + so a query id that legitimately contains the word ``forecast`` (e.g. + ``api_forecast_count``) round-trips correctly. Lookup is bounded by the + number of distinct suffix lengths (longest-first probe of a + length-indexed ``frozenset``) and memoised so the hot path through + ``_inherit_prior_for_skipped_queries`` pays one hash lookup per repeat + call. + """ + name_len = len(metric_name) + for length in _SUFFIX_LENGTHS_DESC: + if name_len <= length: + continue + if metric_name[-length:] in _SUFFIXES_BY_LENGTH[length]: + return metric_name[:-length] + return "" + + +def _resolve_spec(specs: list[ModelSpec], name: str) -> ModelSpec: + """Look up a configured ``ModelSpec`` by name; defaults to params=``{}``. + + The fallback model is allowed to live outside ``defaults.models`` (a + SeasonalNaive safety net for an AutoARIMA-only setup is the canonical + case). We synthesise an empty-params spec so the fallback runs with + stock model defaults rather than failing the lookup. + """ + for spec in specs: + if spec.name == name: + return spec + return ModelSpec(name=name) + + +def _combine_ensemble( + component_forecasts: dict[str, pd.DataFrame], + weights: dict[str, float], + levels: list[int], +) -> pd.DataFrame: + """Build a single forecast frame from inverse-MAPE-weighted components. + + Aligns on ``unique_id`` + ``ds``. Each numeric column is the weighted + sum across components present at that row; missing columns (a model + that didn't emit a level we requested) contribute weight 0 so the + ensemble doesn't quietly inherit one component's missing band. + """ + import pandas as pd # noqa: PLC0415 + + base_cols = ["unique_id", "ds"] + value_cols = ["yhat"] + [ + f"yhat_{kind}_{level}" for kind in ("lower", "upper") for level in levels + ] + + import numpy as np # noqa: PLC0415 + + accumulator: pd.DataFrame | None = None + weight_accumulator: dict[str, np.ndarray[Any, Any]] | None = None + for name, weight in weights.items(): + df = component_forecasts.get(name) + if df is None or df.empty: + continue + if accumulator is None or weight_accumulator is None: + accumulator = df[base_cols].reset_index(drop=True).copy() + weight_accumulator = {col: np.zeros(len(df), dtype=float) for col in value_cols} + for col in value_cols: + accumulator[col] = 0.0 + for col in value_cols: + if col not in df.columns: + continue + # Switch to numpy arrays for the accumulation: pandas would + # try to align on the (matching) index every iteration, which + # both wastes work and confuses pandas-stubs about the + # resulting dtype. + current = accumulator[col].to_numpy(dtype=float) + accumulator[col] = current + df[col].to_numpy(dtype=float) * weight + weight_accumulator[col] = weight_accumulator[col] + weight + if accumulator is None or weight_accumulator is None: + # No usable components — return an empty frame the caller can detect. + return pd.DataFrame(columns=[*base_cols, *value_cols]) + # Convert weight sums back to plain Series for the divide-by-weight step; + # rows where total weight is 0 retain NaN rather than dividing by zero. + for col in value_cols: + denom = pd.Series(weight_accumulator[col]) + accumulator[col] = accumulator[col] / denom.where(denom > 0, other=float("nan")) + return accumulator + + +def _run_backtest( + model: ForecastModel, + df: pd.DataFrame, + horizon_steps: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, +) -> tuple[pd.DataFrame, pd.DataFrame] | None: + return backtest( + model=model, + df=df, + horizon_steps=horizon_steps, + freq=freq, + levels=levels, + regressors=regressors, + ) + + +def _pandas_freq(step: timedelta) -> str: + seconds = int(step.total_seconds()) + if seconds % 3600 == 0: + return f"{seconds // 3600}h" + if seconds % 60 == 0: + return f"{seconds // 60}min" + return f"{seconds}s" + + +def _summarise_backtest( + predictions: pd.DataFrame, + test: pd.DataFrame, + levels: list[int], +) -> tuple[float, float, dict[int, BandSummary]] | None: + """Extract the trailing (actual, yhat, per-level band) tuple, or None. + + Returns ``None`` if either frame is empty or missing the ``y``/``yhat`` + columns; non-finite cells are tolerated and surface downstream as NaN. + """ + if predictions.empty or test.empty: + return None + try: + last_actual = float(test["y"].iloc[-1]) + last_yhat = float(predictions["yhat"].iloc[-1]) + except (KeyError, IndexError, ValueError): + return None + bands: dict[int, BandSummary] = {} + for level in levels: + lo_col = f"yhat_lower_{level}" + hi_col = f"yhat_upper_{level}" + if lo_col not in predictions.columns or hi_col not in predictions.columns: + continue + try: + lower = float(predictions[lo_col].iloc[-1]) + upper = float(predictions[hi_col].iloc[-1]) + except (ValueError, IndexError): + continue + bands[level] = BandSummary(lower=lower, upper=upper) + return last_actual, last_yhat, bands + + +def _run_fit( + model: ForecastModel, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, +) -> pd.DataFrame: + return model.fit_predict( + df=df, horizon=horizon, freq=freq, levels=levels, regressors=regressors + ) diff --git a/forecaster/src/promforecast/runner.py b/forecaster/src/promforecast/runner/_scheduler.py similarity index 71% rename from forecaster/src/promforecast/runner.py rename to forecaster/src/promforecast/runner/_scheduler.py index c2b3980..6da6fea 100644 --- a/forecaster/src/promforecast/runner.py +++ b/forecaster/src/promforecast/runner/_scheduler.py @@ -4,6 +4,11 @@ for ``server.refresh_interval``. A per-group lock prevents a slow run from being triggered twice concurrently. Failure isolation is layered: per query, per series, per regressor. + +This module owns the :class:`Runner` orchestration class. The dataclasses, +pure helpers, per-family emission post-processing, and budget allocation +all live in sibling modules under ``runner/``; this file is intentionally +narrow to the scheduler + per-fit pipeline. """ from __future__ import annotations @@ -11,45 +16,40 @@ import asyncio import contextlib import math -import random import time from collections.abc import Awaitable, Callable from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING import structlog -from . import band_coverage as band_coverage_mod -from . import capacity as capacity_mod -from . import cold_start as cold_start_mod -from . import conformal as conformal_mod -from . import contribution as contribution_mod -from . import decomposition as decomposition_mod -from . import deviation as deviation_mod -from . import discovery as discovery_mod -from . import drift as drift_mod -from . import maintenance as maintenance_mod -from . import models as model_registry -from . import point_factory as point_factory_mod -from . import preprocess as preprocess_mod -from . import quality as quality_mod -from . import regressors as regressors_mod -from . import seasonality as seasonality_mod -from . import slo as slo_mod -from . import telemetry as telemetry_mod -from .accuracy import ( +from .. import band_coverage as band_coverage_mod +from .. import capacity as capacity_mod +from .. import cold_start as cold_start_mod +from .. import conformal as conformal_mod +from .. import contribution as contribution_mod +from .. import decomposition as decomposition_mod +from .. import deviation as deviation_mod +from .. import discovery as discovery_mod +from .. import drift as drift_mod +from .. import maintenance as maintenance_mod +from .. import models as model_registry +from .. import preprocess as preprocess_mod +from .. import quality as quality_mod +from .. import regressors as regressors_mod +from .. import seasonality as seasonality_mod +from .. import slo as slo_mod +from .. import telemetry as telemetry_mod +from ..accuracy import ( AccuracyResult, HorizonSpec, - backtest, evaluate, horizons_from_durations, overall_mape, overall_mase, ) -from .config import ( - CanaryConfig, +from ..config import ( GroupConfig, ModelSpec, QueryConfig, @@ -58,367 +58,76 @@ effective_season_length, effective_step, ) -from .exporter import ( - AccuracyPoint, +from ..exporter import ( BandCoveragePoint, - CalibrationOffsetPoint, - ComponentDecompositionPoint, - ConditionalDeviationPoint, - ConditionalForecastPoint, - ContributionPoint, - DeviationPoint, DriftPoint, - EnsembleWeightPoint, - ErrorBudgetPoint, Exporter, ForecastPoint, GroupSnapshot, - GrowthRatePoint, InvalidMetricNameError, PreprocessStats, QualityPoint, - RecommendedMaintenanceWindowPoint, - ThresholdEtaPoint, - TimeToDrainPoint, - WillBreachPoint, validate_metric_name, ) -from .point_factory import BacktestOutcome, BandSummary, SeriesPointFactory -from .selector import ensemble_weights, select -from .sink import ForecastCurvePoint, ForecastSink, SinkError -from .source import PromSource, SeriesFrame +from ..point_factory import BacktestOutcome, SeriesPointFactory +from ..selector import ensemble_weights, select +from ..sink import ForecastCurvePoint, ForecastSink, SinkError +from ..source import PromSource, SeriesFrame +from ._budgets import _apply_canary, _apply_overflow, _compute_global_budgets +from ._helpers import ( + _combine_ensemble, + _format_band_window, + _pandas_freq, + _resolve_spec, + _run_backtest, + _run_fit, + _series_key_from_labels, +) +from ._state import ( + ReloadResult, + _BacktestNeeds, + _CapacityInput, + _FitResult, + _inherit_prior_for_skipped_queries, + _merge_fit_into_ctx, + _RunContext, +) +from .emission.calibration import ( + _apply_offsets_to_curve_points, + _apply_offsets_to_snapshot_points, + _apply_offsets_to_threshold_eta_inputs, +) +from .emission.conditional import _flatten_regressors_to_last_value +from .emission.safeguards import ( + _apply_safeguards_to_curve_points, + _apply_safeguards_to_forecast_frame, + _apply_safeguards_to_snapshot_points, +) if TYPE_CHECKING: import pandas as pd - from .config import Config - from .models import ForecastModel - from .selector import SelectionResult + from ..config import Config + from ..selector import SelectionResult logger = structlog.get_logger(__name__) +# Reference to the parent package; used inside the Runner class to read +# ``SLOW_FIT_RATIO`` and ``_summarise_backtest`` dynamically so the +# pre-split test pattern +# ``monkeypatch.setattr(promforecast.runner, "SLOW_FIT_RATIO", ...)`` +# (and the same for ``_summarise_backtest``) continues to work after the +# split — the patched value is picked up on the next attribute lookup. +# The import is intentionally written as ``import promforecast.runner`` +# (not ``from . import ...``) so the binding is the package object +# itself, whose attributes are dynamically looked up at call time. The +# package is partial at import time but fully populated by the time any +# Runner method runs. +import promforecast.runner as _pkg # noqa: E402 -OverflowStrategy = Literal["drop_lowest_priority", "reject", "sample"] - -# A fit is considered "slow" once its wall-clock crosses this fraction of -# ``safety.fit_timeout``. We pick 0.5 (half) so the warning fires comfortably -# before the timeout actually kills the fit - long enough that operators -# can observe a degraded model before alerts go red, short enough that -# noise-induced 5-10% wobble doesn't trip it. -SLOW_FIT_RATIO = 0.5 - - -@dataclass -class _CapacityInput: - """One emitted (model, forecast frame) tuple, deferred to post-calibration. - - The forecast frame is kept by reference (not copied) so capturing it - is essentially free. ``selection`` is the auto-select decision that - produced this emission, ``None`` for explicit / ensemble component - rows. ``apply_calibration`` is False for the synthetic ensemble row - because conformal calibration is not run against ensemble outputs. - """ - - model_name: str - forecast: Any # pandas.DataFrame (avoid the import in non-TYPE_CHECKING land) - selection: Any # SelectionResult | None - apply_calibration: bool = True - - -@dataclass -class _FitResult: - """Per-series outputs returned by ``Runner._fit_series``.""" - - points: list[ForecastPoint] = field(default_factory=list) - accuracy: list[AccuracyPoint] = field(default_factory=list) - deviation: list[DeviationPoint] = field(default_factory=list) - quality: list[QualityPoint] = field(default_factory=list) - ensemble: list[EnsembleWeightPoint] = field(default_factory=list) - contribution: list[ContributionPoint] = field(default_factory=list) - # Full forecast curve (every horizon step) per emitted model. Consumed - # only by the ``remote_write`` sink — the snapshot exporter ignores it. - curve: list[ForecastCurvePoint] = field(default_factory=list) - # Per-model wall-clock fit time (seconds), aggregated to a single - # number per (query, model) by ``_RunContext`` below. Surfaces as - # ``forecast_fit_duration_seconds`` for fit-time observability. - fit_durations: dict[str, float] = field(default_factory=dict) - # Set of model names whose fit crossed the slow-fit threshold during - # this series; bumped into the per-(query, model) counter on the run - # context. - slow_fits: set[str] = field(default_factory=set) - # Per-(regressor_id, reason) regressor-resolution failures recorded for - # this series. Aggregated to (query_id, regressor_id, reason) on the - # run context. - regressor_failures: list[tuple[str, str]] = field(default_factory=list) - # Per-(model, level) conformal calibration offset for this series. - # Aggregated into ``calibration_offsets`` on the run context. - calibration_offsets: list[CalibrationOffsetPoint] = field(default_factory=list) - # Capacity-planning samples. ETAs are per-(model, threshold[, level]); - # growth rates are per-(model, window). Both are gauges (snapshot-style), - # aggregated into the run context without monotonic merging. - threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) - growth_rates: list[GrowthRatePoint] = field(default_factory=list) - # Per-(model, threshold[, level]) 0/1 predictive will-breach samples - # derived from the ETAs above. Snapshot-style; folded straight into - # the run context without monotonic merging. - will_breach: list[WillBreachPoint] = field(default_factory=list) - # Drain-ETA family for ``le_drain`` thresholds. Sibling of - # ``threshold_etas`` — separate field rather than a re-used one so - # the exporter's per-family rendering can preserve ``+Inf`` samples - # (the "queue is not draining" alert needs them to round-trip). - time_to_drain: list[TimeToDrainPoint] = field(default_factory=list) - # SLO error-budget snapshot rows (remaining, burn rate, exhausted_in). - # Snapshot-style; folded straight into the run context. - error_budget: list[ErrorBudgetPoint] = field(default_factory=list) - # Recommended maintenance windows — top-N lowest-load windows in the - # forecast horizon for this (series, model). Snapshot-style. - recommended_windows: list[RecommendedMaintenanceWindowPoint] = field(default_factory=list) - # Conditional forecast emission: snapshot + deviation + curve. All - # three are produced together when ``emission.conditional: true`` is - # on for the query. The curve flows into the optional remote_write - # sink alongside the unconditional curve. - conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) - conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) - conditional_curve: list[ForecastCurvePoint] = field(default_factory=list) - # Component decomposition emission. Per-(series, model, component) - # end-of-horizon snapshot rows for /metrics + per-step curve points - # for the optional remote_write sink (consumed by the stacked-area - # explainability dashboard). - component_points: list[ComponentDecompositionPoint] = field(default_factory=list) - component_curve: list[ForecastCurvePoint] = field(default_factory=list) - # Per-reason conditional-fit failure counter. Bumped on every fit - # that times out or raises; merged into ``ctx.failures`` so the - # operator sees them under ``forecast_failures_total`` with a - # bounded ``conditional_fit_*`` reason set. - conditional_failures: dict[str, int] = field(default_factory=dict) - # ``(model_name, reason)`` skip records the runner folds into the - # exporter's per-(model, reason) decomposition-skipped counter. - decomposition_skipped: list[tuple[str, str]] = field(default_factory=list) - # Per-model forecast dataframes retained until after conformal - # calibration so the capacity helpers can compute ETAs against the - # *calibrated* bands (a calibration that widens a band should pull - # the conservative ETA earlier or later by the same offset). Cleared - # at the end of ``_fit_series`` so we don't keep multi-MB dataframes - # alive in ``_RunContext``. - capacity_inputs: list[_CapacityInput] = field(default_factory=list) - # Imputed-fraction reported by preprocessing; threaded into the quality - # score so heavily-imputed series score lower on the coverage dimension. - imputed_fraction: float = 0.0 - # Count-aware safeguard counters: per-``kind`` increments folded - # into the run context's per-(query, kind) total. ``kind`` is - # ``negative`` (a value clipped from below zero) or ``rounded`` (a - # non-trivial fractional component that was rounded to integer). - emission_clipped: dict[str, int] = field(default_factory=dict) - # Cold-start: number of usable siblings the borrow query produced for - # this series; ``0`` when cold-start did not run. - cold_start_siblings: int = 0 - # Forecast-drift gauges: one per (series, model). NaN when no prior - # fit was cached for the (group, query, series, model) tuple. - drift_points: list[DriftPoint] = field(default_factory=list) - # Drift alert: how many threshold crossings happened for this query - # in this run. The runner folds it into the per-(query) total. - drift_alerts: int = 0 - - -@dataclass(frozen=True) -class _BacktestNeeds: - """Which downstream consumers want backtest data this run. - - Threaded through ``_run_backtests`` so it can skip post-processing the - consumers don't want. The fit itself is unconditional once *any* consumer - is interested — the per-model fit is the dominant cost; trimming the - cheap MAPE/MASE evaluation or the trailing-band summary saves only a - fraction of that. The bitmap mostly buys clarity (callers see exactly - which fields they will read) and a small win when ``accuracy.evaluate: false``. - """ - - accuracy: bool # forecast_accuracy_mape / mase samples emitted - selection: bool # auto-select needs overall_mape per model - deviation: bool # forecast_deviation_ratio / outside_band emitted - quality: bool # forecast_quality_score (uses results AND summary) - - @property - def any(self) -> bool: - return self.accuracy or self.selection or self.deviation or self.quality - - @property - def needs_results(self) -> bool: - """Per-horizon AccuracyResult list is needed by accuracy / selection / quality.""" - return self.accuracy or self.selection or self.quality - - @property - def needs_summary(self) -> bool: - """Trailing (actual, yhat, bands) summary is needed by deviation / quality.""" - return self.deviation or self.quality - - -@dataclass(frozen=True) -class ReloadResult: - """Summary of what changed during ``Runner.apply_config``. - - Returned to the operator (e.g. via the ``/-/reload`` endpoint response) - so they can confirm the diff matches their intent without diffing - /metrics output by hand. - """ - - added: list[str] = field(default_factory=list) - removed: list[str] = field(default_factory=list) - changed: list[str] = field(default_factory=list) - - -@dataclass -class _RunContext: - """Mutable accumulator threaded through ``_run_query`` calls. - - Group-level totals (points, drops, failures) used to live as locals in - ``run_group``; pulling them into a struct keeps the per-query helper a - method instead of a closure and lets tests inspect per-query state. - """ - - points: list[ForecastPoint] = field(default_factory=list) - accuracy_points: list[AccuracyPoint] = field(default_factory=list) - deviation_points: list[DeviationPoint] = field(default_factory=list) - quality_points: list[QualityPoint] = field(default_factory=list) - ensemble_weight_points: list[EnsembleWeightPoint] = field(default_factory=list) - contribution_points: list[ContributionPoint] = field(default_factory=list) - curve_points: list[ForecastCurvePoint] = field(default_factory=list) - series_count: int = 0 - dropped: dict[str, int] = field(default_factory=dict) - failures: dict[str, int] = field(default_factory=dict) - global_remaining: int = 0 - # Most-recent fit duration per (query_id, model_name). Multiple series - # under the same query keep overwriting each other intentionally — at - # snapshot time we want a representative most-recent number, not a sum. - fit_durations: dict[tuple[str, str], float] = field(default_factory=dict) - # Cumulative slow-fit counter per (query_id, model_name) across the - # series in this group run. Reset every run; the exporter merges with - # the previous snapshot's counter to keep a monotonic counter view. - slow_fits: dict[tuple[str, str], int] = field(default_factory=dict) - # Cumulative regressor-resolution failure counter per - # (query_id, regressor_id, reason). Merged with the previous snapshot - # in ``run_group`` so the counter stays monotonic across runs. - regressor_failures: dict[tuple[str, str, str], int] = field(default_factory=dict) - # Per-series conformal calibration offsets accumulated this run. - calibration_offsets: list[CalibrationOffsetPoint] = field(default_factory=list) - # Capacity-planning families (ETA + growth rate). Snapshot-style; - # only the current run's samples land on /metrics. - threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) - growth_rates: list[GrowthRatePoint] = field(default_factory=list) - # Will-breach predictive boolean gauges (per series/model/threshold/level). - will_breach_points: list[WillBreachPoint] = field(default_factory=list) - # Drain-ETA points (queue burn-down). Snapshot-style; preserves - # ``+Inf`` samples for the "queue is not draining" alert. - time_to_drain_points: list[TimeToDrainPoint] = field(default_factory=list) - # SLO error-budget snapshot rows accumulated this run. - error_budget_points: list[ErrorBudgetPoint] = field(default_factory=list) - # Recommended maintenance windows accumulated this run. - recommended_maintenance_window_points: list[RecommendedMaintenanceWindowPoint] = field( - default_factory=list - ) - # Conditional forecast emission accumulators (snapshot rows + - # per-(model, level) deviation gauges + per-step curve for the sink). - conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) - conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) - conditional_curve_points: list[ForecastCurvePoint] = field(default_factory=list) - # Per-(series, model, component) end-of-horizon decomposition rows - # + per-step curve for the sink. - component_points: list[ComponentDecompositionPoint] = field(default_factory=list) - component_curve_points: list[ForecastCurvePoint] = field(default_factory=list) - # Per-query preprocessing statistics for the most recent run. - preprocess_stats: dict[str, PreprocessStats] = field(default_factory=dict) - # Per-(query, strategy) gap-fill count for this run; merged with the - # previous snapshot so the counter stays monotonic. - gaps_filled: dict[tuple[str, str], int] = field(default_factory=dict) - # Per-(query, method) outlier replacement count for this run. - outliers_replaced: dict[tuple[str, str], int] = field(default_factory=dict) - # Per-query change-point detection count for this run. - change_points: dict[str, int] = field(default_factory=dict) - # Per-(query, kind) emission clipping counter for this run. Merged - # with the previous snapshot in ``run_group`` so the exposed counter - # stays monotonic across runs. - emission_clipped: dict[tuple[str, str], int] = field(default_factory=dict) - # Per-query usable-sibling count for the most recent cold-start fit. - # The runner prefetches the borrow query once per primary query (see - # ``_prefetch_cold_start_siblings``), so every cold-start fit within - # one run of a given query observes the same usable-sibling count — - # the gauge is deterministic per (group, query) regardless of which - # short series of the query was processed last. - cold_start_siblings: dict[str, int] = field(default_factory=dict) - # Per-(series, model) drift gauges from this run. Same shape as - # ``deviation_points`` — the renderer reads them directly. - drift_points: list[DriftPoint] = field(default_factory=list) - # Per-(query) drift threshold-crossing counter for this run. - drift_alerts: dict[str, int] = field(default_factory=dict) - # Per-query series count and run duration for the most recent run. - # Used to populate the ``query`` label on - # ``forecast_series_count`` and ``forecast_run_duration_seconds``. - series_count_per_query: dict[str, int] = field(default_factory=dict) - run_duration_per_query: dict[str, float] = field(default_factory=dict) - - -def _compute_global_budgets(groups: list[GroupConfig], max_total_series: int) -> dict[str, int]: - """Static per-group share of ``max_total_series``, weighted by priority. - - A static split avoids cross-group coordination at runtime: groups run on - independent timers and may overlap. With weights ``priority/sum(priorities)`` - the lowest-priority groups give up budget first, which matches the intent - of ``series_overflow: drop_lowest_priority``. - """ - if max_total_series <= 0 or not groups: - return {g.name: 0 for g in groups} - weights = {g.name: max(1, g.priority) for g in groups} - total_weight = sum(weights.values()) - return { - name: max(0, (max_total_series * weight) // total_weight) - for name, weight in weights.items() - } - - -def _apply_overflow( - frames: list[SeriesFrame], cap: int, strategy: OverflowStrategy -) -> tuple[list[SeriesFrame], int]: - """Reduce ``frames`` to fit ``cap`` according to ``strategy``. - - Returns ``(allowed, dropped_count)``. With ``cap <= 0`` everything is - dropped; ``reject`` drops the whole query rather than half-emitting it. - """ - if cap <= 0: - return [], len(frames) - if len(frames) <= cap: - return frames, 0 - if strategy == "reject": - return [], len(frames) - if strategy == "sample": - return random.sample(frames, cap), len(frames) - cap - return frames[:cap], len(frames) - cap - - -def _apply_canary( - frames: list[SeriesFrame], - canary: CanaryConfig, -) -> tuple[list[SeriesFrame], int]: - """Random-sample ``ceil(weight * N)`` frames when canary mode is on. - - Returns ``(kept, dropped)``. ``ceil`` (not ``floor``) so a tiny - population doesn't round down to zero — ``weight=0.1`` on a 5-series - query still fits one series, which is the useful canary case. The - caller emits the drop count as - ``forecast_series_dropped_total{reason="canary"}`` so the sampling - is observable from /metrics. - - Sampling uses :func:`random.sample` (no fixed seed) so successive runs - rotate through the population. A fixed seed would bias the canary to - the same N series forever and defeat coverage over time. - """ - if not canary.enabled or not frames: - return frames, 0 - if canary.weight >= 1.0: - return frames, 0 - keep = max(1, math.ceil(len(frames) * canary.weight)) - if keep >= len(frames): - return frames, 0 - sampled = random.sample(frames, keep) - return sampled, len(frames) - keep +__all__ = [ + "Runner", +] class Runner: @@ -904,6 +613,7 @@ async def run_group( # noqa: PLR0912, PLR0915 fit_durations=ctx.fit_durations, slow_fits=merged_slow, calibration_offsets=ctx.calibration_offsets, + conditional_calibration_offsets=ctx.conditional_calibration_offsets, preprocess_stats=ctx.preprocess_stats, gaps_filled_cumulative=merged_gaps, outliers_replaced_cumulative=merged_outliers, @@ -1799,12 +1509,30 @@ async def _fit_series( # noqa: PLR0912, PLR0915 # regressors") rather than the unconditional band. For built-in # models that ignore the regressors parameter the conditional # output equals the unconditional output; the family still emits - # so downstream alerts have a stable schema. The pass reuses the - # unconditional fit's conformal offsets + count-aware safeguard - # toggles so the two families never disagree on calibrated band - # width or clipping policy. + # so downstream alerts have a stable schema. When conformal + # calibration is enabled, a fresh per-fit calibration on a + # held-constant-regressor holdout produces dedicated conditional + # offsets so the band carries its own empirical-coverage + # guarantee (rather than inheriting the unconditional pass's + # offsets, which scored residuals on a holdout where regressors + # *did* vary). Per-(model) failures fall back to inherited + # offsets and bump ``forecast_conditional_calibration_failures_total``. if query.emission.conditional: cond_clip, cond_rounder = effective_emission_safeguards(query) + conditional_offsets_by_model: dict[str, dict[int, float]] = {} + if d.accuracy.conformal.enabled and result.capacity_inputs: + conditional_offsets_by_model = await self._compute_conditional_conformal_offsets( + df=df, + group_name=group_name, + query_id=query.id, + emitted_inputs=list(result.capacity_inputs), + model_specs=model_specs, + freq=freq, + levels=levels, + season_length=eff_season_length, + regressors=regressors_frame, + holdout_fraction=d.accuracy.conformal.holdout_fraction, + ) await self._emit_conditional_forecasts( df=df, query=query, @@ -1820,6 +1548,7 @@ async def _fit_series( # noqa: PLR0912, PLR0915 factory=factory, result=result, offsets_by_model=offsets_by_model, + conditional_offsets_by_model=conditional_offsets_by_model, clip=cond_clip, rounder=cond_rounder, ) @@ -2273,6 +2002,7 @@ async def _emit_conditional_forecasts( factory: SeriesPointFactory, result: _FitResult, offsets_by_model: dict[str, dict[int, float]], + conditional_offsets_by_model: dict[str, dict[int, float]], clip: bool, rounder: bool, ) -> None: @@ -2280,18 +2010,27 @@ async def _emit_conditional_forecasts( The pass adds one fit per emitted (model, series) on top of the regular fit, so the cost roughly doubles the forecast budget - for queries that opt in. See ``docs/conditional-forecasts.md``. + for queries that opt in. See the "Conditional forecasts" section + of ``docs/inputs/regressors.md``. When ``regressors`` is None (no regressors configured), the held-constant frame is also None and the re-fit produces the same numbers as the unconditional pass — the family still emits so dashboards and alerts can target it unconditionally. - ``offsets_by_model`` reuses the unconditional fit's conformal - offsets so the conditional bands carry the same calibration - widening as the unconditional ones — otherwise an operator - comparing the two would see different empirical-coverage - guarantees and infer something incorrect. + ``conditional_offsets_by_model`` carries per-(model, level) + offsets from a dedicated calibration pass that held regressors + flat at their final pre-holdout value — matching the + production-time conditional inference contract. When present + for a model, those offsets widen the conditional band (and + emit a ``forecast_conditional_band_calibration_offset`` gauge + through ``result.conditional_calibration_offsets``). When + absent (calibration off, fit failed, holdout too small), the + runner falls back to ``offsets_by_model`` — the inherited + unconditional offsets that pre-v1.6.5 builds always used. The + fallback path is no worse than the prior behaviour but + ``forecast_conditional_calibration_failures_total`` ticks so an + operator can see when fresh calibration silently isn't running. ``clip`` / ``rounder`` mirror the count-aware safeguards applied to the unconditional emission, so a count-shaped query sees @@ -2391,11 +2130,23 @@ async def _emit_conditional_forecasts( fit_seconds=fit_seconds, ) - # Reuse the unconditional offsets so the conditional bands - # carry the same empirical-coverage guarantee. ``offsets_by_model`` - # may not contain ``model_name`` (no calibration ran or - # ensemble row), in which case the helper is a no-op. - model_offsets = offsets_by_model.get(model_name) + # Prefer the fresh per-fit conditional offsets when the + # dedicated calibration pass produced them; otherwise fall + # back to the inherited unconditional offsets so the band + # is no worse off than the pre-v1.6.5 behaviour. When fresh + # offsets were computed, emit a + # ``forecast_conditional_band_calibration_offset`` row per + # (model, level) so the per-(model, level) widening + # applied to the conditional band specifically is visible + # from /metrics. + fresh_offsets = conditional_offsets_by_model.get(model_name) + if fresh_offsets: + model_offsets: dict[int, float] | None = fresh_offsets + result.conditional_calibration_offsets.extend( + factory.conditional_calibration_offset_points(model_name, fresh_offsets) + ) + else: + model_offsets = offsets_by_model.get(model_name) calibrated_df = _apply_offsets_to_threshold_eta_inputs( cond_forecast, model_offsets, levels ) @@ -2406,11 +2157,25 @@ async def _emit_conditional_forecasts( calibrated_df, levels=levels, clip=clip, rounder=rounder ) + # Mark the conditional band rows with ``calibrated="true"`` + # when fresh per-fit offsets were applied — matches the + # unconditional family's labelling so dashboards filtering + # on ``calibrated`` work consistently across the two + # families. We pass the flag instead of post-processing the + # rows because the conditional path applies offsets to the + # DataFrame *before* invoking the factory, so there's no + # second pass available; the unconditional path goes the + # other way (points first, then ``_apply_offsets_to_*`` + # rewrites them with the label) because its offsets are + # computed after the per-series fits already produced + # snapshot rows. + band_calibrated = bool(fresh_offsets) result.conditional_points.extend( factory.conditional_forecast_points( forecast=adjusted_df, model_name=model_name, selection=ci.selection, + calibrated=band_calibrated, ) ) # Stream the per-step conditional curve into the sink list so @@ -2420,7 +2185,10 @@ async def _emit_conditional_forecasts( # never collide on the TSDB side. result.conditional_curve.extend( factory.conditional_forecast_curve_points( - adjusted_df, model_name, selection=ci.selection + adjusted_df, + model_name, + selection=ci.selection, + calibrated=band_calibrated, ) ) if math.isfinite(actual): @@ -2759,7 +2527,7 @@ def _record_slow_fit_if_needed( timeout_seconds = self._config.safety.fit_timeout.total_seconds() if timeout_seconds <= 0: return - threshold = timeout_seconds * SLOW_FIT_RATIO + threshold = timeout_seconds * _pkg.SLOW_FIT_RATIO if fit_seconds < threshold: return result.slow_fits.add(model_name) @@ -2859,7 +2627,9 @@ async def _one(spec: ModelSpec) -> tuple[pd.DataFrame, pd.DataFrame] | None: continue test, predictions = outcome if needs.needs_summary: - summary = _summarise_backtest(predictions=predictions, test=test, levels=levels) + summary = _pkg._summarise_backtest( + predictions=predictions, test=test, levels=levels + ) # Deviation and quality both rely on the trailing tuple; if it # can't be derived (degenerate frames) drop the model rather # than emit half-populated rows downstream. @@ -3040,6 +2810,137 @@ async def _compute_conformal_offsets( results[model_name] = calib.offsets return results + async def _compute_conditional_conformal_offsets( # noqa: PLR0912 + self, + *, + df: pd.DataFrame, + group_name: str, + query_id: str, + emitted_inputs: list[_CapacityInput], + model_specs: list[ModelSpec], + freq: str, + levels: list[int], + season_length: int, + regressors: pd.DataFrame | None, + holdout_fraction: float, + ) -> dict[str, dict[int, float]]: + """Mirror of :meth:`_compute_conformal_offsets` for the conditional band. + + The only difference from the unconditional pass is the regressor + frame: the calibration fit's training regressors are flat-held at + the **final pre-holdout** value, matching the production-time + conditional inference contract. Computing fresh per-fit offsets + here means the conditional band's empirical-coverage guarantee + is grounded in residuals scored against the same regressor + regime production sees, rather than the unconditional pass's + residuals where regressors *did* vary across the holdout. + + Per-(model) failures isolate cleanly and bump + ``forecast_conditional_calibration_failures_total`` with a + bounded ``reason``. The caller falls back to + ``unconditional_offsets_by_model`` for any model missing from + the returned dict, so the fallback path is no worse than the + pre-v1.6.5 behaviour. + + Short-circuits to the empty dict (and bumps no failure counter) + when none of the emitted models actually consumes regressors — + the fresh fit would produce the same numbers as the unconditional + fit, so emitting two parallel offset gauges with identical + values would just be noise. + """ + emitted_model_names: set[str] = set() + for ci in emitted_inputs: + if ci.model_name and ci.model_name != "ensemble": + emitted_model_names.add(ci.model_name) + if not emitted_model_names: + return {} + # If every emitted model ignores regressors, the conditional fit + # produces the same numbers as the unconditional fit — no point + # spending CPU to compute identical offsets. + if all(not model_registry.uses_regressors(name) for name in emitted_model_names): + return {} + split = conformal_mod.holdout_split(df, holdout_fraction=holdout_fraction) + if split is None: + self._exporter.increment_conditional_calibration_failure( + group_name, query_id, "degenerate_holdout" + ) + return {} + train_df, calib_df = split + calib_actuals = calib_df["y"].to_numpy(dtype=float) + calib_steps = len(calib_df) + min_calib_for_quantile = 2 + if calib_steps < min_calib_for_quantile: + self._exporter.increment_conditional_calibration_failure( + group_name, query_id, "degenerate_holdout" + ) + return {} + # Build the held-constant regressor frame matching the + # production-time conditional contract: every row equals the + # final value of the pre-holdout window. ``None`` propagates + # through to a no-regressors fit for the (unusual) case where + # the configured regressors all resolved to nothing during the + # main fit. + train_regressors: pd.DataFrame | None + if regressors is not None and len(regressors) >= len(train_df): + train_regressors = _flatten_regressors_to_last_value( + regressors.iloc[: len(train_df)].reset_index(drop=True) + ) + else: + train_regressors = None + + spec_by_name = {spec.name: spec for spec in model_specs} + results: dict[str, dict[int, float]] = {} + for model_name in sorted(emitted_model_names): + # Models that don't consume regressors produce identical + # numbers either way; skip the extra fit (and skip the + # gauge emission so the operator isn't shown a duplicate + # of the unconditional widening). + if not model_registry.uses_regressors(model_name): + continue + spec = spec_by_name.get(model_name) or ModelSpec(name=model_name) + try: + calib_predictions = await self._fit_one( + spec=spec, + df=train_df, + horizon=calib_steps, + freq=freq, + levels=levels, + season_length=season_length, + regressors=train_regressors, + ) + except TimeoutError: + logger.warning( + "conditional_conformal_calibration_timeout", + group=group_name, + query=query_id, + model=model_name, + timeout_s=self._config.safety.fit_timeout.total_seconds(), + ) + self._exporter.increment_conditional_calibration_failure( + group_name, query_id, "fit_timeout" + ) + continue + except Exception as exc: + logger.warning( + "conditional_conformal_calibration_fit_failed", + group=group_name, + query=query_id, + model=model_name, + error=str(exc), + ) + self._exporter.increment_conditional_calibration_failure( + group_name, query_id, "fit_error" + ) + continue + calib = conformal_mod.compute_offsets( + calib_actuals=calib_actuals, + calib_predictions=calib_predictions, + levels=levels, + ) + if calib.offsets: + results[model_name] = calib.offsets + return results + async def _fit_one( self, *, @@ -3141,694 +3042,3 @@ async def stop(self) -> None: if self._sink is not None: with contextlib.suppress(Exception): await self._sink.aclose() - - -def _run_fit( - model: ForecastModel, - df: pd.DataFrame, - horizon: int, - freq: str, - levels: list[int], - regressors: pd.DataFrame | None = None, -) -> pd.DataFrame: - return model.fit_predict( - df=df, horizon=horizon, freq=freq, levels=levels, regressors=regressors - ) - - -# Conformal calibration is applied symmetrically to both the snapshot -# ``ForecastPoint`` list (consumed by /metrics) and the -# ``ForecastCurvePoint`` list (consumed by the remote_write sink). The two -# carry the same (metric, labels, value) shape but ``ForecastCurvePoint`` -# also has a ``timestamp_ms`` field. The decision logic — "if this is a -# lower/upper row for a calibrated (model, level), widen and label it" — -# is shared via :func:`_calibrated_band_value`; the per-type list rebuild -# lives in the two thin wrappers so each wrapper is straight-line code -# without a constructor callback. - - -def _calibrated_band_value( - metric: str, - labels: dict[str, str], - value: float, - offsets_by_model: dict[str, dict[int, float]], -) -> tuple[float, bool] | None: - """Return ``(new_value, calibrated)`` if the point is a calibratable band, else ``None``. - - ``None`` means "leave the point alone". ``(value, False)`` means the - offset is zero/missing for this (model, level), so the band is left as - the model emitted it; ``(value, True)`` means we widened and the - caller should add a ``calibrated="true"`` label. The yhat row (metric - exactly ending in ``_forecast``) is always returned as ``None`` — - calibration is a band adjustment, not a point-forecast adjustment. - """ - model_name = labels.get("model", "") - offsets = offsets_by_model.get(model_name) - if offsets is None or metric.endswith("_forecast"): - return None - try: - level = int(labels.get("level", "")) - except ValueError: - return None - offset = offsets.get(level) - if offset is None or offset <= 0: - return None - if metric.endswith("_forecast_lower"): - return value - offset, True - if metric.endswith("_forecast_upper"): - return value + offset, True - return None - - -def _apply_offsets_to_threshold_eta_inputs( - forecast: pd.DataFrame, - offsets: dict[int, float] | None, - levels: list[int], -) -> pd.DataFrame: - """Return a copy of ``forecast`` with band columns shifted by the per-level offset. - - The conformal calibration offset widens both sides of each band - symmetrically: ``yhat_upper_ += offset`` and - ``yhat_lower_ -= offset``. Re-applying that shift here so the - capacity ETA computation sees the same bands /metrics exposes keeps - ``time_to_threshold{level="95"}`` and ``yhat_upper{level="95"}`` - self-consistent. - - ``offsets=None`` or an empty dict is the common case (no calibration - ran for this model) and the frame is returned unmodified. Only - non-zero positive offsets actually mutate columns; a zero offset is - a no-op so we skip the copy. - """ - if not offsets or forecast.empty: - return forecast - if not any(o > 0 for o in offsets.values()): - return forecast - out = forecast.copy() - for level, offset in offsets.items(): - if offset <= 0 or level not in levels: - continue - lo_col = f"yhat_lower_{level}" - hi_col = f"yhat_upper_{level}" - if lo_col in out.columns: - out[lo_col] = out[lo_col].to_numpy(dtype=float) - offset - if hi_col in out.columns: - out[hi_col] = out[hi_col].to_numpy(dtype=float) + offset - return out - - -def _apply_offsets_to_snapshot_points( - points: list[ForecastPoint], - offsets_by_model: dict[str, dict[int, float]], -) -> list[ForecastPoint]: - """Widen ``ForecastPoint`` bands by the per-(model, level) offset. - - Returns a freshly constructed list; the input list is not mutated. - """ - if not offsets_by_model: - return points - out: list[ForecastPoint] = [] - for point in points: - adjusted = _calibrated_band_value(point.metric, point.labels, point.value, offsets_by_model) - if adjusted is None: - out.append(point) - continue - new_value, calibrated = adjusted - new_labels = {**point.labels, "calibrated": "true"} if calibrated else point.labels - out.append(ForecastPoint(metric=point.metric, labels=new_labels, value=new_value)) - return out - - -def _apply_offsets_to_curve_points( - points: list[ForecastCurvePoint], - offsets_by_model: dict[str, dict[int, float]], -) -> list[ForecastCurvePoint]: - """Widen ``ForecastCurvePoint`` bands by the per-(model, level) offset. - - Returns a freshly constructed list; the input list is not mutated. - Carries the original ``timestamp_ms`` through unchanged — calibration - shifts the band level, not when the forecast is for. - """ - if not offsets_by_model: - return points - out: list[ForecastCurvePoint] = [] - for point in points: - adjusted = _calibrated_band_value(point.metric, point.labels, point.value, offsets_by_model) - if adjusted is None: - out.append(point) - continue - new_value, calibrated = adjusted - new_labels = {**point.labels, "calibrated": "true"} if calibrated else point.labels - out.append( - ForecastCurvePoint( - metric=point.metric, - labels=new_labels, - value=new_value, - timestamp_ms=point.timestamp_ms, - ) - ) - return out - - -# Count-aware emission safeguards. -# -# When a query's ``emission.clip_non_negative`` is True every emitted -# forecast / band value is clamped at zero from below; when its -# ``emission.round_to_integer`` is True every value is rounded to the -# nearest integer. Both are applied symmetrically to the snapshot -# ``ForecastPoint`` list (consumed by /metrics) and to the -# ``ForecastCurvePoint`` list (consumed by the remote_write sink) so -# scrape and push see the same numbers. -# -# Counter semantics live in :func:`_classify_emission_correction` — -# ``kind="negative"`` ticks only when the raw value was actually below -# zero (a real model-output anomaly worth surfacing), and -# ``kind="rounded"`` ticks only when the fractional component was at -# least ``_ROUND_COUNTER_TRIGGER`` (so the counter doesn't blow up on -# every continuous emission that happens to be slightly non-integer). - -_ROUND_COUNTER_TRIGGER = 0.05 - - -def _classify_emission_correction( - raw: float, - *, - clip: bool, - rounder: bool, -) -> tuple[float, list[str]]: - """Return ``(adjusted_value, kinds)`` after applying the safeguards. - - ``kinds`` is a list of bounded tokens ``["negative", "rounded"]`` - that the caller folds into the per-query emission-clipped counter. - Each entry is recorded *once per emission* (a value that is both - negative and rounded triggers both counters); a value that didn't - need correction produces an empty list and contributes no counter - bumps. - - Non-finite values pass through unmodified — clipping or rounding a - NaN would surface the silent transformation as a confusing - /metrics line later. - """ - if not math.isfinite(raw): - return raw, [] - value = raw - kinds: list[str] = [] - if clip and value < 0.0: - value = 0.0 - kinds.append("negative") - if rounder: - rounded = float(round(value)) - if abs(value - rounded) >= _ROUND_COUNTER_TRIGGER: - kinds.append("rounded") - value = rounded - return value, kinds - - -def _apply_safeguards_to_snapshot_points( - points: list[ForecastPoint], - *, - clip: bool, - rounder: bool, - counter: dict[str, int], -) -> list[ForecastPoint]: - """Apply count-aware safeguards to ``ForecastPoint``s and bump ``counter``. - - Returns a freshly constructed list; the input list is not mutated. - ``counter`` is updated in-place with ``kind -> count`` so the - runner can fold it into the per-(query, kind) total without - threading a second return value. - """ - if not (clip or rounder): - return points - out: list[ForecastPoint] = [] - for point in points: - new_value, kinds = _classify_emission_correction(point.value, clip=clip, rounder=rounder) - for kind in kinds: - counter[kind] = counter.get(kind, 0) + 1 - if kinds or new_value != point.value: - out.append(ForecastPoint(metric=point.metric, labels=point.labels, value=new_value)) - else: - out.append(point) - return out - - -def _apply_safeguards_to_curve_points( - points: list[ForecastCurvePoint], - *, - clip: bool, - rounder: bool, -) -> list[ForecastCurvePoint]: - """Apply count-aware safeguards to ``ForecastCurvePoint``s. - - Counter bumps live with the snapshot pass — the runner emits both - families from the same fit and counting twice would double-bump - the operator-facing total. The curve pass therefore only mutates - values; it does not touch the counter. - """ - if not (clip or rounder): - return points - out: list[ForecastCurvePoint] = [] - for point in points: - new_value, _ = _classify_emission_correction(point.value, clip=clip, rounder=rounder) - if new_value != point.value: - out.append( - ForecastCurvePoint( - metric=point.metric, - labels=point.labels, - value=new_value, - timestamp_ms=point.timestamp_ms, - ) - ) - else: - out.append(point) - return out - - -def _merge_fit_into_ctx(ctx: _RunContext, fit_result: _FitResult, query_id: str) -> None: - """Splice a ``_FitResult`` into the per-group ``_RunContext``. - - Pulled out of ``_run_query`` so that hot path stays readable; keeps the - point-list extends and per-(query, model) counter bumps in one place. - """ - ctx.points.extend(fit_result.points) - ctx.accuracy_points.extend(fit_result.accuracy) - ctx.deviation_points.extend(fit_result.deviation) - ctx.quality_points.extend(fit_result.quality) - ctx.ensemble_weight_points.extend(fit_result.ensemble) - ctx.contribution_points.extend(fit_result.contribution) - ctx.curve_points.extend(fit_result.curve) - ctx.calibration_offsets.extend(fit_result.calibration_offsets) - ctx.threshold_etas.extend(fit_result.threshold_etas) - ctx.growth_rates.extend(fit_result.growth_rates) - ctx.will_breach_points.extend(fit_result.will_breach) - ctx.time_to_drain_points.extend(fit_result.time_to_drain) - ctx.error_budget_points.extend(fit_result.error_budget) - ctx.recommended_maintenance_window_points.extend(fit_result.recommended_windows) - ctx.conditional_points.extend(fit_result.conditional_points) - ctx.conditional_deviation_points.extend(fit_result.conditional_deviation_points) - ctx.conditional_curve_points.extend(fit_result.conditional_curve) - ctx.component_points.extend(fit_result.component_points) - ctx.component_curve_points.extend(fit_result.component_curve) - for reason, count in fit_result.conditional_failures.items(): - ctx.failures[reason] = ctx.failures.get(reason, 0) + count - for model_name, fit_seconds in fit_result.fit_durations.items(): - ctx.fit_durations[(query_id, model_name)] = fit_seconds - for model_name in fit_result.slow_fits: - key = (query_id, model_name) - ctx.slow_fits[key] = ctx.slow_fits.get(key, 0) + 1 - for regressor_id, reason in fit_result.regressor_failures: - rkey = (query_id, regressor_id, reason) - ctx.regressor_failures[rkey] = ctx.regressor_failures.get(rkey, 0) + 1 - for kind, count in fit_result.emission_clipped.items(): - ekey = (query_id, kind) - ctx.emission_clipped[ekey] = ctx.emission_clipped.get(ekey, 0) + count - if fit_result.cold_start_siblings > 0: - ctx.cold_start_siblings[query_id] = fit_result.cold_start_siblings - ctx.drift_points.extend(fit_result.drift_points) - if fit_result.drift_alerts: - ctx.drift_alerts[query_id] = ctx.drift_alerts.get(query_id, 0) + fit_result.drift_alerts - - -def _inherit_prior_for_skipped_queries( # noqa: PLR0912 - ctx: _RunContext, - prior: GroupSnapshot, - *, - run_query_ids: set[str], -) -> None: - """Preload ``ctx`` with prior-snapshot data for queries skipped this tick. - - With per-query refresh intervals, ``run_group`` may run only a - subset of the group's queries on any given tick. Without this - preload the snapshot would lose every other query's last-known - forecast / accuracy / deviation rows the next time the partial - snapshot is published. Counter-style fields (slow_fits, gaps_filled, - ...) are already merged with the prior via the explicit - counter-merge logic in :meth:`Runner.run_group`; here we only need - to copy snapshot-style (gauge / list) data. - - ``run_query_ids`` is the set of query IDs that *will* run this - tick; everything else gets its prior data inherited. ``curve_points`` - are intentionally *not* inherited because they drive the remote_write - sink — pushing previously-fit curves every refresh would duplicate - data in the TSDB. - """ - # Point lists keyed by query-id label. - for point in prior.accuracy_points: - if point.labels.get("id") not in run_query_ids: - ctx.accuracy_points.append(point) - for dpoint in prior.deviation_points: - if dpoint.labels.get("id") not in run_query_ids: - ctx.deviation_points.append(dpoint) - for qpoint in prior.quality_points: - if qpoint.labels.get("id") not in run_query_ids: - ctx.quality_points.append(qpoint) - for epoint in prior.ensemble_weights: - if epoint.labels.get("id") not in run_query_ids: - ctx.ensemble_weight_points.append(epoint) - for cpoint in prior.calibration_offsets: - if cpoint.labels.get("id") not in run_query_ids: - ctx.calibration_offsets.append(cpoint) - for drift_point in prior.drift_points: - if drift_point.labels.get("id") not in run_query_ids: - ctx.drift_points.append(drift_point) - # Point lists keyed by metric name (which encodes the query id). - for fpoint in prior.points: - q_id = _query_id_from_metric(fpoint.metric) - if q_id not in run_query_ids: - ctx.points.append(fpoint) - for contribution_point in prior.contribution_points: - if _query_id_from_metric(contribution_point.metric) not in run_query_ids: - ctx.contribution_points.append(contribution_point) - for tpoint in prior.threshold_etas: - if _query_id_from_metric(tpoint.metric) not in run_query_ids: - ctx.threshold_etas.append(tpoint) - for gpoint in prior.growth_rates: - if _query_id_from_metric(gpoint.metric) not in run_query_ids: - ctx.growth_rates.append(gpoint) - for wbpoint in prior.will_breach_points: - if _query_id_from_metric(wbpoint.metric) not in run_query_ids: - ctx.will_breach_points.append(wbpoint) - for drain_point in prior.time_to_drain_points: - if _query_id_from_metric(drain_point.metric) not in run_query_ids: - ctx.time_to_drain_points.append(drain_point) - for ebpoint in prior.error_budget_points: - if _query_id_from_metric(ebpoint.metric) not in run_query_ids: - ctx.error_budget_points.append(ebpoint) - # ``forecast_recommended_maintenance_window`` is a global metric - # name (no per-id prefix); use the ``id`` label to scope inheritance. - for rwpoint in prior.recommended_maintenance_window_points: - if rwpoint.labels.get("id") not in run_query_ids: - ctx.recommended_maintenance_window_points.append(rwpoint) - for cfpoint in prior.conditional_points: - if _query_id_from_metric(cfpoint.metric) not in run_query_ids: - ctx.conditional_points.append(cfpoint) - for cdpoint in prior.conditional_deviation_points: - if cdpoint.labels.get("id") not in run_query_ids: - ctx.conditional_deviation_points.append(cdpoint) - for compoint in prior.component_points: - if _query_id_from_metric(compoint.metric) not in run_query_ids: - ctx.component_points.append(compoint) - # Per-(query, model) and per-query dicts that aren't counters. - for (q_id, model_name), fit_seconds in prior.fit_durations.items(): - if q_id not in run_query_ids: - ctx.fit_durations[(q_id, model_name)] = fit_seconds - for q_id, stats in prior.preprocess_stats.items(): - if q_id not in run_query_ids: - # ``PreprocessStats`` is mutable (the runner mutates it - # in-place when accumulating per-query stats). Copying - # defensively here prevents a future writer on the new ctx - # from mutating the prior snapshot mid-render: a follower - # serving ``/metrics`` would otherwise see torn state. - ctx.preprocess_stats[q_id] = PreprocessStats( - gaps_filled=dict(stats.gaps_filled), - max_gap_seconds=stats.max_gap_seconds, - outliers_replaced=dict(stats.outliers_replaced), - change_points=stats.change_points, - effective_lookback_seconds=stats.effective_lookback_seconds, - ) - for q_id, count in prior.cold_start_siblings.items(): - if q_id not in run_query_ids: - ctx.cold_start_siblings[q_id] = count - for q_id, dur in prior.run_duration_per_query.items(): - if q_id not in run_query_ids: - ctx.run_duration_per_query[q_id] = dur - for q_id, count in prior.series_count_per_query.items(): - if q_id not in run_query_ids: - ctx.series_count_per_query[q_id] = count - - -def _apply_safeguards_to_forecast_frame( - forecast: pd.DataFrame, - *, - levels: list[int], - clip: bool, - rounder: bool, -) -> pd.DataFrame: - """Return a copy of ``forecast`` with count-aware safeguards applied. - - Used by the conditional emission path so the snapshot points, the - sink curve, and the deviation gauges built from the same frame all - see the same clipped/rounded values. Counter bumps live with the - unconditional pass — the conditional emission re-uses the same fit - semantics and counting twice would double-charge the per-(query, - kind) total. - - Uses numpy vectorisation so a typical horizon (288 rows by 5 columns - at default 2 levels) is one pass per column instead of 1440 - Python-level float conversions. - """ - if forecast.empty or not (clip or rounder): - return forecast - import numpy as np # noqa: PLC0415 - import pandas as pd # noqa: PLC0415 - - out = forecast.copy() - cols = ["yhat"] + [ - col for level in levels for col in (f"yhat_lower_{level}", f"yhat_upper_{level}") - ] - for col in cols: - if col not in out.columns: - continue - # Coerce to float so clip / round have well-defined semantics - # against pandas extension-array columns. ``errors="coerce"`` - # turns non-numeric entries into NaN, which both numpy ops - # tolerate and propagate — matching the Python-loop version's - # behaviour of passing such entries through unchanged. - arr = pd.to_numeric(out[col], errors="coerce").to_numpy(dtype=float) - # Pre-record the NaN mask so non-finite inputs round-trip - # unchanged: ``np.round`` of NaN is NaN already, but clipping - # would turn NaN into 0.0 under naive ``np.maximum`` semantics — - # we restore NaN at the end so the contract matches the scalar - # ``_classify_emission_correction`` path. - nan_mask = ~np.isfinite(arr) - if clip: - # ``np.maximum`` preserves the input shape and dtype (unlike - # ``np.where`` whose typed return is multidim) — the column - # is 1-D so this is the right primitive here. - arr = np.maximum(arr, 0.0) - if rounder: - arr = np.round(arr) - if nan_mask.any(): - arr[nan_mask] = np.nan - out[col] = arr - return out - - -def _flatten_regressors_to_last_value( - regressors: pd.DataFrame | None, -) -> pd.DataFrame | None: - """Return a copy of ``regressors`` whose every row equals the last row. - - Implements the "held at current value" semantics of the conditional - forecast: each regressor column is replaced with a constant series - equal to its most recent in-sample value. Calendar features (which - legitimately vary per timestamp) are preserved through the same - flattening — the conditional contract is "imagine current state - continued unchanged", which by construction freezes the calendar as - well. Returns ``None`` when no regressors were configured so the - caller skips the re-fit's regressor argument cleanly. - - Uses scalar broadcast (``flat[col] = scalar``) instead of building a - Python list per column so a wide regressor frame doesn't materialise - N copies of the same value through the heap. pandas treats the - scalar assignment as a constant broadcast and writes a single typed - array. - """ - if regressors is None or regressors.empty: - return regressors - import pandas as pd # noqa: PLC0415 - - last_row = regressors.iloc[-1] - flat = pd.DataFrame(index=regressors.index) - for col in regressors.columns: - if col == "ds": - flat[col] = regressors[col].to_numpy() - continue - # Scalar broadcast: pandas writes a constant array of the right - # dtype without allocating a Python list of N references. - flat[col] = last_row[col] - return flat - - -def _series_key_from_labels(labels: dict[str, str]) -> str: - """Stable per-series key derived from the label set. - - Used to address per-series state in the drift cache and band-coverage - tracker. Skips ``__name__`` (Prometheus's reserved metric-name - pseudo-label) so two queries that return the same labelset on a - different metric share state appropriately — both forecast the same - underlying signal. - - The key is ``"k1=v1,k2=v2"`` with sorted keys so equal label - dictionaries produce the same key regardless of iteration order. - """ - items = sorted((k, v) for k, v in labels.items() if k != "__name__") - return ",".join(f"{k}={v}" for k, v in items) - - -def _format_band_window(window: timedelta) -> str: - """Render a ``timedelta`` window as a Prometheus-style label.""" - return point_factory_mod.format_window(window.total_seconds()) - - -# Suffixes that the runner / point_factory emits on forecast-derived -# metrics. ``_query_id_from_metric`` matches the longest one first so an -# id like ``api_forecast_count`` (a legal Prometheus identifier) is not -# mis-routed by an internal ``_forecast`` substring. -_FORECAST_METRIC_SUFFIXES: tuple[str, ...] = ( - "_forecast_contribution_ratio", - "_forecast_time_to_threshold_seconds", - "_forecast_time_to_drain_seconds", - "_forecast_error_budget_remaining_seconds", - "_forecast_error_budget_exhausted_in_seconds", - "_forecast_error_budget_burn_rate", - "_forecast_growth_rate", - "_forecast_will_breach", - "_forecast_conditional_lower", - "_forecast_conditional_upper", - "_forecast_conditional", - "_forecast_component", - "_forecast_lower", - "_forecast_upper", - "_forecast", -) - - -def _query_id_from_metric(metric_name: str) -> str: - """Pull the query ``id`` back out of a forecast metric name. - - Matches the known emission suffix set rather than ``rfind("_forecast")`` - so a query id that legitimately contains the word ``forecast`` (e.g. - ``api_forecast_count``) round-trips correctly. - """ - for suffix in _FORECAST_METRIC_SUFFIXES: - if metric_name.endswith(suffix): - stem = metric_name[: -len(suffix)] - if stem: - return stem - return "" - - -def _resolve_spec(specs: list[ModelSpec], name: str) -> ModelSpec: - """Look up a configured ``ModelSpec`` by name; defaults to params=``{}``. - - The fallback model is allowed to live outside ``defaults.models`` (a - SeasonalNaive safety net for an AutoARIMA-only setup is the canonical - case). We synthesise an empty-params spec so the fallback runs with - stock model defaults rather than failing the lookup. - """ - for spec in specs: - if spec.name == name: - return spec - return ModelSpec(name=name) - - -def _combine_ensemble( - component_forecasts: dict[str, pd.DataFrame], - weights: dict[str, float], - levels: list[int], -) -> pd.DataFrame: - """Build a single forecast frame from inverse-MAPE-weighted components. - - Aligns on ``unique_id`` + ``ds``. Each numeric column is the weighted - sum across components present at that row; missing columns (a model - that didn't emit a level we requested) contribute weight 0 so the - ensemble doesn't quietly inherit one component's missing band. - """ - import pandas as pd # noqa: PLC0415 - - base_cols = ["unique_id", "ds"] - value_cols = ["yhat"] + [ - f"yhat_{kind}_{level}" for kind in ("lower", "upper") for level in levels - ] - - import numpy as np # noqa: PLC0415 - - accumulator: pd.DataFrame | None = None - weight_accumulator: dict[str, np.ndarray[Any, Any]] | None = None - for name, weight in weights.items(): - df = component_forecasts.get(name) - if df is None or df.empty: - continue - if accumulator is None or weight_accumulator is None: - accumulator = df[base_cols].reset_index(drop=True).copy() - weight_accumulator = {col: np.zeros(len(df), dtype=float) for col in value_cols} - for col in value_cols: - accumulator[col] = 0.0 - for col in value_cols: - if col not in df.columns: - continue - # Switch to numpy arrays for the accumulation: pandas would - # try to align on the (matching) index every iteration, which - # both wastes work and confuses pandas-stubs about the - # resulting dtype. - current = accumulator[col].to_numpy(dtype=float) - accumulator[col] = current + df[col].to_numpy(dtype=float) * weight - weight_accumulator[col] = weight_accumulator[col] + weight - if accumulator is None or weight_accumulator is None: - # No usable components — return an empty frame the caller can detect. - return pd.DataFrame(columns=[*base_cols, *value_cols]) - # Convert weight sums back to plain Series for the divide-by-weight step; - # rows where total weight is 0 retain NaN rather than dividing by zero. - for col in value_cols: - denom = pd.Series(weight_accumulator[col]) - accumulator[col] = accumulator[col] / denom.where(denom > 0, other=float("nan")) - return accumulator - - -def _run_backtest( - model: ForecastModel, - df: pd.DataFrame, - horizon_steps: int, - freq: str, - levels: list[int], - regressors: pd.DataFrame | None = None, -) -> tuple[pd.DataFrame, pd.DataFrame] | None: - return backtest( - model=model, - df=df, - horizon_steps=horizon_steps, - freq=freq, - levels=levels, - regressors=regressors, - ) - - -def _pandas_freq(step: timedelta) -> str: - seconds = int(step.total_seconds()) - if seconds % 3600 == 0: - return f"{seconds // 3600}h" - if seconds % 60 == 0: - return f"{seconds // 60}min" - return f"{seconds}s" - - -def _summarise_backtest( - predictions: pd.DataFrame, - test: pd.DataFrame, - levels: list[int], -) -> tuple[float, float, dict[int, BandSummary]] | None: - """Extract the trailing (actual, yhat, per-level band) tuple, or None. - - Returns ``None`` if either frame is empty or missing the ``y``/``yhat`` - columns; non-finite cells are tolerated and surface downstream as NaN. - """ - if predictions.empty or test.empty: - return None - try: - last_actual = float(test["y"].iloc[-1]) - last_yhat = float(predictions["yhat"].iloc[-1]) - except (KeyError, IndexError, ValueError): - return None - bands: dict[int, BandSummary] = {} - for level in levels: - lo_col = f"yhat_lower_{level}" - hi_col = f"yhat_upper_{level}" - if lo_col not in predictions.columns or hi_col not in predictions.columns: - continue - try: - lower = float(predictions[lo_col].iloc[-1]) - upper = float(predictions[hi_col].iloc[-1]) - except (ValueError, IndexError): - continue - bands[level] = BandSummary(lower=lower, upper=upper) - return last_actual, last_yhat, bands diff --git a/forecaster/src/promforecast/runner/_state.py b/forecaster/src/promforecast/runner/_state.py new file mode 100644 index 0000000..c9c3dcd --- /dev/null +++ b/forecaster/src/promforecast/runner/_state.py @@ -0,0 +1,492 @@ +"""Per-fit and per-group state accumulators threaded through the runner. + +``_FitResult`` holds the per-series outputs returned by +``Runner._fit_series``; ``_RunContext`` is the per-group mutable +accumulator that the runner extends as each series in a group is +processed. ``_merge_fit_into_ctx`` splices the two together, and +``_inherit_prior_for_skipped_queries`` preloads the context with +prior-snapshot data for any query that's not running on the current +tick (the per-query refresh-interval case). + +The inheritance loop is driven by a small ``_InheritanceSpec`` registry +so adding an emission family means appending one tuple to +:data:`_INHERITANCE_REGISTRY` rather than hand-coding a copy block. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +from ..exporter import ( + AccuracyPoint, + CalibrationOffsetPoint, + ComponentDecompositionPoint, + ConditionalCalibrationOffsetPoint, + ConditionalDeviationPoint, + ConditionalForecastPoint, + ContributionPoint, + DeviationPoint, + DriftPoint, + EnsembleWeightPoint, + ErrorBudgetPoint, + ForecastPoint, + GroupSnapshot, + GrowthRatePoint, + PreprocessStats, + QualityPoint, + RecommendedMaintenanceWindowPoint, + ThresholdEtaPoint, + TimeToDrainPoint, + WillBreachPoint, +) +from ..sink import ForecastCurvePoint +from ._helpers import _query_id_from_metric + + +@dataclass +class _CapacityInput: + """One emitted (model, forecast frame) tuple, deferred to post-calibration. + + The forecast frame is kept by reference (not copied) so capturing it + is essentially free. ``selection`` is the auto-select decision that + produced this emission, ``None`` for explicit / ensemble component + rows. ``apply_calibration`` is False for the synthetic ensemble row + because conformal calibration is not run against ensemble outputs. + """ + + model_name: str + forecast: Any # pandas.DataFrame (avoid the import in non-TYPE_CHECKING land) + selection: Any # SelectionResult | None + apply_calibration: bool = True + + +@dataclass +class _FitResult: + """Per-series outputs returned by ``Runner._fit_series``.""" + + points: list[ForecastPoint] = field(default_factory=list) + accuracy: list[AccuracyPoint] = field(default_factory=list) + deviation: list[DeviationPoint] = field(default_factory=list) + quality: list[QualityPoint] = field(default_factory=list) + ensemble: list[EnsembleWeightPoint] = field(default_factory=list) + contribution: list[ContributionPoint] = field(default_factory=list) + # Full forecast curve (every horizon step) per emitted model. Consumed + # only by the ``remote_write`` sink — the snapshot exporter ignores it. + curve: list[ForecastCurvePoint] = field(default_factory=list) + # Per-model wall-clock fit time (seconds), aggregated to a single + # number per (query, model) by ``_RunContext`` below. Surfaces as + # ``forecast_fit_duration_seconds`` for fit-time observability. + fit_durations: dict[str, float] = field(default_factory=dict) + # Set of model names whose fit crossed the slow-fit threshold during + # this series; bumped into the per-(query, model) counter on the run + # context. + slow_fits: set[str] = field(default_factory=set) + # Per-(regressor_id, reason) regressor-resolution failures recorded for + # this series. Aggregated to (query_id, regressor_id, reason) on the + # run context. + regressor_failures: list[tuple[str, str]] = field(default_factory=list) + # Per-(model, level) conformal calibration offset for this series. + # Aggregated into ``calibration_offsets`` on the run context. + calibration_offsets: list[CalibrationOffsetPoint] = field(default_factory=list) + # Per-(model, level) conditional-band conformal calibration offset + # for this series. Same shape as ``calibration_offsets`` but derived + # from a calibration fit whose regressors are held flat at the final + # pre-holdout value (matching the production-time conditional + # inference contract). Aggregated into + # ``conditional_calibration_offsets`` on the run context. + conditional_calibration_offsets: list[ConditionalCalibrationOffsetPoint] = field( + default_factory=list + ) + # Capacity-planning samples. ETAs are per-(model, threshold[, level]); + # growth rates are per-(model, window). Both are gauges (snapshot-style), + # aggregated into the run context without monotonic merging. + threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) + growth_rates: list[GrowthRatePoint] = field(default_factory=list) + # Per-(model, threshold[, level]) 0/1 predictive will-breach samples + # derived from the ETAs above. Snapshot-style; folded straight into + # the run context without monotonic merging. + will_breach: list[WillBreachPoint] = field(default_factory=list) + # Drain-ETA family for ``le_drain`` thresholds. Sibling of + # ``threshold_etas`` — separate field rather than a re-used one so + # the exporter's per-family rendering can preserve ``+Inf`` samples + # (the "queue is not draining" alert needs them to round-trip). + time_to_drain: list[TimeToDrainPoint] = field(default_factory=list) + # SLO error-budget snapshot rows (remaining, burn rate, exhausted_in). + # Snapshot-style; folded straight into the run context. + error_budget: list[ErrorBudgetPoint] = field(default_factory=list) + # Recommended maintenance windows — top-N lowest-load windows in the + # forecast horizon for this (series, model). Snapshot-style. + recommended_windows: list[RecommendedMaintenanceWindowPoint] = field(default_factory=list) + # Conditional forecast emission: snapshot + deviation + curve. All + # three are produced together when ``emission.conditional: true`` is + # on for the query. The curve flows into the optional remote_write + # sink alongside the unconditional curve. + conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) + conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) + conditional_curve: list[ForecastCurvePoint] = field(default_factory=list) + # Component decomposition emission. Per-(series, model, component) + # end-of-horizon snapshot rows for /metrics + per-step curve points + # for the optional remote_write sink (consumed by the stacked-area + # explainability dashboard). + component_points: list[ComponentDecompositionPoint] = field(default_factory=list) + component_curve: list[ForecastCurvePoint] = field(default_factory=list) + # Per-reason conditional-fit failure counter. Bumped on every fit + # that times out or raises; merged into ``ctx.failures`` so the + # operator sees them under ``forecast_failures_total`` with a + # bounded ``conditional_fit_*`` reason set. + conditional_failures: dict[str, int] = field(default_factory=dict) + # ``(model_name, reason)`` skip records the runner folds into the + # exporter's per-(model, reason) decomposition-skipped counter. + decomposition_skipped: list[tuple[str, str]] = field(default_factory=list) + # Per-model forecast dataframes retained until after conformal + # calibration so the capacity helpers can compute ETAs against the + # *calibrated* bands (a calibration that widens a band should pull + # the conservative ETA earlier or later by the same offset). Cleared + # at the end of ``_fit_series`` so we don't keep multi-MB dataframes + # alive in ``_RunContext``. + capacity_inputs: list[_CapacityInput] = field(default_factory=list) + # Imputed-fraction reported by preprocessing; threaded into the quality + # score so heavily-imputed series score lower on the coverage dimension. + imputed_fraction: float = 0.0 + # Count-aware safeguard counters: per-``kind`` increments folded + # into the run context's per-(query, kind) total. ``kind`` is + # ``negative`` (a value clipped from below zero) or ``rounded`` (a + # non-trivial fractional component that was rounded to integer). + emission_clipped: dict[str, int] = field(default_factory=dict) + # Cold-start: number of usable siblings the borrow query produced for + # this series; ``0`` when cold-start did not run. + cold_start_siblings: int = 0 + # Forecast-drift gauges: one per (series, model). NaN when no prior + # fit was cached for the (group, query, series, model) tuple. + drift_points: list[DriftPoint] = field(default_factory=list) + # Drift alert: how many threshold crossings happened for this query + # in this run. The runner folds it into the per-(query) total. + drift_alerts: int = 0 + + +@dataclass(frozen=True) +class _BacktestNeeds: + """Which downstream consumers want backtest data this run. + + Threaded through ``_run_backtests`` so it can skip post-processing the + consumers don't want. The fit itself is unconditional once *any* consumer + is interested — the per-model fit is the dominant cost; trimming the + cheap MAPE/MASE evaluation or the trailing-band summary saves only a + fraction of that. The bitmap mostly buys clarity (callers see exactly + which fields they will read) and a small win when ``accuracy.evaluate: false``. + """ + + accuracy: bool # forecast_accuracy_mape / mase samples emitted + selection: bool # auto-select needs overall_mape per model + deviation: bool # forecast_deviation_ratio / outside_band emitted + quality: bool # forecast_quality_score (uses results AND summary) + + @property + def any(self) -> bool: + return self.accuracy or self.selection or self.deviation or self.quality + + @property + def needs_results(self) -> bool: + """Per-horizon AccuracyResult list is needed by accuracy / selection / quality.""" + return self.accuracy or self.selection or self.quality + + @property + def needs_summary(self) -> bool: + """Trailing (actual, yhat, bands) summary is needed by deviation / quality.""" + return self.deviation or self.quality + + +@dataclass(frozen=True) +class ReloadResult: + """Summary of what changed during ``Runner.apply_config``. + + Returned to the operator (e.g. via the ``/-/reload`` endpoint response) + so they can confirm the diff matches their intent without diffing + /metrics output by hand. + """ + + added: list[str] = field(default_factory=list) + removed: list[str] = field(default_factory=list) + changed: list[str] = field(default_factory=list) + + +@dataclass +class _RunContext: + """Mutable accumulator threaded through ``_run_query`` calls. + + Group-level totals (points, drops, failures) used to live as locals in + ``run_group``; pulling them into a struct keeps the per-query helper a + method instead of a closure and lets tests inspect per-query state. + """ + + points: list[ForecastPoint] = field(default_factory=list) + accuracy_points: list[AccuracyPoint] = field(default_factory=list) + deviation_points: list[DeviationPoint] = field(default_factory=list) + quality_points: list[QualityPoint] = field(default_factory=list) + ensemble_weight_points: list[EnsembleWeightPoint] = field(default_factory=list) + contribution_points: list[ContributionPoint] = field(default_factory=list) + curve_points: list[ForecastCurvePoint] = field(default_factory=list) + series_count: int = 0 + dropped: dict[str, int] = field(default_factory=dict) + failures: dict[str, int] = field(default_factory=dict) + global_remaining: int = 0 + # Most-recent fit duration per (query_id, model_name). Multiple series + # under the same query keep overwriting each other intentionally — at + # snapshot time we want a representative most-recent number, not a sum. + fit_durations: dict[tuple[str, str], float] = field(default_factory=dict) + # Cumulative slow-fit counter per (query_id, model_name) across the + # series in this group run. Reset every run; the exporter merges with + # the previous snapshot's counter to keep a monotonic counter view. + slow_fits: dict[tuple[str, str], int] = field(default_factory=dict) + # Cumulative regressor-resolution failure counter per + # (query_id, regressor_id, reason). Merged with the previous snapshot + # in ``run_group`` so the counter stays monotonic across runs. + regressor_failures: dict[tuple[str, str, str], int] = field(default_factory=dict) + # Per-series conformal calibration offsets accumulated this run. + calibration_offsets: list[CalibrationOffsetPoint] = field(default_factory=list) + # Per-series conditional-band conformal offsets accumulated this run. + # Sibling of ``calibration_offsets`` — rendered as a parallel gauge + # family so operators can read the per-(model, level) widening on the + # conditional band specifically. + conditional_calibration_offsets: list[ConditionalCalibrationOffsetPoint] = field( + default_factory=list + ) + # Capacity-planning families (ETA + growth rate). Snapshot-style; + # only the current run's samples land on /metrics. + threshold_etas: list[ThresholdEtaPoint] = field(default_factory=list) + growth_rates: list[GrowthRatePoint] = field(default_factory=list) + # Will-breach predictive boolean gauges (per series/model/threshold/level). + will_breach_points: list[WillBreachPoint] = field(default_factory=list) + # Drain-ETA points (queue burn-down). Snapshot-style; preserves + # ``+Inf`` samples for the "queue is not draining" alert. + time_to_drain_points: list[TimeToDrainPoint] = field(default_factory=list) + # SLO error-budget snapshot rows accumulated this run. + error_budget_points: list[ErrorBudgetPoint] = field(default_factory=list) + # Recommended maintenance windows accumulated this run. + recommended_maintenance_window_points: list[RecommendedMaintenanceWindowPoint] = field( + default_factory=list + ) + # Conditional forecast emission accumulators (snapshot rows + + # per-(model, level) deviation gauges + per-step curve for the sink). + conditional_points: list[ConditionalForecastPoint] = field(default_factory=list) + conditional_deviation_points: list[ConditionalDeviationPoint] = field(default_factory=list) + conditional_curve_points: list[ForecastCurvePoint] = field(default_factory=list) + # Per-(series, model, component) end-of-horizon decomposition rows + # + per-step curve for the sink. + component_points: list[ComponentDecompositionPoint] = field(default_factory=list) + component_curve_points: list[ForecastCurvePoint] = field(default_factory=list) + # Per-query preprocessing statistics for the most recent run. + preprocess_stats: dict[str, PreprocessStats] = field(default_factory=dict) + # Per-(query, strategy) gap-fill count for this run; merged with the + # previous snapshot so the counter stays monotonic. + gaps_filled: dict[tuple[str, str], int] = field(default_factory=dict) + # Per-(query, method) outlier replacement count for this run. + outliers_replaced: dict[tuple[str, str], int] = field(default_factory=dict) + # Per-query change-point detection count for this run. + change_points: dict[str, int] = field(default_factory=dict) + # Per-(query, kind) emission clipping counter for this run. Merged + # with the previous snapshot in ``run_group`` so the exposed counter + # stays monotonic across runs. + emission_clipped: dict[tuple[str, str], int] = field(default_factory=dict) + # Per-query usable-sibling count for the most recent cold-start fit. + # The runner prefetches the borrow query once per primary query (see + # ``_prefetch_cold_start_siblings``), so every cold-start fit within + # one run of a given query observes the same usable-sibling count — + # the gauge is deterministic per (group, query) regardless of which + # short series of the query was processed last. + cold_start_siblings: dict[str, int] = field(default_factory=dict) + # Per-(series, model) drift gauges from this run. Same shape as + # ``deviation_points`` — the renderer reads them directly. + drift_points: list[DriftPoint] = field(default_factory=list) + # Per-(query) drift threshold-crossing counter for this run. + drift_alerts: dict[str, int] = field(default_factory=dict) + # Per-query series count and run duration for the most recent run. + # Used to populate the ``query`` label on + # ``forecast_series_count`` and ``forecast_run_duration_seconds``. + series_count_per_query: dict[str, int] = field(default_factory=dict) + run_duration_per_query: dict[str, float] = field(default_factory=dict) + + +def _merge_fit_into_ctx(ctx: _RunContext, fit_result: _FitResult, query_id: str) -> None: + """Splice a ``_FitResult`` into the per-group ``_RunContext``. + + Pulled out of ``_run_query`` so that hot path stays readable; keeps the + point-list extends and per-(query, model) counter bumps in one place. + """ + ctx.points.extend(fit_result.points) + ctx.accuracy_points.extend(fit_result.accuracy) + ctx.deviation_points.extend(fit_result.deviation) + ctx.quality_points.extend(fit_result.quality) + ctx.ensemble_weight_points.extend(fit_result.ensemble) + ctx.contribution_points.extend(fit_result.contribution) + ctx.curve_points.extend(fit_result.curve) + ctx.calibration_offsets.extend(fit_result.calibration_offsets) + ctx.conditional_calibration_offsets.extend(fit_result.conditional_calibration_offsets) + ctx.threshold_etas.extend(fit_result.threshold_etas) + ctx.growth_rates.extend(fit_result.growth_rates) + ctx.will_breach_points.extend(fit_result.will_breach) + ctx.time_to_drain_points.extend(fit_result.time_to_drain) + ctx.error_budget_points.extend(fit_result.error_budget) + ctx.recommended_maintenance_window_points.extend(fit_result.recommended_windows) + ctx.conditional_points.extend(fit_result.conditional_points) + ctx.conditional_deviation_points.extend(fit_result.conditional_deviation_points) + ctx.conditional_curve_points.extend(fit_result.conditional_curve) + ctx.component_points.extend(fit_result.component_points) + ctx.component_curve_points.extend(fit_result.component_curve) + for reason, count in fit_result.conditional_failures.items(): + ctx.failures[reason] = ctx.failures.get(reason, 0) + count + for model_name, fit_seconds in fit_result.fit_durations.items(): + ctx.fit_durations[(query_id, model_name)] = fit_seconds + for model_name in fit_result.slow_fits: + key = (query_id, model_name) + ctx.slow_fits[key] = ctx.slow_fits.get(key, 0) + 1 + for regressor_id, reason in fit_result.regressor_failures: + rkey = (query_id, regressor_id, reason) + ctx.regressor_failures[rkey] = ctx.regressor_failures.get(rkey, 0) + 1 + for kind, count in fit_result.emission_clipped.items(): + ekey = (query_id, kind) + ctx.emission_clipped[ekey] = ctx.emission_clipped.get(ekey, 0) + count + if fit_result.cold_start_siblings > 0: + ctx.cold_start_siblings[query_id] = fit_result.cold_start_siblings + ctx.drift_points.extend(fit_result.drift_points) + if fit_result.drift_alerts: + ctx.drift_alerts[query_id] = ctx.drift_alerts.get(query_id, 0) + fit_result.drift_alerts + + +# Per-family inheritance registry. Each entry tells +# ``_inherit_prior_for_skipped_queries`` which prior-snapshot list to +# copy into the new context, keyed off either the point's ``id`` label +# or the query id encoded in its metric-name suffix. Adding a new +# emission family means appending one tuple here rather than editing a +# hand-coded copy block. +_KeyBy = Literal["label", "metric"] + + +@dataclass(frozen=True) +class _InheritanceSpec: + """One per-family rule for ``_inherit_prior_for_skipped_queries``.""" + + prior_attr: str + ctx_attr: str + by: _KeyBy + + +def _query_id_of(point: Any, by: _KeyBy) -> str: + """Extract the query id from ``point`` per the spec's ``by`` mode.""" + if by == "label": + labels = getattr(point, "labels", None) + if not labels: + return "" + return str(labels.get("id", "")) + return _query_id_from_metric(point.metric) + + +_INHERITANCE_REGISTRY: tuple[_InheritanceSpec, ...] = ( + # Lists keyed by the point's ``id`` label. + _InheritanceSpec("accuracy_points", "accuracy_points", "label"), + _InheritanceSpec("deviation_points", "deviation_points", "label"), + _InheritanceSpec("quality_points", "quality_points", "label"), + _InheritanceSpec("ensemble_weights", "ensemble_weight_points", "label"), + _InheritanceSpec("calibration_offsets", "calibration_offsets", "label"), + _InheritanceSpec("conditional_calibration_offsets", "conditional_calibration_offsets", "label"), + _InheritanceSpec("drift_points", "drift_points", "label"), + # ``forecast_recommended_maintenance_window`` is a global metric + # name (no per-id prefix); use the ``id`` label to scope inheritance. + _InheritanceSpec( + "recommended_maintenance_window_points", + "recommended_maintenance_window_points", + "label", + ), + # Conditional deviation point exposes ``id`` as a label, so the + # label-based extractor handles it correctly. + _InheritanceSpec("conditional_deviation_points", "conditional_deviation_points", "label"), + # Lists keyed by the metric-name suffix (which encodes the query id). + _InheritanceSpec("points", "points", "metric"), + _InheritanceSpec("contribution_points", "contribution_points", "metric"), + _InheritanceSpec("threshold_etas", "threshold_etas", "metric"), + _InheritanceSpec("growth_rates", "growth_rates", "metric"), + _InheritanceSpec("will_breach_points", "will_breach_points", "metric"), + _InheritanceSpec("time_to_drain_points", "time_to_drain_points", "metric"), + _InheritanceSpec("error_budget_points", "error_budget_points", "metric"), + _InheritanceSpec("conditional_points", "conditional_points", "metric"), + _InheritanceSpec("component_points", "component_points", "metric"), +) + + +def _inherit_prior_for_skipped_queries( + ctx: _RunContext, + prior: GroupSnapshot, + *, + run_query_ids: set[str], +) -> None: + """Preload ``ctx`` with prior-snapshot data for queries skipped this tick. + + With per-query refresh intervals, ``run_group`` may run only a + subset of the group's queries on any given tick. Without this + preload the snapshot would lose every other query's last-known + forecast / accuracy / deviation rows the next time the partial + snapshot is published. Counter-style fields (slow_fits, gaps_filled, + ...) are already merged with the prior via the explicit + counter-merge logic in :meth:`Runner.run_group`; here we only need + to copy snapshot-style (gauge / list) data. + + ``run_query_ids`` is the set of query IDs that *will* run this + tick; everything else gets its prior data inherited. ``curve_points`` + are intentionally *not* inherited because they drive the remote_write + sink — pushing previously-fit curves every refresh would duplicate + data in the TSDB. + + Per-family list inheritance is registry-driven via + :data:`_INHERITANCE_REGISTRY`; the structured dict-state + inheritance (per-query preprocessing stats, fit durations, + cold-start siblings, series counts, run durations) is split into + :func:`_inherit_prior_dict_state` because it's structurally + different — keys are tuples or strings, values include mutable + dataclasses that need defensive copying. + """ + for spec in _INHERITANCE_REGISTRY: + prior_list = getattr(prior, spec.prior_attr) + ctx_list = getattr(ctx, spec.ctx_attr) + for point in prior_list: + if _query_id_of(point, spec.by) not in run_query_ids: + ctx_list.append(point) + _inherit_prior_dict_state(ctx, prior, run_query_ids=run_query_ids) + + +def _inherit_prior_dict_state( + ctx: _RunContext, + prior: GroupSnapshot, + *, + run_query_ids: set[str], +) -> None: + """Carry forward per-(query, model) and per-query dict state. + + Unlike the list-based inheritance handled by the registry, these + are dict-shaped gauge-style accumulators whose keys are tuples or + strings rather than point objects. ``preprocess_stats`` carries a + mutable :class:`PreprocessStats` so it's defensively copied — a + follower serving ``/metrics`` would otherwise see torn state if a + future writer mutated the prior snapshot mid-render. + """ + for (q_id, model_name), fit_seconds in prior.fit_durations.items(): + if q_id not in run_query_ids: + ctx.fit_durations[(q_id, model_name)] = fit_seconds + for q_id, stats in prior.preprocess_stats.items(): + if q_id not in run_query_ids: + ctx.preprocess_stats[q_id] = PreprocessStats( + gaps_filled=dict(stats.gaps_filled), + max_gap_seconds=stats.max_gap_seconds, + outliers_replaced=dict(stats.outliers_replaced), + change_points=stats.change_points, + effective_lookback_seconds=stats.effective_lookback_seconds, + ) + for q_id, count in prior.cold_start_siblings.items(): + if q_id not in run_query_ids: + ctx.cold_start_siblings[q_id] = count + for q_id, dur in prior.run_duration_per_query.items(): + if q_id not in run_query_ids: + ctx.run_duration_per_query[q_id] = dur + for q_id, scount in prior.series_count_per_query.items(): + if q_id not in run_query_ids: + ctx.series_count_per_query[q_id] = scount diff --git a/forecaster/src/promforecast/runner/emission/__init__.py b/forecaster/src/promforecast/runner/emission/__init__.py new file mode 100644 index 0000000..a1ec974 --- /dev/null +++ b/forecaster/src/promforecast/runner/emission/__init__.py @@ -0,0 +1,9 @@ +"""Per-family emission helpers for the runner. + +Each module here owns the pure-function pieces of one emission family — +the parts that can be reasoned about without a live :class:`~promforecast.runner.Runner` +instance. Methods that need instance state (an executor, a config, an +exporter) stay on ``Runner`` itself in ``scheduler``; this package +collects the rest so a contributor extending an emission family touches +one focused file instead of scrolling through the orchestration layer. +""" diff --git a/forecaster/src/promforecast/runner/emission/calibration.py b/forecaster/src/promforecast/runner/emission/calibration.py new file mode 100644 index 0000000..cd7b879 --- /dev/null +++ b/forecaster/src/promforecast/runner/emission/calibration.py @@ -0,0 +1,149 @@ +"""Pure helpers for conformal calibration application. + +Conformal calibration is applied symmetrically to both the snapshot +``ForecastPoint`` list (consumed by /metrics) and the +``ForecastCurvePoint`` list (consumed by the remote_write sink). The two +carry the same (metric, labels, value) shape but ``ForecastCurvePoint`` +also has a ``timestamp_ms`` field. The decision logic — "if this is a +lower/upper row for a calibrated (model, level), widen and label it" — +is shared via :func:`_calibrated_band_value`; the per-type list rebuild +lives in the two thin wrappers so each wrapper is straight-line code +without a constructor callback. + +The fit-time application (``_compute_conformal_offsets`` and +``_compute_conditional_conformal_offsets``) lives on the ``Runner`` +class because it needs the per-instance fit executor / model registry. +This module collects only the post-fit pure-function pieces. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ...exporter import ForecastPoint +from ...sink import ForecastCurvePoint + +if TYPE_CHECKING: + import pandas as pd + + +def _calibrated_band_value( + metric: str, + labels: dict[str, str], + value: float, + offsets_by_model: dict[str, dict[int, float]], +) -> tuple[float, bool] | None: + """Return ``(new_value, calibrated)`` if the point is a calibratable band, else ``None``. + + ``None`` means "leave the point alone". ``(value, False)`` means the + offset is zero/missing for this (model, level), so the band is left as + the model emitted it; ``(value, True)`` means we widened and the + caller should add a ``calibrated="true"`` label. The yhat row (metric + exactly ending in ``_forecast``) is always returned as ``None`` — + calibration is a band adjustment, not a point-forecast adjustment. + """ + model_name = labels.get("model", "") + offsets = offsets_by_model.get(model_name) + if offsets is None or metric.endswith("_forecast"): + return None + try: + level = int(labels.get("level", "")) + except ValueError: + return None + offset = offsets.get(level) + if offset is None or offset <= 0: + return None + if metric.endswith("_forecast_lower"): + return value - offset, True + if metric.endswith("_forecast_upper"): + return value + offset, True + return None + + +def _apply_offsets_to_threshold_eta_inputs( + forecast: pd.DataFrame, + offsets: dict[int, float] | None, + levels: list[int], +) -> pd.DataFrame: + """Return a copy of ``forecast`` with band columns shifted by the per-level offset. + + The conformal calibration offset widens both sides of each band + symmetrically: ``yhat_upper_ += offset`` and + ``yhat_lower_ -= offset``. Re-applying that shift here so the + capacity ETA computation sees the same bands /metrics exposes keeps + ``time_to_threshold{level="95"}`` and ``yhat_upper{level="95"}`` + self-consistent. + + ``offsets=None`` or an empty dict is the common case (no calibration + ran for this model) and the frame is returned unmodified. Only + non-zero positive offsets actually mutate columns; a zero offset is + a no-op so we skip the copy. + """ + if not offsets or forecast.empty: + return forecast + if not any(o > 0 for o in offsets.values()): + return forecast + out = forecast.copy() + for level, offset in offsets.items(): + if offset <= 0 or level not in levels: + continue + lo_col = f"yhat_lower_{level}" + hi_col = f"yhat_upper_{level}" + if lo_col in out.columns: + out[lo_col] = out[lo_col].to_numpy(dtype=float) - offset + if hi_col in out.columns: + out[hi_col] = out[hi_col].to_numpy(dtype=float) + offset + return out + + +def _apply_offsets_to_snapshot_points( + points: list[ForecastPoint], + offsets_by_model: dict[str, dict[int, float]], +) -> list[ForecastPoint]: + """Widen ``ForecastPoint`` bands by the per-(model, level) offset. + + Returns a freshly constructed list; the input list is not mutated. + """ + if not offsets_by_model: + return points + out: list[ForecastPoint] = [] + for point in points: + adjusted = _calibrated_band_value(point.metric, point.labels, point.value, offsets_by_model) + if adjusted is None: + out.append(point) + continue + new_value, calibrated = adjusted + new_labels = {**point.labels, "calibrated": "true"} if calibrated else point.labels + out.append(ForecastPoint(metric=point.metric, labels=new_labels, value=new_value)) + return out + + +def _apply_offsets_to_curve_points( + points: list[ForecastCurvePoint], + offsets_by_model: dict[str, dict[int, float]], +) -> list[ForecastCurvePoint]: + """Widen ``ForecastCurvePoint`` bands by the per-(model, level) offset. + + Returns a freshly constructed list; the input list is not mutated. + Carries the original ``timestamp_ms`` through unchanged — calibration + shifts the band level, not when the forecast is for. + """ + if not offsets_by_model: + return points + out: list[ForecastCurvePoint] = [] + for point in points: + adjusted = _calibrated_band_value(point.metric, point.labels, point.value, offsets_by_model) + if adjusted is None: + out.append(point) + continue + new_value, calibrated = adjusted + new_labels = {**point.labels, "calibrated": "true"} if calibrated else point.labels + out.append( + ForecastCurvePoint( + metric=point.metric, + labels=new_labels, + value=new_value, + timestamp_ms=point.timestamp_ms, + ) + ) + return out diff --git a/forecaster/src/promforecast/runner/emission/conditional.py b/forecaster/src/promforecast/runner/emission/conditional.py new file mode 100644 index 0000000..6388b80 --- /dev/null +++ b/forecaster/src/promforecast/runner/emission/conditional.py @@ -0,0 +1,51 @@ +"""Pure helpers for conditional forecast emission. + +The full conditional refit / calibration orchestration lives on +``Runner`` because it needs the per-instance fit executor and exporter +references; this module collects the pure-function helpers shared with +the runner so the data manipulation pieces can be reasoned about in +isolation. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd + + +def _flatten_regressors_to_last_value( + regressors: pd.DataFrame | None, +) -> pd.DataFrame | None: + """Return a copy of ``regressors`` whose every row equals the last row. + + Implements the "held at current value" semantics of the conditional + forecast: each regressor column is replaced with a constant series + equal to its most recent in-sample value. Calendar features (which + legitimately vary per timestamp) are preserved through the same + flattening — the conditional contract is "imagine current state + continued unchanged", which by construction freezes the calendar as + well. Returns ``None`` when no regressors were configured so the + caller skips the re-fit's regressor argument cleanly. + + Uses scalar broadcast (``flat[col] = scalar``) instead of building a + Python list per column so a wide regressor frame doesn't materialise + N copies of the same value through the heap. pandas treats the + scalar assignment as a constant broadcast and writes a single typed + array. + """ + if regressors is None or regressors.empty: + return regressors + import pandas as pd # noqa: PLC0415 + + last_row = regressors.iloc[-1] + flat = pd.DataFrame(index=regressors.index) + for col in regressors.columns: + if col == "ds": + flat[col] = regressors[col].to_numpy() + continue + # Scalar broadcast: pandas writes a constant array of the right + # dtype without allocating a Python list of N references. + flat[col] = last_row[col] + return flat diff --git a/forecaster/src/promforecast/runner/emission/safeguards.py b/forecaster/src/promforecast/runner/emission/safeguards.py new file mode 100644 index 0000000..f5dc696 --- /dev/null +++ b/forecaster/src/promforecast/runner/emission/safeguards.py @@ -0,0 +1,182 @@ +"""Count-aware emission safeguards (clip-from-zero, round-to-integer). + +When a query's ``emission.clip_non_negative`` is True every emitted +forecast / band value is clamped at zero from below; when its +``emission.round_to_integer`` is True every value is rounded to the +nearest integer. Both are applied symmetrically to the snapshot +``ForecastPoint`` list (consumed by /metrics) and to the +``ForecastCurvePoint`` list (consumed by the remote_write sink) so +scrape and push see the same numbers. + +Counter semantics live in :func:`_classify_emission_correction` — +``kind="negative"`` ticks only when the raw value was actually below +zero (a real model-output anomaly worth surfacing), and +``kind="rounded"`` ticks only when the fractional component was at +least :data:`_ROUND_COUNTER_TRIGGER` (so the counter doesn't blow up +on every continuous emission that happens to be slightly non-integer). +""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING + +from ...exporter import ForecastPoint +from ...sink import ForecastCurvePoint + +if TYPE_CHECKING: + import pandas as pd + + +_ROUND_COUNTER_TRIGGER = 0.05 + + +def _classify_emission_correction( + raw: float, + *, + clip: bool, + rounder: bool, +) -> tuple[float, list[str]]: + """Return ``(adjusted_value, kinds)`` after applying the safeguards. + + ``kinds`` is a list of bounded tokens ``["negative", "rounded"]`` + that the caller folds into the per-query emission-clipped counter. + Each entry is recorded *once per emission* (a value that is both + negative and rounded triggers both counters); a value that didn't + need correction produces an empty list and contributes no counter + bumps. + + Non-finite values pass through unmodified — clipping or rounding a + NaN would surface the silent transformation as a confusing + /metrics line later. + """ + if not math.isfinite(raw): + return raw, [] + value = raw + kinds: list[str] = [] + if clip and value < 0.0: + value = 0.0 + kinds.append("negative") + if rounder: + rounded = float(round(value)) + if abs(value - rounded) >= _ROUND_COUNTER_TRIGGER: + kinds.append("rounded") + value = rounded + return value, kinds + + +def _apply_safeguards_to_snapshot_points( + points: list[ForecastPoint], + *, + clip: bool, + rounder: bool, + counter: dict[str, int], +) -> list[ForecastPoint]: + """Apply count-aware safeguards to ``ForecastPoint``s and bump ``counter``. + + Returns a freshly constructed list; the input list is not mutated. + ``counter`` is updated in-place with ``kind -> count`` so the + runner can fold it into the per-(query, kind) total without + threading a second return value. + """ + if not (clip or rounder): + return points + out: list[ForecastPoint] = [] + for point in points: + new_value, kinds = _classify_emission_correction(point.value, clip=clip, rounder=rounder) + for kind in kinds: + counter[kind] = counter.get(kind, 0) + 1 + if kinds or new_value != point.value: + out.append(ForecastPoint(metric=point.metric, labels=point.labels, value=new_value)) + else: + out.append(point) + return out + + +def _apply_safeguards_to_curve_points( + points: list[ForecastCurvePoint], + *, + clip: bool, + rounder: bool, +) -> list[ForecastCurvePoint]: + """Apply count-aware safeguards to ``ForecastCurvePoint``s. + + Counter bumps live with the snapshot pass — the runner emits both + families from the same fit and counting twice would double-bump + the operator-facing total. The curve pass therefore only mutates + values; it does not touch the counter. + """ + if not (clip or rounder): + return points + out: list[ForecastCurvePoint] = [] + for point in points: + new_value, _ = _classify_emission_correction(point.value, clip=clip, rounder=rounder) + if new_value != point.value: + out.append( + ForecastCurvePoint( + metric=point.metric, + labels=point.labels, + value=new_value, + timestamp_ms=point.timestamp_ms, + ) + ) + else: + out.append(point) + return out + + +def _apply_safeguards_to_forecast_frame( + forecast: pd.DataFrame, + *, + levels: list[int], + clip: bool, + rounder: bool, +) -> pd.DataFrame: + """Return a copy of ``forecast`` with count-aware safeguards applied. + + Used by the conditional emission path so the snapshot points, the + sink curve, and the deviation gauges built from the same frame all + see the same clipped/rounded values. Counter bumps live with the + unconditional pass — the conditional emission re-uses the same fit + semantics and counting twice would double-charge the per-(query, + kind) total. + + Uses numpy vectorisation so a typical horizon (288 rows by 5 columns + at default 2 levels) is one pass per column instead of 1440 + Python-level float conversions. + """ + if forecast.empty or not (clip or rounder): + return forecast + import numpy as np # noqa: PLC0415 + import pandas as pd # noqa: PLC0415 + + out = forecast.copy() + cols = ["yhat"] + [ + col for level in levels for col in (f"yhat_lower_{level}", f"yhat_upper_{level}") + ] + for col in cols: + if col not in out.columns: + continue + # Coerce to float so clip / round have well-defined semantics + # against pandas extension-array columns. ``errors="coerce"`` + # turns non-numeric entries into NaN, which both numpy ops + # tolerate and propagate — matching the Python-loop version's + # behaviour of passing such entries through unchanged. + arr = pd.to_numeric(out[col], errors="coerce").to_numpy(dtype=float) + # Pre-record the NaN mask so non-finite inputs round-trip + # unchanged: ``np.round`` of NaN is NaN already, but clipping + # would turn NaN into 0.0 under naive ``np.maximum`` semantics — + # we restore NaN at the end so the contract matches the scalar + # :func:`_classify_emission_correction` path. + nan_mask = ~np.isfinite(arr) + if clip: + # ``np.maximum`` preserves the input shape and dtype (unlike + # ``np.where`` whose typed return is multidim) — the column + # is 1-D so this is the right primitive here. + arr = np.maximum(arr, 0.0) + if rounder: + arr = np.round(arr) + if nan_mask.any(): + arr[nan_mask] = np.nan + out[col] = arr + return out diff --git a/forecaster/src/promforecast/validate.py b/forecaster/src/promforecast/validate.py index 3bd631e..bb0b2ae 100644 --- a/forecaster/src/promforecast/validate.py +++ b/forecaster/src/promforecast/validate.py @@ -357,6 +357,16 @@ def _estimate_cardinality(cfg: Config) -> CardinalityEstimate: # noqa: PLR0912, series_cap * emitted_models * (1 + 2 * n_levels) + series_cap * emitted_models * n_levels * 2 ) + # Conditional-band conformal calibration emits one + # ``forecast_conditional_band_calibration_offset`` per + # (series, model, level) on the *emitted* model set — + # same shape as the unconditional ``calibration_lines`` + # term, just on the conditional half. Only charged when + # ``accuracy.conformal.enabled`` is on; otherwise the + # conditional emission still runs but no calibration + # gauges land. + if cfg.defaults.accuracy.conformal.enabled: + conditional_lines += series_cap * emitted_models * n_levels if q.emission.decompose: decomposition_lines += series_cap * emitted_models * n_components # Preprocessing emits per-(group, query) telemetry. The label sets diff --git a/forecaster/tests/perf/__init__.py b/forecaster/tests/perf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/forecaster/tests/perf/test_query_id_lookup_bench.py b/forecaster/tests/perf/test_query_id_lookup_bench.py new file mode 100644 index 0000000..4cba66f --- /dev/null +++ b/forecaster/tests/perf/test_query_id_lookup_bench.py @@ -0,0 +1,110 @@ +"""Micro-benchmark for ``_query_id_from_metric``. + +Gated behind the ``benchmark`` marker so the default ``pytest`` invocation +skips it (the default ``addopts`` selector is ``-m 'not benchmark'``). Opt in +with ``pytest -m benchmark`` from a nightly CI job or local profiling. + +The workload mirrors ``_inherit_prior_for_skipped_queries``'s call shape: a +mix of every emission suffix the runner pushes through the helper, repeated +many times to amortise per-call setup. Comparison is against the original +linear-scan implementation (preserved here as ``_linear_scan_reference``). + +The threshold is intentionally loose: shared CI runners have wildly +variable single-core throughput and the assertion exists to catch a +*regression* (e.g. someone disables ``lru_cache`` by accident) rather +than to pin a specific speed-up factor. Locally the lru_cache-warmed +path measures multiples faster than the linear scan on a workload of +80-90 distinct metric names; the 1.1x floor catches the "cache stopped +working" failure mode without flaking on slow CI. If this assertion +ever fires spuriously, prefer raising the workload size (so the ratio +is computed over a longer wall-clock window) over loosening the +threshold further. +""" + +from __future__ import annotations + +import time + +import pytest + +from promforecast.runner import ( + _FORECAST_METRIC_SUFFIXES, + _query_id_from_metric, +) + +# Regression alarm — anything under 1.0x means the cache stopped working +# or someone reverted the length-bucket probe to the linear scan. Not a +# performance target. +_MIN_SPEEDUP = 1.1 + + +def _linear_scan_reference(metric_name: str) -> str: + """Reference implementation: the pre-v1.6.5 linear ``endswith`` scan.""" + for suffix in _FORECAST_METRIC_SUFFIXES: + if metric_name.endswith(suffix): + stem = metric_name[: -len(suffix)] + if stem: + return stem + return "" + + +def _build_workload(repeat: int = 4000) -> list[str]: + """Build a representative workload covering every emission family.""" + ids = ( + "node_cpu_busy_pct", + "node_filesystem_avail_bytes", + "kafka_consumer_lag", + "http_requests_rate", + "api_forecast_count", # legitimate id with "forecast" substring + ) + metrics: list[str] = [] + for qid in ids: + for suffix in _FORECAST_METRIC_SUFFIXES: + metrics.append(qid + suffix) + # Also include a few unknowns so the "no match" branch is exercised. + metrics.append("node_memory_used_bytes") + metrics.append("unrelated_metric_name") + return metrics * repeat + + +@pytest.mark.benchmark +def test_query_id_lookup_beats_linear_scan() -> None: + workload = _build_workload() + + # Warm both implementations equally so the timing isn't biased by + # CPython's first-call interpreter setup. Then clear the lru_cache + # so the measurement covers cache-miss work too, not just the + # steady-state hit cost. + for metric in workload[:50]: + _linear_scan_reference(metric) + _query_id_from_metric(metric) + _query_id_from_metric.cache_clear() + + start_linear = time.perf_counter() + for metric in workload: + _linear_scan_reference(metric) + elapsed_linear = time.perf_counter() - start_linear + + start_current = time.perf_counter() + for metric in workload: + _query_id_from_metric(metric) + elapsed_current = time.perf_counter() - start_current + + speedup = elapsed_linear / max(elapsed_current, 1e-9) + assert speedup >= _MIN_SPEEDUP, ( + f"current implementation should be >= {_MIN_SPEEDUP}x faster than the " + f"linear scan; got {speedup:.2f}x " + f"(linear={elapsed_linear * 1e3:.2f}ms, current={elapsed_current * 1e3:.2f}ms)" + ) + + +@pytest.mark.benchmark +def test_query_id_lookup_round_trip_matches_linear_scan() -> None: + """The current variant must agree with the linear scan on every workload row. + + Correctness is also covered indirectly by the broader runner test suite; + this benchmark-side check pins the equivalence under a workload shape + that exercises every suffix at least once. + """ + for metric in _build_workload(repeat=1): + assert _query_id_from_metric(metric) == _linear_scan_reference(metric), metric diff --git a/forecaster/tests/test_example_alerts_labelset.py b/forecaster/tests/test_example_alerts_labelset.py index d069765..defce56 100644 --- a/forecaster/tests/test_example_alerts_labelset.py +++ b/forecaster/tests/test_example_alerts_labelset.py @@ -478,4 +478,4 @@ def test_header_declares_sink_assumption() -> None: head = RULES_PATH.read_text(encoding="utf-8").splitlines()[:20] joined = "\n".join(head) assert "sink+VM" in joined or "sink + VM" in joined, joined - assert "docs/architecture/emission-paths.md" in joined, joined + assert "docs/reference/emission-paths.md" in joined, joined diff --git a/forecaster/tests/test_runner_conditional_decompose.py b/forecaster/tests/test_runner_conditional_decompose.py index 22096ab..a2e620f 100644 --- a/forecaster/tests/test_runner_conditional_decompose.py +++ b/forecaster/tests/test_runner_conditional_decompose.py @@ -570,6 +570,510 @@ def fit_predict( assert call_count["n"] == 2 +def _conformal_conditional_config( + *, models_list: list[str] | None = None, holdout_fraction: float = 0.2 +) -> Config: + """Helper: a config with conformal + conditional both enabled.""" + from promforecast.config import ConformalConfig, RegressorConfig # noqa: PLC0415 + + return Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(), + defaults=DefaultsConfig( + min_points=5, + models=models_list or ["RampModel"], + confidence_levels=[80], + horizon=timedelta(hours=2), + step=timedelta(minutes=5), + accuracy=AccuracyConfig( + evaluate=False, + conformal=ConformalConfig(enabled=True, holdout_fraction=holdout_fraction), + ), + ), + groups=[ + GroupConfig( + name="g", + queries=[ + QueryConfig( + id="m", + promql="up", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ) + ], + ) + ], + ) + + +@pytest.mark.asyncio +async def test_conditional_calibration_gauge_emits_when_both_features_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``forecast_conditional_band_calibration_offset`` must land for opted-in queries. + + Pre-v1.6.5 the conditional band reused the unconditional offsets, + so there was no separate gauge to expose the per-(model, level) + widening the conditional band actually received. The v1.6.5 story + introduces the dedicated calibration pass; verify the gauge family + shows up on /metrics whenever both ``accuracy.conformal.enabled`` + and ``emission.conditional`` are on for the query and at least one + emitted model is regressor-aware. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + class _RegressorAware(_RampForecastModel): + uses_regressors = True + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorAware()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _conformal_conditional_config() + # A longer series is needed so the conformal holdout split (20% of + # the lookback) leaves enough points on both sides for a meaningful + # quantile estimate. + source = _FakeSource({"up": [_frame(60)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "forecast_conditional_band_calibration_offset" in body, ( + f"expected dedicated conditional calibration gauge; body=\n{body[:1000]}" + ) + # Both the unconditional and conditional gauges should be present so + # the operator can compare the per-(model, level) widening. + assert "forecast_band_calibration_offset" in body + # Conditional band rows pick up ``calibrated="true"`` when fresh + # offsets were applied — mirrors the unconditional family's + # post-hoc labelling so a single dashboard filter + # (``calibrated="true"``) hits both. Without this label the + # operator can't distinguish a fresh-calibrated conditional band + # from one that fell back to inherited offsets purely from the + # labelset. + calibrated_conditional_bands = [ + line + for line in body.splitlines() + if line.startswith(("m_forecast_conditional_lower{", "m_forecast_conditional_upper{")) + and 'calibrated="true"' in line + ] + assert calibrated_conditional_bands, ( + f'expected conditional band rows to carry calibrated="true"; body=\n{body[:2000]}' + ) + # The yhat row (no band, no calibration) must NOT carry the label. + central_rows = [ + line for line in body.splitlines() if line.startswith("m_forecast_conditional{") + ] + assert central_rows, "expected at least one central conditional row" + for line in central_rows: + assert 'calibrated="true"' not in line, ( + f"calibrated label leaked onto central conditional row: {line}" + ) + + +@pytest.mark.asyncio +async def test_conditional_calibration_offsets_can_differ_from_unconditional( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The fresh conditional pass must produce its own offsets, not a copy of the unconditional. + + The point of the v1.6.5 conditional-band calibration is that the + holdout residuals are scored under flat-held regressors — + matching the production-time conditional inference contract — + rather than under the actual regressor trajectory the + unconditional pass uses. For a regressor-responsive model, the + two regimes produce different residual distributions, so the two + offset gauges must be able to carry different per-(model, level) + values. A test that asserts only "the gauge exists" misses the + regression where the runner silently aliases the unconditional + offsets back into the conditional family. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + class _RegressorSwingModel(_RampForecastModel): + """Calibration fit whose residuals depend on whether regressors vary. + + When the regressors frame's non-``ds`` columns vary across + rows (unconditional calibration), the predicted ``yhat`` is + offset by the regressor's row-by-row delta — so the + predictions track the actuals (small residuals, narrow + calibration offset). When the regressors are flat-held + (conditional calibration), the model predicts a constant — + so residuals fan out and the calibration offset has to + widen to match. + """ + + uses_regressors = True + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + idx = pd.date_range(df["ds"].iloc[-1], periods=horizon + 1, freq=freq)[1:] + # Base ramp matching the in-sample tail so unconditional + # residuals are tiny when regressors vary. + base = float(df["y"].iloc[-1]) if "y" in df.columns else 0.0 + ramp = [base + i * 0.5 for i in range(horizon)] + # Inject the regressor contribution row-by-row so a varying + # regressor *helps* the forecast follow the actual. + if regressors is not None and len(regressors) >= horizon: + non_ds = [c for c in regressors.columns if c != "ds"] + if non_ds: + contrib = regressors[non_ds[0]].to_numpy(dtype=float)[:horizon] + ramp = [r + float(c) for r, c in zip(ramp, contrib, strict=False)] + out = pd.DataFrame({"unique_id": ["s"] * horizon, "ds": idx, "yhat": ramp}) + for level in levels: + # Tight base band; calibration is what widens it to + # match real residuals. + out[f"yhat_lower_{level}"] = [v - 0.01 for v in ramp] + out[f"yhat_upper_{level}"] = [v + 0.01 for v in ramp] + return out + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorSwingModel()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _conformal_conditional_config() + # Need a long enough series so the holdout has enough samples for + # the quantile to discriminate between the two regimes. + source = _FakeSource({"up": [_frame(100)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + + def _value(line: str) -> float: + return float(line.rsplit(" ", 1)[-1].strip()) + + unconditional = { + line for line in body.splitlines() if line.startswith("forecast_band_calibration_offset{") + } + conditional = { + line + for line in body.splitlines() + if line.startswith("forecast_conditional_band_calibration_offset{") + } + assert unconditional and conditional, ( + f"expected both unconditional + conditional gauge families to emit; body=\n{body[:1500]}" + ) + + # The fresh conditional pass must produce its own offsets — at + # least one (model, level) pair must carry a value distinct from + # the unconditional reading. Otherwise the new gauge family + # would just duplicate the unconditional one and the v1.6.5 + # story's contract is silently broken. Compare per-(model, level) + # rather than across all rows so a coincidental match at one + # level doesn't mask a real divergence at another. + def _key_and_value(line: str, marker: str) -> tuple[str, float]: + """Strip the gauge name and metric value to leave just the labelset for keying.""" + head, _, _value_part = line.partition(" ") + # ``head`` is ``{labels}``; drop the metric prefix and + # any conformal-affected ``calibrated`` label so the two + # families key the same way. + labelset = head[len(marker) :] + return labelset, _value(line) + + unc_by_label = dict( + _key_and_value(line, "forecast_band_calibration_offset") for line in unconditional + ) + cond_by_label = dict( + _key_and_value(line, "forecast_conditional_band_calibration_offset") for line in conditional + ) + overlapping = unc_by_label.keys() & cond_by_label.keys() + assert overlapping, ( + f"expected at least one (model, level) pair to appear in both gauges; " + f"uncond keys={list(unc_by_label)[:3]} cond keys={list(cond_by_label)[:3]}" + ) + differing = [k for k in overlapping if abs(unc_by_label[k] - cond_by_label[k]) > 1e-6] + assert differing, ( + f"fresh conditional offsets must differ from unconditional for at least " + f"one (model, level); both gauges report {unc_by_label} / {cond_by_label}" + ) + + +@pytest.mark.asyncio +async def test_conditional_curve_carries_calibrated_label_when_fresh_offsets_apply( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The per-step conditional curve (sink path) must also carry the calibrated label. + + Without this the snapshot (scrape) and the curve (remote_write + push) would disagree on labelling: the snapshot's band rows + would carry ``calibrated="true"`` but the per-step curve + landing in the long-term TSDB wouldn't. Dashboards joining the + two by labelset would lose rows. The label must round-trip + through both emission paths. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.sink import ForecastCurvePoint, ForecastSink # noqa: PLC0415 + + captured: list[ForecastCurvePoint] = [] + + class _CapturingSink(ForecastSink): + async def write( + self, + points: list[ForecastCurvePoint], + *, + on_retry: object = None, + ) -> None: + captured.extend(points) + + async def aclose(self) -> None: + return None + + class _RegressorAware(_RampForecastModel): + uses_regressors = True + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorAware()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _conformal_conditional_config() + source = _FakeSource({"up": [_frame(60)]}) + runner = Runner( + config=cfg, + source=source, + exporter=Exporter(), + sink=_CapturingSink(), + ) + await runner.run_group(cfg.groups[0]) + + band_curve_rows = [ + p + for p in captured + if p.metric in {"m_forecast_conditional_lower", "m_forecast_conditional_upper"} + ] + assert band_curve_rows, ( + f"expected conditional band curve points in the sink push; " + f"got metrics={ {p.metric for p in captured} }" + ) + calibrated_band_rows = [p for p in band_curve_rows if p.labels.get("calibrated") == "true"] + assert calibrated_band_rows, ( + f'expected band curve rows to carry calibrated="true"; ' + f"sample labelsets={[p.labels for p in band_curve_rows[:3]]}" + ) + # The central conditional yhat curve must NOT carry the label + # — calibration shifts the band level, not the point forecast. + central_curve_rows = [p for p in captured if p.metric == "m_forecast_conditional"] + assert central_curve_rows, "expected at least one central conditional curve row" + for p in central_curve_rows: + assert p.labels.get("calibrated") != "true", ( + f"calibrated label leaked onto central curve row: {p.labels}" + ) + + +@pytest.mark.asyncio +async def test_conditional_band_uncalibrated_when_fresh_offsets_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No ``calibrated="true"`` label on the conditional band when fresh calibration fell back. + + Failure paths reuse the inherited unconditional offsets, but the + label is reserved for the *fresh* per-fit pass so an operator can + distinguish "this band was widened by the production-time + inference contract's calibration" from "this band inherited the + unconditional widening because fresh calibration didn't run". + """ + from promforecast import models as model_registry # noqa: PLC0415 + + class _ConditionalCalibFails(_RampForecastModel): + uses_regressors = True + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + is_train_prefix = len(df) < 60 + is_flat_held = ( + regressors is not None + and len(regressors) > 1 + and all( + regressors[col].nunique(dropna=False) == 1 + for col in regressors.columns + if col != "ds" + ) + ) + if is_train_prefix and is_flat_held: + raise RuntimeError("conditional calibration unavailable") + return super().fit_predict(df, horizon, freq, levels, regressors) + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _ConditionalCalibFails()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _conformal_conditional_config() + source = _FakeSource({"up": [_frame(60)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # Conditional bands are emitted (the snapshot still publishes), but + # they do NOT carry calibrated="true" — fresh calibration failed and + # the runner fell back to the inherited unconditional offsets. + conditional_band_rows = [ + line + for line in body.splitlines() + if line.startswith(("m_forecast_conditional_lower{", "m_forecast_conditional_upper{")) + ] + assert conditional_band_rows, ( + f"expected conditional band rows even when fresh calibration failed; body=\n{body[:1500]}" + ) + for line in conditional_band_rows: + assert 'calibrated="true"' not in line, ( + f"calibrated label should be absent on fallback path: {line}" + ) + + +@pytest.mark.asyncio +async def test_conditional_calibration_skipped_when_no_model_uses_regressors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No conditional gauge emitted when every emitted model ignores regressors. + + The fresh calibration would produce identical numbers to the + unconditional pass (the held-constant frame has no effect on a + model that doesn't read regressors), so emitting a parallel offset + gauge with the same values would just be noise. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RampForecastModel()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: False) + + cfg = _conformal_conditional_config() + source = _FakeSource({"up": [_frame(60)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + assert "forecast_conditional_band_calibration_offset" not in body + # Failure counter must also stay quiet — short-circuit isn't a failure. + assert "forecast_conditional_calibration_failures_total" not in body + + +@pytest.mark.asyncio +async def test_conditional_calibration_failure_bumps_counter_and_falls_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failing conditional-calibration fit must bump the failures counter and fall back. + + The runner runs an extra fit per emitted model to derive the + conditional offsets. If that fit raises, the snapshot should still + publish (with the band silently widened by the inherited + unconditional offsets), and the failure counter must tick with a + bounded ``reason``. + """ + from promforecast import models as model_registry # noqa: PLC0415 + + # Track which fit is which by inspecting the input frame's shape + # (training prefix is shorter than the full lookback) and whether the + # regressors frame is flat-held (every row equal). This is more + # robust than counting calls because the implementation may reorder + # or parallelise fits without changing what the test is verifying. + seen_flat_held_calibration = {"n": 0} + + class _ConditionalCalibFails(_RampForecastModel): + uses_regressors = True + + def fit_predict( + self, + df: pd.DataFrame, + horizon: int, + freq: str, + levels: list[int], + regressors: pd.DataFrame | None = None, + ) -> pd.DataFrame: + # The conditional calibration is the only fit whose + # ``regressors`` frame has its non-``ds`` columns flat-held + # at a single value AND whose ``df`` length is the + # train-prefix length (shorter than the full lookback). The + # conditional refit also gets a flat-held frame but on the + # full lookback; the unconditional calibration gets the + # train prefix but with varying regressors. So + # ``flat AND len(df) < 60`` uniquely identifies the + # conditional calibration call. + is_train_prefix = len(df) < 60 + is_flat_held = ( + regressors is not None + and len(regressors) > 1 + and all( + regressors[col].nunique(dropna=False) == 1 + for col in regressors.columns + if col != "ds" + ) + ) + if is_train_prefix and is_flat_held: + seen_flat_held_calibration["n"] += 1 + raise RuntimeError("conditional calibration blew up") + return super().fit_predict(df, horizon, freq, levels, regressors) + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _ConditionalCalibFails()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = _conformal_conditional_config() + source = _FakeSource({"up": [_frame(60)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + # The failures counter must tick with one of the bounded reasons. + failure_lines = [ + line + for line in body.splitlines() + if line.startswith("forecast_conditional_calibration_failures_total{") + and 'reason="fit_error"' in line + and line.rsplit(" ", 1)[-1].strip() != "0.0" + ] + assert failure_lines, ( + f"expected conditional calibration failure counter to tick; body=\n{body[:1500]}" + ) + # The dedicated conditional offset gauge must NOT be emitted for + # the failed model (the runner skipped it cleanly). + assert "forecast_conditional_band_calibration_offset" not in body + # The unconditional snapshot must still publish — failure isolation. + assert "m_forecast_conditional" in body + + +@pytest.mark.asyncio +async def test_conditional_calibration_degenerate_holdout_bumps_counter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A holdout too small to support a quantile must bump the counter with degenerate_holdout.""" + from promforecast import models as model_registry # noqa: PLC0415 + + class _RegressorAware(_RampForecastModel): + uses_regressors = True + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorAware()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + # 12-point series with the default 0.2 holdout fraction → holdout + # would be ~2 points, below the 5-point minimum in + # ``holdout_split``, so the conditional calibration short-circuits + # with a degenerate_holdout failure. + cfg = _conformal_conditional_config() + source = _FakeSource({"up": [_frame(12)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + await runner.run_group(cfg.groups[0]) + + body = exporter.render().decode() + degenerate_lines = [ + line + for line in body.splitlines() + if line.startswith("forecast_conditional_calibration_failures_total{") + and 'reason="degenerate_holdout"' in line + and line.rsplit(" ", 1)[-1].strip() != "0.0" + ] + assert degenerate_lines, ( + f"expected degenerate_holdout failure to be counted; body=\n{body[:1500]}" + ) + + @pytest.mark.asyncio async def test_decomposition_emission_off_by_default(monkeypatch: pytest.MonkeyPatch) -> None: """No component family until ``emission.decompose`` is flipped on.""" @@ -768,3 +1272,94 @@ class _CrostonStub(_RampForecastModel): assert 'query="m"' in body # Component family is *not* emitted when every emitted model skipped. assert "m_forecast_component" not in body + + +@pytest.mark.asyncio +async def test_conditional_calibration_offsets_inherit_on_skipped_query( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``forecast_conditional_band_calibration_offset`` rows must survive a partial tick. + + The v1.6.5 story registers the conditional calibration offsets in + ``_INHERITANCE_REGISTRY`` so per-query refresh cadences don't drop + them off /metrics when the opted-in query is skipped on a tick. The + registry entry is easy to drop in a future refactor; this test pins + the contract so a regression would fail loudly. Without this test, + a future contributor renaming or removing the conditional family + from the registry would silently strip the gauge from partial-tick + snapshots. + """ + from promforecast import models as model_registry # noqa: PLC0415 + from promforecast.config import ConformalConfig, RegressorConfig # noqa: PLC0415 + + class _RegressorAware(_RampForecastModel): + uses_regressors = True + + monkeypatch.setattr(model_registry, "build", lambda *a, **kw: _RegressorAware()) + monkeypatch.setattr(model_registry, "uses_regressors", lambda name: True) + + cfg = Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(), + defaults=DefaultsConfig( + min_points=5, + models=["RampModel"], + confidence_levels=[80], + horizon=timedelta(hours=2), + step=timedelta(minutes=5), + accuracy=AccuracyConfig( + evaluate=False, + conformal=ConformalConfig(enabled=True, holdout_fraction=0.2), + ), + ), + groups=[ + GroupConfig( + name="g", + queries=[ + QueryConfig(id="active_query", promql="qa"), + QueryConfig( + id="conditional_query", + promql="qb", + regressors=[RegressorConfig(id="hour_of_day", type="calendar")], + emission=QueryEmissionConfig(conditional=True), + ), + ], + ) + ], + ) + source = _FakeSource({"qa": [_frame(60)], "qb": [_frame(60)]}) + runner = Runner(config=cfg, source=source, exporter=(exporter := Exporter())) + + # First, a full run so the conditional calibration gauge lands for + # ``conditional_query`` and the snapshot has data to inherit from. + await runner.run_group(cfg.groups[0]) + initial = exporter.render().decode() + initial_offset_lines = [ + line + for line in initial.splitlines() + if line.startswith("forecast_conditional_band_calibration_offset{") + and 'id="conditional_query"' in line + ] + assert initial_offset_lines, ( + f"initial full run must emit the conditional calibration gauge; body=\n{initial[:1500]}" + ) + + # Partial tick that runs *only* ``active_query``. The conditional + # calibration rows for ``conditional_query`` must survive via the + # inheritance registry; without the registry entry the next /metrics + # scrape would silently drop them. + await runner.run_group(cfg.groups[0], queries=[cfg.groups[0].queries[0]]) + partial = exporter.render().decode() + partial_offset_lines = [ + line + for line in partial.splitlines() + if line.startswith("forecast_conditional_band_calibration_offset{") + and 'id="conditional_query"' in line + ] + assert partial_offset_lines, ( + "conditional_band_calibration_offset rows for the skipped query must " + "be inherited via _INHERITANCE_REGISTRY; if this fails, check that " + "conditional_calibration_offsets is still registered.\n" + f"partial body excerpt:\n{partial[:1500]}" + ) diff --git a/forecaster/tests/test_runner_inheritance_registry.py b/forecaster/tests/test_runner_inheritance_registry.py new file mode 100644 index 0000000..08e6bdc --- /dev/null +++ b/forecaster/tests/test_runner_inheritance_registry.py @@ -0,0 +1,160 @@ +"""Meta-tests for the per-family snapshot inheritance registry. + +When a partial tick runs only a subset of a group's queries (per-query +refresh cadences), :func:`promforecast.runner._inherit_prior_for_skipped_queries` +walks :data:`promforecast.runner._INHERITANCE_REGISTRY` and copies the +prior snapshot's rows for any query that's not scheduled this tick. The +registry is the single source of truth — adding a new emission family +means appending one ``_InheritanceSpec`` here, and *forgetting* to add +that one line silently strips the new family from partial-tick +snapshots. + +These meta-tests fail loudly when: + +* a new ``list[...Point]`` field is added to :class:`GroupSnapshot` + without a corresponding registry entry, or +* a registered entry points at a field that no longer exists on either + :class:`GroupSnapshot` or :class:`_RunContext`. + +Without this guard, the registry-driven dispatch the v1.6.5 refactor +introduced trades the pre-split hand-coded copy block for a different +silent-regression vector: a missing registry line drops the family +from partial-tick exposition until the next full run. Catching it at +test time (and naming the offending field) is the lowest-cost defence. +""" + +from __future__ import annotations + +import dataclasses +import typing + +from promforecast import exporter as exporter_mod +from promforecast import runner as runner_mod + + +def _is_point_list_field(field: dataclasses.Field[object]) -> bool: + """Return True if ``field`` looks like a snapshot point list. + + We accept any ``list[...]`` annotation; the registry is keyed off the + attribute name, not the contained type, so a future point type that + doesn't sit in ``promforecast.exporter`` would still need to register. + """ + type_str = str(field.type) + # Defensive: ``from __future__ import annotations`` keeps annotations + # as strings, so we match the textual form rather than calling + # ``typing.get_origin`` which would require resolving the forward + # references first. + return type_str.startswith("list[") + + +def _snapshot_list_field_names() -> set[str]: + fields = dataclasses.fields(exporter_mod.GroupSnapshot) + return {f.name for f in fields if _is_point_list_field(f)} + + +def _registry_prior_attrs() -> set[str]: + return {spec.prior_attr for spec in runner_mod._INHERITANCE_REGISTRY} + + +def _registry_ctx_attrs() -> set[str]: + return {spec.ctx_attr for spec in runner_mod._INHERITANCE_REGISTRY} + + +# Fields on ``GroupSnapshot`` that are intentionally NOT inherited via the +# registry. Each entry needs a one-line justification so future +# maintainers don't expand the whitelist by reflex. +_EXCLUDED_FROM_INHERITANCE: dict[str, str] = { + # Rebuilt fresh on every tick from ``Runner._band_coverage`` (a + # rolling tracker that survives across runs); inheriting the prior + # snapshot's frozen samples would double-count observations. + "band_coverage_points": ( + "BandCoveragePoint rows are rebuilt from the persistent " + "BandCoverageTracker on every tick, not inherited." + ), +} + + +def test_inheritance_registry_covers_every_snapshot_list_field() -> None: + """Every ``list[...]`` field on ``GroupSnapshot`` must either be + registered for inheritance or explicitly whitelisted with a reason. + + If this fails, the named field needs one of: + * a new ``_InheritanceSpec`` entry in ``_INHERITANCE_REGISTRY``, or + * an entry in ``_EXCLUDED_FROM_INHERITANCE`` with a one-line + justification (e.g. "rebuilt from a tracker every tick"). + """ + snapshot_fields = _snapshot_list_field_names() + registered = _registry_prior_attrs() + expected_handled = registered | set(_EXCLUDED_FROM_INHERITANCE) + missing = snapshot_fields - expected_handled + assert not missing, ( + "GroupSnapshot list fields without an inheritance registry entry: " + f"{sorted(missing)}. Add a `_InheritanceSpec` in " + "`promforecast.runner._INHERITANCE_REGISTRY` so partial-tick " + "snapshots don't drop the family, or add an entry in " + "`_EXCLUDED_FROM_INHERITANCE` with the reason." + ) + + +def test_inheritance_registry_targets_real_run_context_attrs() -> None: + """Each registry entry's ``ctx_attr`` must exist on ``_RunContext``. + + Catches a typo or rename where the registry's target field no longer + exists — the inheritance loop would otherwise raise ``AttributeError`` + at runtime on the first partial tick instead of at test time. + """ + ctx_fields = {f.name for f in dataclasses.fields(runner_mod._RunContext)} + targeted = _registry_ctx_attrs() + missing = targeted - ctx_fields + assert not missing, ( + f"_INHERITANCE_REGISTRY ctx_attr values not present on _RunContext: {sorted(missing)}." + ) + + +def test_inheritance_registry_sources_from_real_snapshot_attrs() -> None: + """Each registry entry's ``prior_attr`` must exist on ``GroupSnapshot``.""" + snapshot_fields = {f.name for f in dataclasses.fields(exporter_mod.GroupSnapshot)} + sourced = _registry_prior_attrs() + missing = sourced - snapshot_fields + assert not missing, ( + f"_INHERITANCE_REGISTRY prior_attr values not present on GroupSnapshot: {sorted(missing)}." + ) + + +def test_inheritance_registry_keyby_modes_are_known() -> None: + """Defensive: ``_query_id_of`` dispatches on ``by`` — only ``label`` and + ``metric`` are wired up today. A new mode means a corresponding branch + in :func:`promforecast.runner._state._query_id_of` must land too. + """ + valid = {"label", "metric"} + actual = {spec.by for spec in runner_mod._INHERITANCE_REGISTRY} + invalid = actual - valid + assert not invalid, ( + f"unknown ``by`` modes in _INHERITANCE_REGISTRY: {sorted(invalid)}. " + f"If a new dispatch mode is needed, also extend _query_id_of." + ) + + +def test_typing_get_origin_path_for_list_detection_stays_consistent() -> None: + """Sanity check: dataclasses surface ``list[...]`` annotations as the + same string form across Python 3.12 builds. If the textual form + changes (e.g. ``typing.List`` instead of ``list``) the detection + helper above silently misses fields. + + The check is local to the helper; we just verify it correctly + classifies a known list field and a known non-list field. + """ + name_to_field = {f.name: f for f in dataclasses.fields(exporter_mod.GroupSnapshot)} + # ``points`` is a list field. + assert _is_point_list_field(name_to_field["points"]), ( + "GroupSnapshot.points should be detected as a list field; " + "if this fails the field-detection helper is broken." + ) + # ``group`` is a scalar str field. + assert not _is_point_list_field(name_to_field["group"]), ( + "GroupSnapshot.group should NOT be detected as a list field." + ) + # And typing.get_origin still resolves list properly when given a + # concrete annotation — useful as documentation of the expected + # invariant if Python's annotation handling evolves. + assert typing.get_origin(list[int]) is list diff --git a/forecaster/tests/test_validate_cli.py b/forecaster/tests/test_validate_cli.py index c44de8c..224a5d0 100644 --- a/forecaster/tests/test_validate_cli.py +++ b/forecaster/tests/test_validate_cli.py @@ -199,6 +199,90 @@ def test_cardinality_estimate_counts_threshold_and_growth_rate_lines() -> None: assert est.growth_rate_lines == 100 * 1 * 2 +def test_cardinality_estimate_counts_conditional_calibration_when_conformal_on() -> None: + """The dedicated conditional-band conformal gauge must show up in the estimate. + + Each (series, model, level) emits one + ``forecast_conditional_band_calibration_offset`` row when both + ``accuracy.conformal.enabled`` and ``emission.conditional`` are + true; ``promforecast validate`` must account for it or the + estimate under-reports the post-v1.6.5 footprint. + """ + from datetime import timedelta # noqa: PLC0415 + + from promforecast.config import ( # noqa: PLC0415 + AccuracyConfig, + Config, + ConformalConfig, + DatasourceConfig, + DefaultsConfig, + EmissionConfig, + GroupConfig, + QueryConfig, + QueryEmissionConfig, + SafetyConfig, + ServerConfig, + ) + + series_cap = 100 + models = 2 + levels = 2 + + def _build(*, conformal_enabled: bool) -> Config: + return Config( + datasource=DatasourceConfig(url="http://x"), + server=ServerConfig(), + safety=SafetyConfig(max_series_per_query=series_cap, max_total_series=0), + defaults=DefaultsConfig( + models=["AutoARIMA", "SeasonalNaive"], + confidence_levels=[80, 95], + accuracy=AccuracyConfig( + evaluate=False, + auto_select=False, + horizons=[timedelta(hours=1)], + conformal=ConformalConfig(enabled=conformal_enabled), + ), + emission=EmissionConfig(deviation=False, quality=False), + ), + groups=[ + GroupConfig( + name="g", + queries=[ + QueryConfig( + id="m", + promql="up", + emission=QueryEmissionConfig(conditional=True), + ) + ], + ) + ], + ) + + # Conditional snapshot + deviation are charged either way (the + # baseline below validates that side of the formula). + snapshot_rows = series_cap * models * (1 + 2 * levels) + deviation_rows = series_cap * models * levels * 2 + calibration_rows = series_cap * models * levels + + off = _estimate_cardinality(_build(conformal_enabled=False)) + assert off.conditional_lines == snapshot_rows + deviation_rows + # No calibration gauge when conformal is off — even with conditional on. + assert off.calibration_lines == 0 + + on = _estimate_cardinality(_build(conformal_enabled=True)) + # With conformal on, the unconditional calibration gauge appears + # for every emitted (series, model, level) too — that's the + # pre-v1.6.5 line — *and* the conditional family picks up the + # matching new gauge. + assert on.calibration_lines == series_cap * models * levels # unconditional + assert on.conditional_lines == snapshot_rows + deviation_rows + calibration_rows + # Delta between off and on for the conditional family is exactly + # the new gauge — protects against drift if a future change + # widens the conditional emission and the test author forgets to + # update the formula here. + assert on.conditional_lines - off.conditional_lines == calibration_rows + + def test_validate_strict_cardinality_passes_when_under_budget(tmp_path: Path) -> None: """A config whose estimated total fits inside the cap exits 0 in strict mode.""" path = _write(tmp_path, _GOOD_CONFIG)