From ca0019df94d3c6fa6edbe6a0c88da1a793b9156a Mon Sep 17 00:00:00 2001 From: Werner Dijkerman Date: Sat, 23 May 2026 12:56:17 +0200 Subject: [PATCH] Added very basic load generator to validate setup --- Makefile | 16 +- .../grafana/promforecast-demo-tour.json | 333 ++++++++++++++++++ .../grafana/promforecast-is-it-working.json | 277 +++++++++++++++ docker-compose.dev.yml | 46 ++- docker/dev-config.yaml | 110 +++++- .../provisioning/datasources/datasources.yml | 6 +- docker/http-demo/Dockerfile.app | 17 + docker/http-demo/Dockerfile.loadgen | 13 + docker/http-demo/app.py | 130 +++++++ docker/http-demo/loadgen.py | 55 +++ docker/prometheus.yml | 16 + docs/README.md | 1 + docs/reading-the-numbers.md | 91 +++++ 13 files changed, 1103 insertions(+), 8 deletions(-) create mode 100644 dashboards/grafana/promforecast-demo-tour.json create mode 100644 dashboards/grafana/promforecast-is-it-working.json create mode 100644 docker/http-demo/Dockerfile.app create mode 100644 docker/http-demo/Dockerfile.loadgen create mode 100644 docker/http-demo/app.py create mode 100644 docker/http-demo/loadgen.py create mode 100644 docs/reading-the-numbers.md diff --git a/Makefile b/Makefile index 8078ed3..8a6d409 100644 --- a/Makefile +++ b/Makefile @@ -82,12 +82,16 @@ sys.exit(f'alert drift: chart {sorted(ch-ex) or \"-\"}, example {sorted(ex-ch) o # ---------- Local dev ---------- .PHONY: dev-up -dev-up: ## Start VictoriaMetrics + node-exporter + Prometheus + Redis + forecaster locally. +dev-up: ## Start the minimum dev stack: VM, Prometheus, Redis, node-exporter, forecaster. docker compose -f docker-compose.dev.yml up -d --build +.PHONY: dev-up-load +dev-up-load: ## Start the full demo stack: minimum + http-demo + loadgen + Grafana. + docker compose -f docker-compose.dev.yml --profile load up -d --build + .PHONY: dev-down -dev-down: ## Stop the local dev stack. - docker compose -f docker-compose.dev.yml down -v +dev-down: ## Stop the local dev stack (any profile). + docker compose -f docker-compose.dev.yml --profile load down -v .PHONY: dev-validate dev-validate: ## Sanity-check the running dev stack: forecaster fits and exposes all metric families. @@ -111,7 +115,11 @@ exit('!! forecast_query_cache_events_total{kind=\"miss\"} is zero -- the cache i r=json.loads(u.urlopen('http://localhost:9090/api/v1/query?query=forecast_accuracy_mape').read())['data']['result']; \ print(f' forecast_accuracy_mape: {len(r)} series'); \ exit('!! Prometheus has 0 series - check the prometheus scrape config and forecaster reachability') if not r else None" - @echo ">> dashboards reachable in Grafana at http://localhost:3000 (admin / admin)" + @if docker compose -f docker-compose.dev.yml ps --status running --services 2>/dev/null | grep -qx grafana; then \ + echo ">> dashboards reachable in Grafana at http://localhost:3000 (admin / admin)"; \ + else \ + echo ">> Grafana not running (minimum dev stack). Run 'make dev-up-load' to start the demo stack with Grafana + http-demo + loadgen."; \ + fi .PHONY: load-test load-test: ## Synthetic high-cardinality run for safety-control validation. diff --git a/dashboards/grafana/promforecast-demo-tour.json b/dashboards/grafana/promforecast-demo-tour.json new file mode 100644 index 0000000..f563b27 --- /dev/null +++ b/dashboards/grafana/promforecast-demo-tour.json @@ -0,0 +1,333 @@ +{ + "title": "promforecast — Demo tour (HTTP + host)", + "uid": "promforecast-demo-tour", + "description": "Local-dev tour dashboard. Plots raw HTTP and host metrics next to the forecaster's predictions. The forecaster runs in snapshot mode (no remote_write sink yet), so each panel overlays `actual now` with `forecast(offset = horizon)` so the predicted value made 30 minutes ago aligns on the X axis with what actually happened. Quality / accuracy / deviation panels show how well the model is doing.", + "schemaVersion": 39, + "version": 1, + "editable": true, + "tags": ["promforecast", "demo", "http", "node"], + "timezone": "", + "time": { "from": "now-2h", "to": "now+30m" }, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { "text": "VictoriaMetrics", "value": "victoriametrics" } + }, + { + "name": "endpoint", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(http_requests_rate_forecast, endpoint)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "model", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(http_requests_rate_forecast, model)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "AutoARIMA", "value": "AutoARIMA" } + } + ] + }, + "annotations": { + "list": [ + { + "name": "Outside 95% band", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "enable": true, + "iconColor": "red", + "expr": "forecast_deviation_outside_band{level=\"95\"} == 1", + "titleFormat": "Outside 95% band", + "tagKeys": "id,group,model,endpoint,instance,device" + } + ] + }, + "panels": [ + { "type": "row", "id": 100, "title": "HTTP — actual vs predicted (offset = horizon)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false }, + + { + "type": "timeseries", + "id": 1, + "title": "Request rate per endpoint — actual vs +30m forecast", + "description": "Solid lines: live rate(http_requests_total). Dashed lines: forecast value emitted 30m ago (). When they track, the model is calibrated.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "targets": [ + { "refId": "A", "expr": "sum by (endpoint) (rate(http_requests_total{endpoint=~\"$endpoint\"}[2m]))", "legendFormat": "actual {{endpoint}}" }, + { "refId": "B", "expr": "http_requests_rate_forecast{endpoint=~\"$endpoint\",model=\"$model\"} ", "legendFormat": "forecast {{endpoint}}" } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 5 } }, + "overrides": [ + { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } + ] + } + }, + + { + "type": "timeseries", + "id": 2, + "title": "p95 latency per endpoint — actual vs +30m forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "targets": [ + { "refId": "A", "expr": "histogram_quantile(0.95, sum by (endpoint, le) (rate(http_request_duration_seconds_bucket{endpoint=~\"$endpoint\"}[2m])))", "legendFormat": "actual p95 {{endpoint}}" }, + { "refId": "B", "expr": "http_request_p95_seconds_forecast{endpoint=~\"$endpoint\",model=\"$model\"} ", "legendFormat": "forecast p95 {{endpoint}}" } + ], + "fieldConfig": { + "defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 0 } }, + "overrides": [ + { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } + ] + } + }, + + { + "type": "timeseries", + "id": 3, + "title": "Request rate — forecast band (95%)", + "description": "Forecast lower/upper bounds at the 95% confidence level. When the live rate exits this band you'll see the red annotation above.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "targets": [ + { "refId": "A", "expr": "sum by (endpoint) (rate(http_requests_total{endpoint=~\"$endpoint\"}[2m]))", "legendFormat": "actual {{endpoint}}" }, + { "refId": "B", "expr": "http_requests_rate_forecast_lower{endpoint=~\"$endpoint\",model=\"$model\",level=\"95\"} ", "legendFormat": "lower {{endpoint}}" }, + { "refId": "C", "expr": "http_requests_rate_forecast_upper{endpoint=~\"$endpoint\",model=\"$model\",level=\"95\"} ", "legendFormat": "upper {{endpoint}}" } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 1, "fillOpacity": 10 } }, + "overrides": [ + { "matcher": { "id": "byRegexp", "options": "^lower .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dot" } }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }, + { "matcher": { "id": "byRegexp", "options": "^upper .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dot" } }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] } + ] + } + }, + + { + "type": "timeseries", + "id": 4, + "title": "Errors (5xx) per endpoint — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "targets": [ + { "refId": "A", "expr": "sum by (endpoint) (rate(http_requests_total{endpoint=~\"$endpoint\",status=~\"5..\"}[2m]))", "legendFormat": "actual {{endpoint}}" }, + { "refId": "B", "expr": "http_errors_rate_forecast{endpoint=~\"$endpoint\",model=\"$model\"} ", "legendFormat": "forecast {{endpoint}}" } + ], + "fieldConfig": { + "defaults": { "unit": "reqps" }, + "overrides": [ + { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } + ] + } + }, + + { "type": "row", "id": 200, "title": "Host — CPU, memory, load, disk, network", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "collapsed": false }, + + { + "type": "timeseries", + "id": 10, + "title": "CPU busy % — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 18 }, + "targets": [ + { "refId": "A", "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[2m])) * 100)", "legendFormat": "actual {{instance}}" }, + { "refId": "B", "expr": "node_cpu_busy_pct_forecast{model=\"$model\"} ", "legendFormat": "forecast {{instance}}" } + ], + "fieldConfig": { + "defaults": { "unit": "percent" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { + "type": "timeseries", + "id": 11, + "title": "Memory used — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 18 }, + "targets": [ + { "refId": "A", "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "actual {{instance}}" }, + { "refId": "B", "expr": "node_memory_used_bytes_forecast{model=\"$model\"} ", "legendFormat": "forecast {{instance}}" } + ], + "fieldConfig": { + "defaults": { "unit": "bytes" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { + "type": "timeseries", + "id": 12, + "title": "Load avg (1m, 5m) — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 18 }, + "targets": [ + { "refId": "A", "expr": "node_load1", "legendFormat": "actual load1 {{instance}}" }, + { "refId": "B", "expr": "node_load1_forecast{model=\"$model\"} ", "legendFormat": "forecast load1 {{instance}}" }, + { "refId": "C", "expr": "node_load5", "legendFormat": "actual load5 {{instance}}" }, + { "refId": "D", "expr": "node_load5_forecast{model=\"$model\"} ", "legendFormat": "forecast load5 {{instance}}" } + ], + "fieldConfig": { + "defaults": { "unit": "none" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { + "type": "timeseries", + "id": 13, + "title": "Filesystem used % — actual vs forecast", + "description": "Per-mountpoint. If forecast line rises, the capacity-planning alerts will fire before you run out of disk.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 25 }, + "targets": [ + { "refId": "A", "expr": "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|ramfs\"})", "legendFormat": "actual {{mountpoint}}" }, + { "refId": "B", "expr": "node_filesystem_used_pct_forecast{model=\"$model\"} ", "legendFormat": "forecast {{mountpoint}}" } + ], + "fieldConfig": { + "defaults": { "unit": "percent" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { + "type": "timeseries", + "id": 14, + "title": "Disk I/O — read & write bytes/s — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 }, + "targets": [ + { "refId": "A", "expr": "rate(node_disk_read_bytes_total[2m])", "legendFormat": "actual read {{device}}" }, + { "refId": "B", "expr": "node_disk_read_bytes_rate_forecast{model=\"$model\"} ", "legendFormat": "forecast read {{device}}" }, + { "refId": "C", "expr": "rate(node_disk_written_bytes_total[2m])", "legendFormat": "actual write {{device}}" }, + { "refId": "D", "expr": "node_disk_written_bytes_rate_forecast{model=\"$model\"} ", "legendFormat": "forecast write {{device}}" } + ], + "fieldConfig": { + "defaults": { "unit": "Bps" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { + "type": "timeseries", + "id": 15, + "title": "Network — rx & tx bytes/s — actual vs forecast", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 32 }, + "targets": [ + { "refId": "A", "expr": "rate(node_network_receive_bytes_total{device!~\"lo|docker.*|veth.*|br-.*\"}[2m])", "legendFormat": "actual rx {{device}}" }, + { "refId": "B", "expr": "node_network_receive_bytes_rate_forecast{model=\"$model\"} ", "legendFormat": "forecast rx {{device}}" }, + { "refId": "C", "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|docker.*|veth.*|br-.*\"}[2m])", "legendFormat": "actual tx {{device}}" }, + { "refId": "D", "expr": "node_network_transmit_bytes_rate_forecast{model=\"$model\"} ", "legendFormat": "forecast tx {{device}}" } + ], + "fieldConfig": { + "defaults": { "unit": "Bps" }, + "overrides": [ { "matcher": { "id": "byRegexp", "options": "^forecast .*$" }, "properties": [ { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [8, 4] } } ] } ] + } + }, + + { "type": "row", "id": 300, "title": "Forecast quality, accuracy & deviation", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, "collapsed": false }, + + { + "type": "timeseries", + "id": 20, + "title": "Quality score by group / id (0..1)", + "description": "Composite reliability score per series. Combines backtest MAPE, band width vs actual, and data coverage. Low score = don't trust this forecast.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "targets": [ + { "refId": "A", "expr": "forecast_quality_score{model=\"$model\"}", "legendFormat": "{{group}} / {{id}}" } + ], + "fieldConfig": { "defaults": { "unit": "none", "min": 0, "max": 1 } } + }, + + { + "type": "timeseries", + "id": 21, + "title": "Backtest MAPE per horizon (%)", + "description": "Mean absolute percentage error from the rolling backtest. Lower is better. Dev config evaluates at horizons 3m / 5m / 15m.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "targets": [ + { "refId": "A", "expr": "forecast_accuracy_mape{model=\"$model\"}", "legendFormat": "{{group}} / {{id}} ({{horizon}})" } + ], + "fieldConfig": { "defaults": { "unit": "percent" } } + }, + + { + "type": "timeseries", + "id": 22, + "title": "Deviation ratio: (actual - yhat) / band-width", + "description": "0 means actual is on the forecast central line; ±1 means actual is at the edge of the band; |ratio| > 1 means outside the band.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, + "targets": [ + { "refId": "A", "expr": "forecast_deviation_ratio{model=\"$model\",level=\"95\"}", "legendFormat": "{{group}} / {{id}}" } + ], + "fieldConfig": { + "defaults": { "unit": "none" }, + "overrides": [] + } + }, + + { + "type": "stat", + "id": 23, + "title": "Series count by group", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 48 }, + "targets": [ + { "refId": "A", "expr": "forecast_series_count", "legendFormat": "{{group}}" } + ], + "fieldConfig": { "defaults": { "unit": "short" } } + }, + + { + "type": "stat", + "id": 24, + "title": "Last successful run (per group)", + "description": "time() - forecast_last_run_timestamp_seconds. Should stay under refresh_interval (1m in dev).", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 48 }, + "targets": [ + { "refId": "A", "expr": "time() - forecast_last_run_timestamp_seconds", "legendFormat": "{{group}}" } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 120 }, + { "color": "red", "value": 600 } + ] + } + } + } + }, + + { + "type": "timeseries", + "id": 25, + "title": "Run duration & failures (per group)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 52 }, + "targets": [ + { "refId": "A", "expr": "forecast_run_duration_seconds", "legendFormat": "duration {{group}}" }, + { "refId": "B", "expr": "rate(forecast_failures_total[5m])", "legendFormat": "failures/s {{group}} ({{reason}})" } + ], + "fieldConfig": { "defaults": { "unit": "s" } } + } + ] +} diff --git a/dashboards/grafana/promforecast-is-it-working.json b/dashboards/grafana/promforecast-is-it-working.json new file mode 100644 index 0000000..250fa9a --- /dev/null +++ b/dashboards/grafana/promforecast-is-it-working.json @@ -0,0 +1,277 @@ +{ + "title": "promforecast — Is it working?", + "uid": "promforecast-is-it-working", + "description": "Single-screen answer to 'are the forecasts tracking actuals?'. Scoreboard at the top (quality, coverage, freshness, failures), signature actual-vs-forecast-band panels in the middle, tracking error and quality trend at the bottom. For deep-dive, use the 'Demo tour' dashboard.", + "schemaVersion": 39, + "version": 1, + "editable": true, + "tags": ["promforecast", "overview", "health"], + "timezone": "", + "time": { "from": "now-1h", "to": "now+30m" }, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { "text": "VictoriaMetrics", "value": "victoriametrics" } + }, + { + "name": "signature_endpoint", + "label": "HTTP signature endpoint", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(http_requests_rate_forecast, endpoint)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "/api/users", "value": "/api/users" } + } + ] + }, + "panels": [ + { + "type": "stat", + "id": 1, + "title": "Quality score (avg per group)", + "description": "Mean forecast_quality_score across the group. Red < 0.3 = don't trust. Green > 0.6 = reliable. The forecaster scores series 0..1 combining backtest MAPE/MASE, confidence band width, and data coverage.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 0 }, + "targets": [ + { "refId": "A", "expr": "avg by (group) (forecast_quality_score)", "legendFormat": "{{group}}" } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "vertical", + "colorMode": "background", + "graphMode": "none", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "max": 1, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.3 }, + { "color": "green", "value": 0.6 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 2, + "title": "Trustworthy series (q > 0.5)", + "description": "How many series score above 0.5, per group. Hover the count to see the total. A low ratio means several series are inherently hard to forecast (sparse, near-zero, or highly variable) and the forecaster is correctly flagging them.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 8, "x": 8, "y": 0 }, + "targets": [ + { "refId": "A", "expr": "count by (group) (forecast_quality_score > 0.5)", "legendFormat": "{{group}} (good)" } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "vertical", + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 3 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 3, + "title": "Last successful run (per group)", + "description": "Time since the most recent successful refit. Should stay under refresh_interval (5m in dev). Yellow >2m, red >10m suggests something is stuck.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 4, "x": 16, "y": 0 }, + "targets": [ + { "refId": "A", "expr": "time() - forecast_last_run_timestamp_seconds", "legendFormat": "{{group}}" } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "vertical", + "colorMode": "background", + "graphMode": "none", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 120 }, + { "color": "red", "value": 600 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "id": 4, + "title": "Failures (last 1h)", + "description": "Sum of forecast_failures_total across all groups in the last hour. Zero is the happy path.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 5, "w": 4, "x": 20, "y": 0 }, + "targets": [ + { "refId": "A", "expr": "sum(increase(forecast_failures_total[1h])) or vector(0)", "legendFormat": "total" } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "id": 10, + "title": "Signature: HTTP request rate — actual vs forecast band", + "description": "One endpoint ($signature_endpoint). Solid green line = actual rate (stops at `now`). Shaded blue band = 95% confidence interval from the remote_write sink. Left of `now`: older fits' forward curves now in the past — visually check 'did the forecast track what happened'. Right of `now`: live forecast, where the model thinks the rate is heading.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 11, "w": 12, "x": 0, "y": 5 }, + "targets": [ + { "refId": "A", "expr": "sum(rate(http_requests_total{endpoint=\"$signature_endpoint\"}[2m]))", "legendFormat": "actual" }, + { "refId": "B", "expr": "min(http_requests_rate_forecast_lower{endpoint=\"$signature_endpoint\",level=\"95\"})", "legendFormat": "lower 95%" }, + { "refId": "C", "expr": "max(http_requests_rate_forecast_upper{endpoint=\"$signature_endpoint\",level=\"95\"})", "legendFormat": "upper 95%" }, + { "refId": "D", "expr": "avg(http_requests_rate_forecast{endpoint=\"$signature_endpoint\"})", "legendFormat": "forecast (central)" } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 0, "showPoints": "never" } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "actual" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }, { "id": "custom.lineWidth", "value": 3 } ] }, + { "matcher": { "id": "byName", "options": "forecast (central)" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 1 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [6, 4] } } ] }, + { "matcher": { "id": "byName", "options": "upper 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.fillOpacity", "value": 15 }, { "id": "custom.fillBelowTo", "value": "lower 95%" } ] }, + { "matcher": { "id": "byName", "options": "lower 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] } + ] + } + }, + { + "type": "timeseries", + "id": 11, + "title": "Signature: Host CPU — actual vs forecast band", + "description": "Whole-host CPU busy %. Same treatment as the HTTP panel — green line should sit inside the blue shaded band, and the right of `now` shows where the model thinks CPU is heading.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 11, "w": 12, "x": 12, "y": 5 }, + "targets": [ + { "refId": "A", "expr": "avg(100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[2m])) * 100))", "legendFormat": "actual" }, + { "refId": "B", "expr": "avg(node_cpu_busy_pct_forecast_lower{level=\"95\"})", "legendFormat": "lower 95%" }, + { "refId": "C", "expr": "avg(node_cpu_busy_pct_forecast_upper{level=\"95\"})", "legendFormat": "upper 95%" }, + { "refId": "D", "expr": "avg(node_cpu_busy_pct_forecast)", "legendFormat": "forecast (central)" } + ], + "fieldConfig": { + "defaults": { "unit": "percent", "custom": { "lineWidth": 2, "fillOpacity": 0, "showPoints": "never" } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "actual" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }, { "id": "custom.lineWidth", "value": 3 } ] }, + { "matcher": { "id": "byName", "options": "forecast (central)" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 1 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [6, 4] } } ] }, + { "matcher": { "id": "byName", "options": "upper 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.fillOpacity", "value": 15 }, { "id": "custom.fillBelowTo", "value": "lower 95%" } ] }, + { "matcher": { "id": "byName", "options": "lower 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] } + ] + } + }, + { + "type": "timeseries", + "id": 20, + "title": "Signature: HTTP p95 latency — actual vs forecast band", + "description": "Tail latency for $signature_endpoint. Same treatment as the other Signature panels — green line = actual p95 from `histogram_quantile`, blue shaded band = forecast 95% confidence interval projected forward, dashed blue = central forecast. p95 latency is one of the harder signals to forecast (log-normal tails), so wider bands are expected and honest.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 11, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{endpoint=\"$signature_endpoint\"}[2m])))", "legendFormat": "actual" }, + { "refId": "B", "expr": "min(http_request_p95_seconds_forecast_lower{endpoint=\"$signature_endpoint\",level=\"95\"})", "legendFormat": "lower 95%" }, + { "refId": "C", "expr": "max(http_request_p95_seconds_forecast_upper{endpoint=\"$signature_endpoint\",level=\"95\"})", "legendFormat": "upper 95%" }, + { "refId": "D", "expr": "avg(http_request_p95_seconds_forecast{endpoint=\"$signature_endpoint\"})", "legendFormat": "forecast (central)" } + ], + "fieldConfig": { + "defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 0, "showPoints": "never" } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "actual" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }, { "id": "custom.lineWidth", "value": 3 } ] }, + { "matcher": { "id": "byName", "options": "forecast (central)" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 1 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [6, 4] } } ] }, + { "matcher": { "id": "byName", "options": "upper 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.fillOpacity", "value": 15 }, { "id": "custom.fillBelowTo", "value": "lower 95%" } ] }, + { "matcher": { "id": "byName", "options": "lower 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] } + ] + } + }, + { + "type": "timeseries", + "id": 21, + "title": "Signature: HTTP error rate (5xx) — actual vs forecast band", + "description": "Aggregate error rate across all /api/* endpoints (errors are sparse per endpoint, so the fleet-aggregate view is the meaningful one). Green = actual 5xx rate, blue shaded band = forecast 95% interval. The error rate is the sparsest signal in the demo stack — bands are wide on purpose because the underlying signal is genuinely unpredictable at low rates.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 11, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { "refId": "A", "expr": "sum(rate(http_requests_total{status=~\"5..\"}[2m]))", "legendFormat": "actual" }, + { "refId": "B", "expr": "min(http_errors_rate_forecast_lower{level=\"95\"})", "legendFormat": "lower 95%" }, + { "refId": "C", "expr": "max(http_errors_rate_forecast_upper{level=\"95\"})", "legendFormat": "upper 95%" }, + { "refId": "D", "expr": "avg(http_errors_rate_forecast)", "legendFormat": "forecast (central)" } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 0, "showPoints": "never" } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "actual" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }, { "id": "custom.lineWidth", "value": 3 } ] }, + { "matcher": { "id": "byName", "options": "forecast (central)" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 1 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [6, 4] } } ] }, + { "matcher": { "id": "byName", "options": "upper 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 }, { "id": "custom.fillOpacity", "value": 15 }, { "id": "custom.fillBelowTo", "value": "lower 95%" } ] }, + { "matcher": { "id": "byName", "options": "lower 95%" }, + "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, { "id": "custom.lineWidth", "value": 0 } ] } + ] + } + } + ] +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5a451be..e8c87ad 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -5,7 +5,14 @@ # forecaster :9091 (/metrics, /healthz, /-/reload) # Redis :6379 (query-cache backend; HA isn't wired in compose # since leader election needs a kube API server) -# Grafana :3000 (admin / admin) +# Grafana :3000 (admin / admin) — "load" profile only +# +# Two profiles: +# default (`make dev-up`) — VM, Prometheus, Redis, node-exporter, +# forecaster. Minimum for forecaster +# development. +# load (`make dev-up-load`) — adds http-demo, http-demo-loadgen, +# Grafana. Full demo experience. name: promforecast-dev @@ -29,6 +36,39 @@ services: retries: 20 restart: unless-stopped + http-demo: + # Tiny FastAPI app with a handful of endpoints + /metrics. Driven by + # the loadgen below so the forecaster has http_* time series to fit on. + # Behind the "load" profile so `make dev-up` (minimum dev stack) + # doesn't start it; `make dev-up-load` does. + profiles: ["load"] + build: + context: ./docker/http-demo + dockerfile: Dockerfile.app + image: promforecast-http-demo:dev + container_name: pf-http-demo + ports: + - "8000:8000" + restart: unless-stopped + + http-demo-loadgen: + # ~20-30 req/min with a slow sinusoidal modulation so the rate has shape. + # Same profile as http-demo since one without the other is useless. + profiles: ["load"] + build: + context: ./docker/http-demo + dockerfile: Dockerfile.loadgen + image: promforecast-http-demo-loadgen:dev + container_name: pf-http-demo-loadgen + environment: + TARGET: http://http-demo:8000 + BASE_RPM: "25" + AMPLITUDE_RPM: "15" + PERIOD_SECONDS: "600" + depends_on: + - http-demo + restart: unless-stopped + node-exporter: image: prom/node-exporter:v1.11.1 container_name: pf-node-exporter @@ -103,6 +143,10 @@ services: restart: unless-stopped grafana: + # Grafana is in the "load" profile because the bundled dashboards are + # tied to the http-demo data; a minimum `make dev-up` for forecaster + # development doesn't need it. `make dev-up-load` brings it up. + profiles: ["load"] image: grafana/grafana:13.0.1 container_name: pf-grafana environment: diff --git a/docker/dev-config.yaml b/docker/dev-config.yaml index b805efb..bbfd1e7 100644 --- a/docker/dev-config.yaml +++ b/docker/dev-config.yaml @@ -20,16 +20,36 @@ query_cache: ttl: 30s redis_address: redis:6379 +# Push the *full* forecast curve (every step from t+1 to t+horizon) into VM +# with future timestamps after each fit. Without this the /metrics exporter +# only emits one gauge per series — the horizon point — which appears as a +# 5-min-wide staircase in Grafana. With remote_write on, dashboards plot a +# smooth curve overlapping the actual signal, and `offset ` is no +# longer needed on the dashboard side. +# +# VM accepts future timestamps because docker-compose.dev.yml starts it with +# --futureRetention=2d. +sink: + remote_write: + enabled: true + url: http://victoriametrics:8428/api/v1/import/prometheus + server: listen: ":9091" - refresh_interval: 1m + # 5m gives AutoARIMA + backtest fits time to finish before the next tick. + # Earlier 1m caused the per-group lock to back up runs to ~10 min effective + # cadence on http_demo and node_storage. + refresh_interval: 5m # Tight caps to exercise overflow handling end-to-end in dev. safety: max_series_per_query: 50 max_total_series: 500 series_overflow: drop_lowest_priority - fit_timeout: 60s + # 120s suits AutoARIMA on the bursty disk-I/O and per-endpoint HTTP series + # in this stack. With 60s we saw recurring fit_timeout / backtest_failed + # events. + fit_timeout: 120s query_timeout: 30s defaults: @@ -84,3 +104,89 @@ groups: - id: node_memory_used_bytes promql: | node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes + - id: node_load1 + promql: node_load1 + - id: node_load5 + promql: node_load5 + + - name: node_storage + priority: 8 + preprocess: + gaps: + strategy: forward_fill + max_gap: 10m + queries: + # Filesystem fill: predict where used % is heading. The capacity + # dashboard uses this to surface "disk will be full in N hours". + - id: node_filesystem_used_pct + promql: | + 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"}) + - id: node_disk_read_bytes_rate + promql: rate(node_disk_read_bytes_total[2m]) + - id: node_disk_written_bytes_rate + promql: rate(node_disk_written_bytes_total[2m]) + + - name: node_network + priority: 6 + preprocess: + gaps: + strategy: forward_fill + max_gap: 10m + queries: + - id: node_network_receive_bytes_rate + promql: | + rate(node_network_receive_bytes_total{device!~"lo|docker.*|veth.*|br-.*"}[2m]) + - id: node_network_transmit_bytes_rate + promql: | + rate(node_network_transmit_bytes_total{device!~"lo|docker.*|veth.*|br-.*"}[2m]) + + - name: http_demo + priority: 9 + preprocess: + gaps: + strategy: forward_fill + max_gap: 10m + # Loadgen modulates request rate on a 10-min sine (PERIOD_SECONDS=600 in + # docker/http-demo/loadgen.py). Override season_length so SeasonalNaive + # locks onto that cycle instead of the generic 15-step default. + # + # The endpoint filter (/api/.*|/healthz) keeps the forecast scope to + # endpoints the loadgen actually targets. Self-scrapes (/metrics) and + # incidental browser hits (/, /favicon.ico) would otherwise produce + # near-zero series that pathologically score q=0 under MAPE. + queries: + - id: http_requests_rate + promql: | + sum by (endpoint) ( + rate(http_requests_total{endpoint=~"/api/.*|/healthz"}[2m]) + ) + season_length: 10 + # Error rate. flaky returns 500 only ~7% of calls so this is a + # genuinely intermittent signal — score it with MASE (via + # data_profile: intermittent) instead of MAPE, which blows up on + # near-zero actuals. + - id: http_errors_rate + promql: | + sum by (endpoint) ( + rate(http_requests_total{status=~"5..",endpoint=~"/api/.*|/healthz"}[2m]) + ) + season_length: 10 + data_profile: intermittent + # p95 latency derived from the histogram. Forecasting tail latency + # is one of the more interesting demos because it's both noisy and + # tied to load. + - id: http_request_p95_seconds + promql: | + histogram_quantile( + 0.95, + sum by (endpoint, le) ( + rate(http_request_duration_seconds_bucket{endpoint=~"/api/.*|/healthz"}[2m]) + ) + ) + season_length: 10 + # In-flight concurrency. Gauge, not derived from a counter, so this + # is a different shape of series. + - id: http_in_flight + promql: max by (job) (http_requests_in_flight) + season_length: 10 diff --git a/docker/grafana/provisioning/datasources/datasources.yml b/docker/grafana/provisioning/datasources/datasources.yml index 5474ac1..1a23e58 100644 --- a/docker/grafana/provisioning/datasources/datasources.yml +++ b/docker/grafana/provisioning/datasources/datasources.yml @@ -1,11 +1,16 @@ apiVersion: 1 datasources: + # VictoriaMetrics is the default: it holds everything Prometheus has + # (via remote_write, 30d retention) plus the forecaster's sink-pushed + # forecast curves. Querying Prometheus directly is only useful for + # debugging the scrape pipeline. - name: VictoriaMetrics type: prometheus access: proxy uid: victoriametrics url: http://victoriametrics:8428 + isDefault: true editable: true jsonData: httpMethod: POST @@ -16,7 +21,6 @@ datasources: access: proxy uid: prometheus url: http://prometheus:9090 - isDefault: true editable: true jsonData: httpMethod: POST diff --git a/docker/http-demo/Dockerfile.app b/docker/http-demo/Dockerfile.app new file mode 100644 index 0000000..db49130 --- /dev/null +++ b/docker/http-demo/Dockerfile.app @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app +RUN pip install --no-cache-dir \ + "fastapi==0.115.0" \ + "uvicorn[standard]==0.30.6" \ + "prometheus-client==0.21.0" + +COPY app.py /app/app.py + +EXPOSE 8000 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"] diff --git a/docker/http-demo/Dockerfile.loadgen b/docker/http-demo/Dockerfile.loadgen new file mode 100644 index 0000000..b34db7e --- /dev/null +++ b/docker/http-demo/Dockerfile.loadgen @@ -0,0 +1,13 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app +RUN pip install --no-cache-dir "httpx==0.27.2" + +COPY loadgen.py /app/loadgen.py + +CMD ["python", "/app/loadgen.py"] diff --git a/docker/http-demo/app.py b/docker/http-demo/app.py new file mode 100644 index 0000000..3ec44d0 --- /dev/null +++ b/docker/http-demo/app.py @@ -0,0 +1,130 @@ +# Tiny FastAPI app for the dev stack: a handful of HTTP endpoints with varied +# latency / error profiles and a /metrics endpoint that exposes prometheus_client +# counters/histograms/gauges. Driven by docker/http-demo/loadgen.py so the +# forecaster has http_* time series to fit on. +from __future__ import annotations + +import asyncio +import os +import random +import time + +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse, PlainTextResponse +from prometheus_client import ( + CONTENT_TYPE_LATEST, + CollectorRegistry, + Counter, + Gauge, + Histogram, + generate_latest, +) + +registry = CollectorRegistry() + +http_requests_total = Counter( + "http_requests_total", + "Total HTTP requests handled.", + ["method", "endpoint", "status"], + registry=registry, +) +http_request_duration_seconds = Histogram( + "http_request_duration_seconds", + "HTTP request latency in seconds.", + ["method", "endpoint"], + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0), + registry=registry, +) +http_requests_in_flight = Gauge( + "http_requests_in_flight", + "In-flight HTTP requests.", + registry=registry, +) +http_payload_bytes = Histogram( + "http_payload_bytes", + "Response payload size in bytes.", + ["endpoint"], + buckets=(64, 256, 1024, 4096, 16384, 65536), + registry=registry, +) + +APP_NAME = os.getenv("APP_NAME", "http-demo") +app = FastAPI(title=APP_NAME) + + +@app.middleware("http") +async def observe(request: Request, call_next): + endpoint = request.scope.get("route").path if request.scope.get("route") else request.url.path + method = request.method + http_requests_in_flight.inc() + start = time.perf_counter() + try: + response: Response = await call_next(request) + status = response.status_code + except Exception: + status = 500 + raise + finally: + elapsed = time.perf_counter() - start + http_requests_in_flight.dec() + http_request_duration_seconds.labels(method=method, endpoint=endpoint).observe(elapsed) + http_requests_total.labels(method=method, endpoint=endpoint, status=str(status)).inc() + return response + + +def _sleep_lognormal(mu: float, sigma: float, cap: float) -> float: + delay = min(random.lognormvariate(mu, sigma), cap) + return delay + + +@app.get("/") +async def root() -> dict: + body = {"app": APP_NAME, "endpoints": ["/", "/api/users", "/api/products", "/api/slow", "/api/flaky", "/healthz"]} + http_payload_bytes.labels(endpoint="/").observe(len(str(body))) + return body + + +@app.get("/api/users") +async def users() -> dict: + # Fast endpoint, ~5-20ms. + await asyncio.sleep(_sleep_lognormal(mu=-4.5, sigma=0.4, cap=0.2)) + payload = {"users": [{"id": i, "name": f"user-{i}"} for i in range(random.randint(5, 25))]} + http_payload_bytes.labels(endpoint="/api/users").observe(len(str(payload))) + return payload + + +@app.get("/api/products") +async def products() -> dict: + # Medium endpoint, ~30-150ms with occasional spikes. + await asyncio.sleep(_sleep_lognormal(mu=-3.0, sigma=0.6, cap=1.5)) + payload = {"products": [{"sku": f"sku-{i}", "price": round(random.uniform(1, 99), 2)} for i in range(20)]} + http_payload_bytes.labels(endpoint="/api/products").observe(len(str(payload))) + return payload + + +@app.get("/api/slow") +async def slow() -> dict: + # Slow endpoint, ~0.3-2s. Useful for histogram_quantile to have something + # to show. + await asyncio.sleep(_sleep_lognormal(mu=-1.0, sigma=0.5, cap=3.0)) + return {"status": "ok"} + + +@app.get("/api/flaky") +async def flaky() -> Response: + # ~7% of calls return 500. Gives http_requests_total{status="500"} a non-zero rate + # so the forecaster has an error series to fit. + await asyncio.sleep(_sleep_lognormal(mu=-3.5, sigma=0.5, cap=1.0)) + if random.random() < 0.07: + return JSONResponse({"error": "transient"}, status_code=500) + return JSONResponse({"status": "ok"}) + + +@app.get("/healthz") +async def healthz() -> dict: + return {"status": "ok"} + + +@app.get("/metrics") +async def metrics() -> PlainTextResponse: + return PlainTextResponse(generate_latest(registry).decode("utf-8"), media_type=CONTENT_TYPE_LATEST) diff --git a/docker/http-demo/loadgen.py b/docker/http-demo/loadgen.py new file mode 100644 index 0000000..29ce6a3 --- /dev/null +++ b/docker/http-demo/loadgen.py @@ -0,0 +1,55 @@ +# Tiny load generator: fires ~20-30 req/min by default at the http-demo app, +# with a slow sinusoidal modulation on top so the rate has shape (and the +# forecaster has a non-trivial signal to fit). Endpoints are picked from a +# weighted mix so http_requests_total{endpoint=...} differs per route. +from __future__ import annotations + +import asyncio +import math +import os +import random +import time +from contextlib import suppress + +import httpx + +TARGET = os.getenv("TARGET", "http://http-demo:8000") +BASE_RPM = float(os.getenv("BASE_RPM", "25")) +AMPLITUDE_RPM = float(os.getenv("AMPLITUDE_RPM", "15")) +PERIOD_SECONDS = float(os.getenv("PERIOD_SECONDS", "600")) # 10-minute "diurnal" cycle +JITTER = float(os.getenv("JITTER", "0.25")) + +ENDPOINTS = [ + ("/api/users", 5), + ("/api/products", 3), + ("/api/slow", 1), + ("/api/flaky", 2), + ("/healthz", 1), +] + + +def pick_endpoint() -> str: + paths, weights = zip(*ENDPOINTS, strict=True) + return random.choices(paths, weights=weights, k=1)[0] + + +def current_rpm(t0: float) -> float: + phase = 2 * math.pi * ((time.monotonic() - t0) % PERIOD_SECONDS) / PERIOD_SECONDS + return max(1.0, BASE_RPM + AMPLITUDE_RPM * math.sin(phase)) + + +async def hammer() -> None: + t0 = time.monotonic() + timeout = httpx.Timeout(5.0) + async with httpx.AsyncClient(timeout=timeout, base_url=TARGET) as client: + while True: + rpm = current_rpm(t0) + interval = 60.0 / rpm + interval *= 1.0 + random.uniform(-JITTER, JITTER) + with suppress(Exception): + await client.get(pick_endpoint()) + await asyncio.sleep(max(0.05, interval)) + + +if __name__ == "__main__": + asyncio.run(hammer()) diff --git a/docker/prometheus.yml b/docker/prometheus.yml index e20dabf..fe69f53 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -27,6 +27,10 @@ scrape_configs: static_configs: - targets: ["node-exporter:9100"] + - job_name: http-demo + static_configs: + - targets: ["http-demo:8000"] + - job_name: promforecast # honor_labels: forecast metrics carry the source ``instance`` label # (e.g. ``dev-host-a:9100``) from the underlying time-series. Without @@ -37,3 +41,15 @@ scrape_configs: honor_labels: true static_configs: - targets: ["forecaster:9091"] + # Drop forecast snapshot gauges (``*_forecast``, ``*_forecast_lower``, + # ``*_forecast_upper``) from the scrape. The forecaster's remote_write + # sink already pushes the full forecast curve into VM directly; the + # scraped snapshot is a duplicate with a different label set (carries + # ``instance="forecaster:9091"``) that confuses dashboards. Operational + # metrics (``forecast_failures_total``, ``forecast_quality_score``, + # ``forecast_run_duration_seconds``, etc.) stay scraped — they only + # exist on /metrics. + metric_relabel_configs: + - source_labels: [__name__] + regex: '.+_forecast(_lower|_upper)?' + action: drop diff --git a/docs/README.md b/docs/README.md index f23bef7..a81da29 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,7 @@ The docs follow the [Diataxis](https://diataxis.fr/) framework: - [Conformal calibration](conformal.md) — empirical-coverage band correction - [Capacity planning](capacity.md) — damped trend, time-to-threshold ETAs, growth rate - [Forecast drift](drift.md) — prediction-vs-prediction shift telemetry +- [Reading the numbers](reading-the-numbers.md) — how to interpret quality and accuracy in practice (mean vs p95, horizon paradox, "wait three cycles", when to actually worry) - [Scheduling](scheduling.md) — per-query refresh intervals, max-concurrent-fits cap - [Forecast preview](preview.md) — opt-in ad-hoc what-if endpoint - [Regressors](regressors.md) — calendar, custom PromQL, and holiday regressors diff --git a/docs/reading-the-numbers.md b/docs/reading-the-numbers.md new file mode 100644 index 0000000..421a6c5 --- /dev/null +++ b/docs/reading-the-numbers.md @@ -0,0 +1,91 @@ +# Reading the numbers + +This page explains how to interpret promforecast’s quality and accuracy metrics so you can tell at a glance whether the system is healthy, whether a specific forecast is trustworthy, and when a “bad” number isn’t actually a problem. + +## The two numbers that matter most + +For daily operation, these two metrics tell you almost everything you need: + +| Metric | What it tells you | Good values | +|-----------------------------|--------------------------------------------|------------------------------| +| `forecast_quality_score` | Per-(series, model) composite reliability in `[0, 1]` | `> 0.6` reliable; `< 0.3` don’t trust | +| `forecast_accuracy_mape` | Backtest MAPE per `(group, horizon)`, in percent | Depends on signal shape — see below | + +If you check the dashboard once a day, look at **average quality per group** and **trustworthy-series count** (series scoring > 0.5). Both green = the bulk of your forecasts are useful. + +## What MAPE actually tells you + +MAPE is the backtest accuracy (held-out actuals vs. predicted, as percentage). +There is **no universal “good MAPE”** — it depends heavily on the signal. + +| Signal shape | Typical MAPE range | Example | +|-----------------------|--------------------|---------| +| Smooth, predictable | `< 5 %` | Stable network throughput | +| Cyclical with noise | `10–30 %` | CPU with daily/weekly patterns | +| Bursty | `50–100 %` | Disk I/O with occasional spikes | +| Sparse / near-zero | `100 % +` (or NaN) | Rare error rates | + +For sparse/near-zero series, MAPE becomes unreliable. Set `data_profile: intermittent` on the query to switch to MASE scoring (see [models.md](models.md)). + +## Mean quality vs. p95 MAPE — why they can disagree + +- **Mean quality** averages across *all* series in the group. +- **p95 MAPE** highlights the worst ~5 % of `(series × horizon)` measurements. + +A group can have excellent mean quality while p95 MAPE is high — that just means one or two series are noisy. +**Rule of thumb:** use mean quality for “is the group healthy overall?”; use p95 MAPE + `bottomk(3, forecast_quality_score)` when hunting the specific bad series. + +## The horizon paradox + +Shorter horizons are *not* always more accurate on noisy signals. +Short horizons inherit minute-to-minute noise; long horizons benefit from implicit averaging. +Seeing `p95 MAPE 3m > p95 MAPE 15m` is usually normal for spiky data — inspect the raw series to confirm. + +## After a restart: wait three cycles + +Quality and MAPE reset on restart. +- First cycle: noisy, not representative. +- Cycles 2–3: numbers stabilise. +- After ~30 min of uptime: metrics are meaningful. + +Don’t judge model performance from the first cycle. + +## Interpretive examples from a typical dev stack + +| Group | Mean quality | What it tells you | +|----------------|--------------|-------------------| +| `node_network` | `~0.99` | Smooth, predictable — excellent forecasts | +| `node_storage` | `~0.65` | Bursty I/O — mid-range quality is the ceiling | +| `node_capacity`| `~0.45` | Moderately noisy CPU/memory — expected | +| `http_demo` | `~0.30` | Sparse endpoints + log-normal latency — low quality is honest | + +Your own “good” numbers depend on what you forecast: smooth platform metrics sit near 1.0; application latency percentiles rarely do. + +## When to actually worry + +Use this checklist when a number looks bad: + +- **Quality drops 0.5+ in one cycle** (while uptime is high) → regime shift. Check `forecast_drift_score` and `forecast_input_change_point_total`. +- **All series in a group drop to quality 0** → datasource problem. Verify the PromQL, datasource URL, and target `up` metric. +- **One series stuck at quality 0** while siblings are fine → isolated failure. Query `forecast_quality_score{group=""} == 0`. +- **MAPE climbing but quality stays high** → bands are widening (conformal calibration working). Check `forecast_band_empirical_coverage`. +- **Visual forecast looks wrong but quality is high** → check Grafana variable selection (`model` / `endpoint` dropdown). + +## What each panel of “Is it working?” answers + +| Panel | Question it answers | +|---------------------------|---------------------| +| Quality score per group | Is the group trustworthy on average? | +| Trustworthy series count | How many series score above the usable threshold? | +| Last successful run | Did the scheduler tick recently? (should be < `refresh_interval`) | +| Failures (last 1 h) | Did anything go wrong? Zero is the happy path | +| Signature panels | Does the green (actual) line sit inside the blue (95 % forecast band)? | + +For per-series drill-down (quality, drift, deviation, durations, errors) switch to the **“Demo tour”** dashboard. + +## See also + +- [telemetry.md](telemetry.md) — full list of emitted metrics and labels +- [conformal.md](conformal.md) — how bands are empirically calibrated +- [drift.md](drift.md) — `forecast_drift_score` semantics +- [scheduling.md](scheduling.md) — how `refresh_interval` and per-query overrides work