From c48e8b89c7a26493c434824a4f9f49b0696b53e8 Mon Sep 17 00:00:00 2001 From: yahiaongh Date: Thu, 23 Jul 2026 20:20:55 +0100 Subject: [PATCH] feat: add Prometheus + Grafana observability stack with provisioned datasource and dashboard covering all three instrumented services (Milestone 5); bring MILESTONES.md fully up to date --- README.md | 7 +- docs/MILESTONES.md | 41 +++--- infra/docker-compose.yml | 38 +++++ .../provisioning/dashboards/dashboards.yml | 11 ++ .../dashboards/sentinelops-overview.json | 131 ++++++++++++++++++ .../provisioning/datasources/prometheus.yml | 9 ++ infra/prometheus/prometheus.yml | 26 ++++ 7 files changed, 243 insertions(+), 20 deletions(-) create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/grafana/provisioning/dashboards/sentinelops-overview.json create mode 100644 infra/grafana/provisioning/datasources/prometheus.yml create mode 100644 infra/prometheus/prometheus.yml diff --git a/README.md b/README.md index 9160526..4429b96 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ flowchart LR | `log-producer` | Python | Simulates 4 microservices, injects realistic anomaly bursts | ✅ Running | | `ingestion-go` | Go | Kafka consumer → batched writes to TimescaleDB, self-migrating schema | ✅ Running | | `anomaly-rust` | Rust | Polls continuous aggregate, z-score + EWMA anomaly detection | ✅ Running | -| `llm-query-python` | Python | Natural-language RAG query interface over detected anomalies | 🚧 Not started | -| `dashboard-nextjs` | TypeScript | Real-time dashboard | 🚧 Not started | +| `llm-query-python` | Python | Natural-language RAG query interface over detected anomalies (local Ollama inference) | ✅ Running | +| `dashboard-nextjs` | TypeScript | Real-time anomaly feed, service health, LLM query terminal | ✅ Running | +| Prometheus + Grafana | — | Metrics collection and dashboards for all instrumented services | ✅ Running | See [docs/MILESTONES.md](docs/MILESTONES.md) for the full roadmap and what's shipped so far. @@ -55,6 +56,8 @@ docker compose logs -f anomaly-rust # Browse the raw event stream open http://localhost:8080 # Redpanda Console +open http://localhost:3001 # Grafana (admin / devpassword by default) +open http://localhost:3000 # SentinelOps dashboard # Check ingestion service metrics curl http://localhost:9100/metrics | grep sentinelops diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index bd70290..43eba07 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -29,30 +29,35 @@ Status and scope for each phase of SentinelOps. Dates are approximate; this is a - Caught and fixed a real directional-severity bug in detection logic, with regression tests - Extended CI to 7 jobs: added Rust lint/test/fmt and a third Docker image build -## 🚧 Milestone 3 — Python LLM/RAG query service (next) -- FastAPI service exposing natural-language queries over detected anomalies and log context -- RAG pipeline: retrieve relevant log/anomaly context, summarize via LLM -- Incident summarization ("what happened to checkout between 2-3am?") - -## 🚧 Milestone 4 — Next.js dashboard -- Real-time anomaly feed -- Natural-language query chat interface -- Service health overview - -## 🚧 Milestone 5 — Observability stack -- Prometheus scraping all services (currently exposed but not scraped) -- Grafana dashboards - -## 🚧 Milestone 6 — Kubernetes deployment +## ✅ Milestone 3 — Python LLM/RAG query service +- FastAPI service exposing natural-language queries over detected anomalies and service stats +- Retrieval grounded in TimescaleDB (`anomalies`, `log_events_1min`) — verified the model's numbers match real rows exactly, not hallucinated +- Async job queue (`202 Accepted` + poll) instead of a blocking request, designed around measured local-hardware inference latency (~2.8 tokens/sec on CPU-only Ollama) +- Diagnosed and fixed a real Ollama networking issue (host-only binding blocking container access) via a systemd override + +## ✅ Milestone 4 — Next.js dashboard +- Real-time anomaly feed (scrolling log-tail style), service health grid, and an LLM query terminal +- Custom design system (IBM Plex Sans/Mono, SRE command-center palette) — not a default template look +- Server-side API routes keep database credentials and the LLM service URL out of the browser +- Mid-build security upgrade to Next.js 16 + ESLint 9 to close several known CVEs before writing any app code +- Extended CI to 9 jobs + +## ✅ Milestone 5 — Observability stack +- Prometheus scraping ingestion-go, anomaly-rust, and llm-query-python on 15s intervals +- Grafana with provisioned datasource and dashboard (committed JSON, not clicked together — reproducible from a fresh `docker compose up`) +- 9-panel dashboard covering ingestion throughput/errors, batch performance, anomaly detection rate and severity breakdown, and LLM query latency/failure rate + +## 🚧 Milestone 6 — Kubernetes deployment (next) - Manifests for all services - Local cluster deployment (k3d/kind) as a production-shaped deploy path ## Known gaps (tracked, not blocking) -- No Grafana/Prometheus scrape config yet — metrics are exposed but not yet centrally collected -- Fixed: log producer was assigning `level` independently of `status_code` (a 200 response could randomly be logged as ERROR) — root cause of inflated error-rate metrics. Fixed by deriving level from status_code. -- Fixed: the ANOMALY_PROBABILITY tuning (0.02 → 0.0015) from the previous fix never actually took effect — a copy/paste error had applied the new value to `redpanda-console`'s environment block (which ignores it) instead of `log-producer`'s. Corrected and verified live: error rates dropped from ~30-50% to ~0-15%, matching expected burst-driven anomaly frequency (confirmed against actual burst log timestamps in a sample window). +- No automated tests for the Next.js dashboard (API routes or components) — only manual/visual verification so far +- `services/dashboard-nextjs/README.md` is still `create-next-app`'s default boilerplate, not project-specific docs ## Recently closed +- Log producer was assigning `level` independently of `status_code` (a 200 response could randomly be logged as ERROR) — root cause of inflated error-rate metrics. Fixed by deriving level from status_code. +- A follow-up fix (`ANOMALY_PROBABILITY` 0.02 → 0.0015) never actually took effect due to a copy/paste error that applied it to the wrong service's environment block. Corrected and verified live: error rates dropped from ~30-50% to ~0-15%, matching expected burst-driven anomaly frequency. - `anomaly-rust`: added `detect()` end-to-end tests, including a regression guard for the directional-severity bug - `anomaly-rust`: EWMA baseline now persisted in `AnomalyRecord` for error-burst anomalies - `ingestion-go`: added mocked-pool unit tests for `store.go`'s write path (`pgxmock`), alongside existing integration tests \ No newline at end of file diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 9aeb6af..3df55cb 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -168,9 +168,47 @@ services: start_period: 20s restart: unless-stopped networks: [sentinelops-net] + prometheus: + image: prom/prometheus:v2.55.1 + container_name: sentinelops-prometheus + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=15d" + ports: + - "9090:9090" + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:9090/-/healthy || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + restart: unless-stopped + networks: [sentinelops-net] + grafana: + image: grafana/grafana:11.3.1 + container_name: sentinelops-grafana + depends_on: + prometheus: + condition: service_healthy + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-devpassword} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - grafana-data:/var/lib/grafana + ports: + - "3001:3000" + restart: unless-stopped + networks: [sentinelops-net] networks: sentinelops-net: driver: bridge volumes: timescale-data: + prometheus-data: + grafana-data: \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..a104cf5 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: SentinelOps + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/sentinelops-overview.json b/infra/grafana/provisioning/dashboards/sentinelops-overview.json new file mode 100644 index 0000000..9b1963d --- /dev/null +++ b/infra/grafana/provisioning/dashboards/sentinelops-overview.json @@ -0,0 +1,131 @@ +{ + "title": "SentinelOps Overview", + "uid": "sentinelops-overview", + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, + "title": "Events Ingested (rate/s)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(sentinelops_ingestion_events_written_total[1m]))", + "legendFormat": "events written/s" + } + ] + }, + { + "id": 2, + "title": "Ingestion Write Errors (rate/s)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(sentinelops_ingestion_write_errors_total[1m]))", + "legendFormat": "write errors/s" + } + ] + }, + { + "id": 3, + "title": "Batch Flush Duration (p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(sentinelops_ingestion_batch_flush_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95 flush duration" + } + ] + }, + { + "id": 4, + "title": "Batch Size (p50/p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(sentinelops_ingestion_batch_size_bucket[5m])) by (le))", + "legendFormat": "p50 batch size" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(sentinelops_ingestion_batch_size_bucket[5m])) by (le))", + "legendFormat": "p95 batch size" + } + ] + }, + { + "id": 5, + "title": "Anomalies Detected (by severity)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum(rate(sentinelops_anomaly_detected_total[5m])) by (severity)", + "legendFormat": "{{severity}}" + } + ] + }, + { + "id": 6, + "title": "Anomaly Detection Cycles (rate/s)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { + "expr": "rate(sentinelops_anomaly_detection_cycles_total[5m])", + "legendFormat": "detection cycles/s" + } + ] + }, + { + "id": 7, + "title": "Anomaly Engine DB Query Errors", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "rate(sentinelops_anomaly_db_query_errors_total[5m])", + "legendFormat": "db query errors/s" + } + ] + }, + { + "id": 8, + "title": "LLM Queries Submitted (rate/s)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [ + { + "expr": "rate(sentinelops_llm_queries_total[5m])", + "legendFormat": "queries/s" + }, + { + "expr": "rate(sentinelops_llm_query_failures_total[5m])", + "legendFormat": "failures/s" + } + ] + }, + { + "id": 9, + "title": "LLM Query Duration (p50/p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 32 }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(sentinelops_llm_query_duration_seconds_bucket[10m])) by (le))", + "legendFormat": "p50 duration" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(sentinelops_llm_query_duration_seconds_bucket[10m])) by (le))", + "legendFormat": "p95 duration" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/datasources/prometheus.yml b/infra/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..d1792ff --- /dev/null +++ b/infra/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false \ No newline at end of file diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..0f1380d --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,26 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "ingestion-go" + static_configs: + - targets: ["ingestion-go:9100"] + labels: + service: "ingestion-go" + + - job_name: "anomaly-rust" + static_configs: + - targets: ["anomaly-rust:9200"] + labels: + service: "anomaly-rust" + + - job_name: "llm-query-python" + static_configs: + - targets: ["llm-query:9300"] + labels: + service: "llm-query-python" + + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] \ No newline at end of file