From 72df8fa9ed8146e9d60c44386300135a98acc0ec Mon Sep 17 00:00:00 2001 From: GentleCold Date: Sat, 13 Jun 2026 17:11:30 +0800 Subject: [PATCH] chore(monitoring): shorten grafana query windows - Reduce Grafana dashboard query windows for benchmark-facing panels to 30 seconds. - Keep monitoring documentation examples aligned with the shorter dashboard windows. - Remove brittle Grafana dashboard JSON assertions from CPU unit tests. --- .../grafana/dashboards/daser-overview.json | 32 +- docs/monitoring/prometheus.md | 6 +- tests/unit/test_benchmark_unified_utils.py | 279 ------------------ 3 files changed, 19 insertions(+), 298 deletions(-) diff --git a/deploy/monitoring/grafana/dashboards/daser-overview.json b/deploy/monitoring/grafana/dashboards/daser-overview.json index f1e37e6..e7b9108 100644 --- a/deploy/monitoring/grafana/dashboards/daser-overview.json +++ b/deploy/monitoring/grafana/dashboards/daser-overview.json @@ -252,7 +252,7 @@ }, "targets": [ { - "expr": "(sum(rate(daser_cache_lookup_total{job=\"daser\", result=\"hit\"}[5m])) or vector(0)) / clamp_min(sum(rate(daser_cache_lookup_total{job=\"daser\"}[5m])), 1)", + "expr": "(sum(rate(daser_cache_lookup_total{job=\"daser\", result=\"hit\"}[30s])) or vector(0)) / clamp_min(sum(rate(daser_cache_lookup_total{job=\"daser\"}[30s])), 1)", "legendFormat": "request" } ], @@ -316,7 +316,7 @@ }, "targets": [ { - "expr": "rate(daser_cache_matched_tokens_total{job=\"daser\"}[5m]) / clamp_min(rate(daser_cache_requested_tokens_total{job=\"daser\"}[5m]), 1)", + "expr": "rate(daser_cache_matched_tokens_total{job=\"daser\"}[30s]) / clamp_min(rate(daser_cache_requested_tokens_total{job=\"daser\"}[30s]), 1)", "legendFormat": "token" } ], @@ -470,7 +470,7 @@ }, "targets": [ { - "expr": "sum by (result) (rate(daser_cache_lookup_total{job=\"daser\"}[5m]))", + "expr": "sum by (result) (rate(daser_cache_lookup_total{job=\"daser\"}[30s]))", "legendFormat": "{{result}}" } ], @@ -563,15 +563,15 @@ }, "targets": [ { - "expr": "histogram_quantile(0.5, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[5m]))", + "expr": "histogram_quantile(0.5, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[30s]))", "legendFormat": "p50" }, { - "expr": "histogram_quantile(0.95, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[5m]))", + "expr": "histogram_quantile(0.95, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[30s]))", "legendFormat": "p95" }, { - "expr": "histogram_quantile(0.99, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[5m]))", + "expr": "histogram_quantile(0.99, rate(daser_cache_prefix_reuse_tokens_bucket{job=\"daser\"}[30s]))", "legendFormat": "p99" } ], @@ -674,11 +674,11 @@ }, "targets": [ { - "expr": "increase(daser_cache_late_evicted_commits_total{job=\"daser\"}[5m])", + "expr": "increase(daser_cache_late_evicted_commits_total{job=\"daser\"}[30s])", "legendFormat": "late commits" }, { - "expr": "sum by (reason) (rate(daser_cache_evicted_chunks_total{job=\"daser\"}[5m]))", + "expr": "sum by (reason) (rate(daser_cache_evicted_chunks_total{job=\"daser\"}[30s]))", "legendFormat": "evict {{reason}}" } ], @@ -766,11 +766,11 @@ }, "targets": [ { - "expr": "histogram_quantile(0.5, sum by (le, op) (rate(daser_transfer_duration_seconds_bucket{job=\"daser\"}[5m]))) * 1000", + "expr": "histogram_quantile(0.5, sum by (le, op) (rate(daser_transfer_duration_seconds_bucket{job=\"daser\"}[30s]))) * 1000", "legendFormat": "p50 {{op}}" }, { - "expr": "histogram_quantile(0.95, sum by (le, op) (rate(daser_transfer_duration_seconds_bucket{job=\"daser\"}[5m]))) * 1000", + "expr": "histogram_quantile(0.95, sum by (le, op) (rate(daser_transfer_duration_seconds_bucket{job=\"daser\"}[30s]))) * 1000", "legendFormat": "p95 {{op}}" } ], @@ -851,7 +851,7 @@ }, "targets": [ { - "expr": "sum by (op) (rate(daser_transfer_bytes_total{job=\"daser\"}[5m])) / 1e9", + "expr": "sum by (op) (rate(daser_transfer_bytes_total{job=\"daser\"}[30s])) / 1e9", "legendFormat": "{{op}}" } ], @@ -898,11 +898,11 @@ }, "targets": [ { - "expr": "histogram_quantile(0.5, sum by (le, op) (rate(daser_transfer_chunk_size_bytes_bucket{job=\"daser\"}[5m])))", + "expr": "histogram_quantile(0.5, sum by (le, op) (rate(daser_transfer_chunk_size_bytes_bucket{job=\"daser\"}[30s])))", "legendFormat": "p50 {{op}}" }, { - "expr": "histogram_quantile(0.95, sum by (le, op) (rate(daser_transfer_chunk_size_bytes_bucket{job=\"daser\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le, op) (rate(daser_transfer_chunk_size_bytes_bucket{job=\"daser\"}[30s])))", "legendFormat": "p95 {{op}}" } ], @@ -968,7 +968,7 @@ }, "targets": [ { - "expr": "max_over_time(((sum(increase(daser_l1_hits_total{job=\"daser\"}[1m])) or vector(0)) / clamp_min((sum(increase(daser_l1_hits_total{job=\"daser\"}[1m])) or vector(0)) + (sum(increase(daser_l1_misses_total{job=\"daser\"}[1m])) or vector(0)), 1))[2m:1s])", + "expr": "max_over_time(((sum(increase(daser_l1_hits_total{job=\"daser\"}[30s])) or vector(0)) / clamp_min((sum(increase(daser_l1_hits_total{job=\"daser\"}[30s])) or vector(0)) + (sum(increase(daser_l1_misses_total{job=\"daser\"}[30s])) or vector(0)), 1))[30s:1s])", "legendFormat": "L1 hit rate" } ], @@ -1038,7 +1038,7 @@ }, "targets": [ { - "expr": "max_over_time(((sum(daser_l1_bytes_used{job=\"daser\"}) or vector(0)) / clamp_min((sum(daser_l1_bytes_capacity{job=\"daser\"}) or vector(0)), 1))[2m:1s])", + "expr": "max_over_time(((sum(daser_l1_bytes_used{job=\"daser\"}) or vector(0)) / clamp_min((sum(daser_l1_bytes_capacity{job=\"daser\"}) or vector(0)), 1))[30s:1s])", "legendFormat": "L1 used" } ], @@ -1299,7 +1299,7 @@ }, "targets": [ { - "expr": "max_over_time(((sum(increase(vllm:request_success_total{job=\"vllm\"}[1m])) or vector(0)) / 60)[2m:1s])", + "expr": "max_over_time(((sum(increase(vllm:request_success_total{job=\"vllm\"}[30s])) or vector(0)) / 30)[30s:1s])", "legendFormat": "requests/s" } ], diff --git a/docs/monitoring/prometheus.md b/docs/monitoring/prometheus.md index fc85b08..3779cca 100644 --- a/docs/monitoring/prometheus.md +++ b/docs/monitoring/prometheus.md @@ -143,9 +143,9 @@ job="vllm" — vLLM inference server Use token hit ratio rather than only request hit ratio when judging benefit: ```promql -rate(daser_cache_matched_tokens_total[5m]) +rate(daser_cache_matched_tokens_total[30s]) / -clamp_min(rate(daser_cache_requested_tokens_total[5m]), 1) +clamp_min(rate(daser_cache_requested_tokens_total[30s]), 1) ``` ### Transfer & Storage @@ -170,7 +170,7 @@ The IPC server also logs transfer summaries with decimal GB/s throughput: Grafana should compute throughput from bytes: ```promql -rate(daser_transfer_bytes_total[5m]) / 1e9 +rate(daser_transfer_bytes_total[30s]) / 1e9 ``` ### vLLM Inference (from vLLM scrape target) diff --git a/tests/unit/test_benchmark_unified_utils.py b/tests/unit/test_benchmark_unified_utils.py index 29e5232..e2b661c 100644 --- a/tests/unit/test_benchmark_unified_utils.py +++ b/tests/unit/test_benchmark_unified_utils.py @@ -1461,285 +1461,6 @@ def test_benchmark_docs_only_reference_python_runner() -> None: assert "python benchmarks/run_bench.py" in docs -def test_grafana_prometheus_datasource_uses_one_second_interval() -> None: - """Grafana should query Prometheus at the configured scrape interval.""" - datasource = ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "provisioning" - / "datasources" - / "prometheus.yml" - ).read_text() - - assert "timeInterval: 1s" in datasource - - -def test_vllm_dashboard_does_not_span_idle_gaps() -> None: - """vLLM latency panels should not keep showing stale benchmark values.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - for title in ( - "TTFT (Time To First Token)", - "TPOT (Time Per Output Token)", - "vLLM Request Rate", - ): - custom = panels[title]["fieldConfig"]["defaults"]["custom"] - assert custom["spanNulls"] is False - - latency_panels = ( - panels["TTFT (Time To First Token)"], - panels["TPOT (Time Per Output Token)"], - ) - for panel in latency_panels: - for target in panel["targets"]: - expr = target["expr"] - assert "[$__rate_interval]" in expr - assert "[5m]" not in expr - assert "and on()" in expr - assert "sum(increase(vllm:request_success_total" in expr - - -def test_vllm_request_rate_panel_uses_aggregate_series() -> None: - """vLLM request rate should render as one aggregate series.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - expr = panels["vLLM Request Rate"]["targets"][0]["expr"] - - assert expr.startswith("max_over_time") - assert "sum(increase(vllm:request_success_total" in expr - assert "[1m]" in expr - assert "/ 60" in expr - assert "max_over_time" in expr - assert "[2m:1s]" in expr - assert "or vector(0)" in expr - - -def test_dashboard_service_status_panels_use_scrape_health() -> None: - """Service status panels should show DOWN when scrape targets have no data.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - assert panels["DaseR"]["targets"][0]["expr"] == 'max(up{job="daser"}) or vector(0)' - assert panels["vLLM"]["targets"][0]["expr"] == 'max(up{job="vllm"}) or vector(0)' - - for title in ("DaseR", "vLLM"): - mappings = panels[title]["fieldConfig"]["defaults"]["mappings"] - assert mappings[0]["options"]["0"]["text"] == "DOWN" - assert mappings[0]["options"]["1"]["text"] == "UP" - - -def test_daser_dashboard_hit_rate_panels_render_zero_for_missing_hits() -> None: - """Hit-rate panels should not go empty when only miss counters exist.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - request_hit_rate = panels["Request Hit Rate"]["targets"][0]["expr"] - l1_hit_rate = panels["L1 Hit Rate"]["targets"][0]["expr"] - - assert ( - request_hit_rate - == '(sum(rate(daser_cache_lookup_total{job="daser", result="hit"}[5m])) ' - 'or vector(0)) / clamp_min(sum(rate(daser_cache_lookup_total{job="daser"}' - "[5m])), 1)" - ) - assert "or vector(0)" in l1_hit_rate - - -def test_daser_dashboard_uses_fixed_daser_job_label() -> None: - """DaseR panels should not depend on a browser-local job variable.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - - assert dashboard.get("templating", {}).get("list", []) == [] - - for panel in dashboard["panels"]: - for target in panel.get("targets", []): - expr = target.get("expr", "") - if "daser_" in expr: - assert "$job" not in expr - assert 'job="daser"' in expr - - -def test_daser_dashboard_transfer_latency_omits_p99() -> None: - """Transfer latency should show p50 and p95 only.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - legends = { - target["legendFormat"] for target in panels["Transfer Latency"]["targets"] - } - - assert legends == {"p50 {{op}}", "p95 {{op}}"} - - -def test_daser_dashboard_does_not_span_idle_gaps() -> None: - """DaseR panels should leave benchmark idle gaps disconnected.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - for title in ( - "Cache Lookups", - "Prefix Reuse Distribution (tokens)", - "Evictions & Late Commits", - "Transfer Latency", - "Throughput GB/s", - "Chunk Size", - "L1 Hit Rate", - "L1 Usage", - ): - custom = panels[title]["fieldConfig"]["defaults"]["custom"] - assert custom["spanNulls"] is False - - -def test_daser_dashboard_l1_panels_tolerate_missing_l1_misses() -> None: - """L1 panels should use aggregate expressions that survive missing labels.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - hit_rate = panels["L1 Hit Rate"]["targets"][0]["expr"] - usage = panels["L1 Usage"]["targets"][0]["expr"] - - assert "sum(increase(daser_l1_hits_total" in hit_rate - assert "sum(increase(daser_l1_misses_total" in hit_rate - assert "max_over_time" in hit_rate - assert "[2m:1s]" in hit_rate - assert "or vector(0)" in hit_rate - assert "sum(daser_l1_bytes_used" in usage - assert "sum(daser_l1_bytes_capacity" in usage - assert "max_over_time" in usage - assert "[2m:1s]" in usage - assert "or vector(0)" in usage - - -def test_daser_dashboard_sparse_series_panels_render_zero_fallbacks() -> None: - """Sparse benchmark panels should emit a visible zero series outside runs.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - l1_hit_rate = panels["L1 Hit Rate"]["targets"][0]["expr"] - l1_usage = panels["L1 Usage"]["targets"][0]["expr"] - request_rate = panels["vLLM Request Rate"]["targets"][0]["expr"] - - assert "increase(daser_l1_hits_total" in l1_hit_rate - assert "increase(daser_l1_misses_total" in l1_hit_rate - assert "max_over_time" in l1_hit_rate - assert "[2m:1s]" in l1_hit_rate - assert "or vector(0)" in l1_hit_rate - assert "max_over_time" in l1_usage - assert "[2m:1s]" in l1_usage - assert "or vector(0)" in l1_usage - assert "increase(vllm:request_success_total" in request_rate - assert "max_over_time" in request_rate - assert "[2m:1s]" in request_rate - assert "/ 60" in request_rate - assert "or vector(0)" in request_rate - - -def test_daser_dashboard_sparse_panels_use_visible_series_styling() -> None: - """Sparse panels should make short benchmark samples visible.""" - dashboard = json.loads( - ( - REPO_ROOT - / "deploy" - / "monitoring" - / "grafana" - / "dashboards" - / "daser-overview.json" - ).read_text() - ) - panels = {panel["title"]: panel for panel in dashboard["panels"]} - - for title in ("L1 Hit Rate", "L1 Usage", "vLLM Request Rate"): - panel = panels[title] - custom = panel["fieldConfig"]["defaults"]["custom"] - assert panel["options"]["legend"]["displayMode"] == "list" - assert custom["showPoints"] == "auto" - assert custom["lineWidth"] >= 2 - assert custom["fillOpacity"] == 0 - - def test_daser_metrics_probe_reports_prometheus_scrape_state( monkeypatch, capsys ) -> None: