From 6a6ddea1a7f94e87a8cf33a67e8ca1b6676b8f9e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 27 Feb 2026 12:14:25 +0000 Subject: [PATCH 01/56] feat: add plugin --- dashboards/customized/influxdb_v3/README.md | 202 ++++++++++++++++++ .../customized/influxdb_v3/plugin/__init__.py | 85 ++++++++ .../influxdb_v3/plugin/rollup_6h.py | 188 ++++++++++++++++ .../influxdb_v3/plugin/rollup_monthly.py | 98 +++++++++ .../customized/influxdb_v3/plugin/utils.py | 166 ++++++++++++++ 5 files changed, 739 insertions(+) create mode 100644 dashboards/customized/influxdb_v3/README.md create mode 100644 dashboards/customized/influxdb_v3/plugin/__init__.py create mode 100644 dashboards/customized/influxdb_v3/plugin/rollup_6h.py create mode 100644 dashboards/customized/influxdb_v3/plugin/rollup_monthly.py create mode 100644 dashboards/customized/influxdb_v3/plugin/utils.py diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md new file mode 100644 index 0000000..e47f983 --- /dev/null +++ b/dashboards/customized/influxdb_v3/README.md @@ -0,0 +1,202 @@ +# InfluxDB 3 Aggregation Triggers for Analytics Dashboards + +This document explains how aggregated analytics tables are produced in **InfluxDB 3** using **scheduled triggers and a Python plugin**. +These aggregates are used by Grafana dashboards for AI DIAL Realtime Analytics. + +--- + +## Database Layout + +We use **two databases**: + +| Database | Purpose | +| --------------- | ---------------------------------------------------------- | +| `default` | Stores **raw analytics events** (source of truth). | +| `analytics_agg` | Stores **aggregated / roll-up tables** used by dashboards. | + +--- + +## Aggregation Tables (in `analytics_agg`) + +Aggregated data is written into the following **tables** (tables are created automatically on first write): + +| Table Name | Description | +| --------------------- | -------------------------------------------------------------- | +| `default_agg_stats` | 6-hour aggregates: tokens, price, request counts, unique users | +| `default_agg_topic` | Topic-level aggregates (request counts, tokens, price) | +| `default_agg_topic_2` | Additional topic-level metrics with alternative grouping | +| `default_agg_kpi` | User / project-level KPIs (cost, usage, tokens) | +| `default_agg_chatid` | Request counts grouped by `chat_id` | +| `default_agg_month` | Monthly roll-ups used by KPI dashboards | + +> Tables are **schema-on-write** in InfluxDB 3 — no explicit creation step is required. + +--- + +## Aggregation Logic Overview + +All aggregation logic lives in a **single Python scheduled plugin**, structured as a small package: + +```txt +influx_v3/plugin/ + __init__.py # Scheduled plugin entrypoint + rollup_6h.py # 6-hour aggregations + rollup_monthly.py # Monthly aggregations + utils.py # Shared helpers (time windows, SQL, writing) +``` + +This plugin: + +* queries raw data from `default` +* writes roll-ups into `analytics_agg` +* supports **scheduled execution** and **manual backfills** using the same code paths + +--- + +## Scheduled Triggers + +InfluxDB 3 executes plugins via **triggers**, which can be time-based using `cron:` or `every:` specifications. + +We define **two triggers**, both referencing the same plugin package. + +--- + +### 6-Hour Aggregation Trigger + +Runs **exactly at**: + +```txt +00:00, 06:00, 12:00, 18:00 (UTC) +``` + +Trigger spec: + +```txt +cron:0 0 */6 * * * +``` + +What it does: + +* aggregates the previous 6-hour window of raw data +* writes results into: + + * `default_agg_stats` + * `default_agg_topic` + * `default_agg_topic_2` + * `default_agg_kpi` + * `default_agg_chatid` + +Example trigger creation: + +```sh +influxdb3 create trigger \ + --database default \ + --path analytics_rollups \ + --trigger-spec "cron:0 0 */6 * * *" \ + --trigger-arguments mode=6h,raw_table=analytics,agg_database=analytics_agg,window_hours=6,offset_minutes=2 \ + analytics_6h_rollups +``` + +--- + +### Monthly Aggregation Trigger + +Runs at: + +```txt +00:00:00 UTC on the 1st day of each month +``` + +Trigger spec: + +```txt +cron:0 0 0 1 * * +``` + +What it does: + +* reads from 6-hour aggregate tables +* computes monthly totals, averages, and uniques +* writes results into `default_agg_month` + +Example trigger creation: + +```sh +influxdb3 create trigger \ + --database default \ + --path analytics_rollups \ + --trigger-spec "cron:0 0 0 1 * *" \ + --trigger-arguments mode=monthly,agg_database=analytics_agg \ + analytics_monthly_rollups +``` + +--- + +## Backfill Procedure + +Scheduled triggers **do not process historical data automatically**. +If you already have existing raw data, you **must backfill aggregates explicitly**. + +### Built-in Backfill Support + +The plugin supports two optional arguments: + +* `start_time` — RFC3339 timestamp (inclusive) +* `end_time` — RFC3339 timestamp (exclusive) + +When provided, these **override the scheduled window**. + +Example (single 6-hour window): + +```text +mode=6h, +raw_table=analytics, +agg_database=analytics_agg, +start_time=2026-01-01T00:00:00Z, +end_time=2026-01-01T06:00:00Z +``` + +--- + +### Recommended Backfill Strategy + +### Step 1: Disable scheduled triggers + +```sh +influxdb3 update trigger analytics_6h_rollups --disable +``` + +### Step 2: Backfill in deterministic 6-hour windows + +* Run the plugin repeatedly for each window +* Windows must not overlap +* Use the same plugin and logic as production + +Typical approaches: + +* one-off Kubernetes Job +* admin CLI loop +* temporary high-frequency trigger + +### Step 3: Re-enable scheduled triggers + +```bash +influxdb3 update trigger analytics_6h_rollups --enable +``` + +Because: + +* each aggregate point is written with a deterministic timestamp +* tags uniquely identify the aggregation dimensions + +👉 **Re-running the same window is safe and idempotent.** + +--- + +## Validation + +After triggers are enabled: + +* Query `analytics_agg.default_agg_stats` to confirm 6-hour data +* Query `analytics_agg.default_agg_month` after the first month boundary +* Verify Grafana dashboards point to `analytics_agg` tables diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py new file mode 100644 index 0000000..e4e037d --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -0,0 +1,85 @@ +""" +{ + "plugin_type": [ + "scheduled" + ], + "scheduled_args_config": [ + { + "name": "mode", + "example": "6h", + "description": "Which rollup to run: '6h' (default) or 'monthly'.", + "required": false + }, + { + "name": "raw_table", + "example": "analytics", + "description": "Raw source table name (in the trigger database).", + "required": false + }, + { + "name": "agg_database", + "example": "analytics_agg", + "description": "Database to write aggregates into.", + "required": false + }, + { + "name": "window_hours", + "example": "6", + "description": "For mode=6h: window size in hours.", + "required": false + }, + { + "name": "offset_minutes", + "example": "2", + "description": "For mode=6h: shift end of window backward to avoid late data.", + "required": false + }, + { + "name": "start_time", + "example": "2026-01-01T00:00:00Z", + "description": "Optional backfill start (RFC3339). If set with end_time, overrides call_time-derived window.", + "required": false + }, + { + "name": "end_time", + "example": "2026-01-01T06:00:00Z", + "description": "Optional backfill end (RFC3339). If set with start_time, overrides call_time-derived window.", + "required": false + } + ] +} +""" + +from __future__ import annotations + +import uuid + +from .rollup_6h import run_6h +from .rollup_monthly import run_monthly +from .utils import parse_call_time + + +def process_scheduled_call(influxdb3_local, call_time, args=None): + """ + InfluxDB 3 scheduled plugin entrypoint. + """ + task_id = str(uuid.uuid4()) + args = args or {} + + ct = parse_call_time(call_time) + mode = str(args.get("mode", "6h")).strip().lower() + + influxdb3_local.info( + f"[{task_id}] scheduled call mode={mode} call_time={ct.isoformat()} args={args}" + ) + + try: + if mode in ("month", "monthly"): + run_monthly(influxdb3_local, ct, args, task_id=task_id) + else: + run_6h(influxdb3_local, ct, args, task_id=task_id) + + influxdb3_local.info(f"[{task_id}] completed OK") + except Exception as e: + influxdb3_local.error(f"[{task_id}] failed: {e}") + raise diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_6h.py b/dashboards/customized/influxdb_v3/plugin/rollup_6h.py new file mode 100644 index 0000000..1106590 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/rollup_6h.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict + +from .utils import ( + query_rows, + rfc3339, + token_class_case_sql, + window_from_args_or_call_time, + write_points, +) + + +def run_6h( + influxdb3_local, call_time: datetime, args: Dict[str, Any], *, task_id: str +) -> None: + raw_table = str(args.get("raw_table", "analytics")) + agg_db = str(args.get("agg_database", "analytics_agg")) + + start, end = window_from_args_or_call_time( + call_time, args, window_hours_default=6, offset_minutes_default=2 + ) + start_s = rfc3339(start) + end_s = rfc3339(end) + + influxdb3_local.info(f"[{task_id}] 6h rollup window: {start_s} .. {end_s}") + + # 1) default_agg_stats + stats_sql = f""" +SELECT + '{start_s}' AS time, + deployment, + model, + project_id, + parent_deployment, + language, + SUM(prompt_tokens) AS prompt_tokens, + SUM(completion_tokens) AS completion_tokens, + SUM(price) AS price, + SUM(number_request_messages) AS number_request_messages, + SUM(deployment_price) AS deployment_price, + COUNT(*) AS request_count, + COUNT(DISTINCT user_hash) AS unique_user_count +FROM {raw_table} +WHERE time >= '{start_s}' AND time < '{end_s}' +GROUP BY deployment, model, project_id, parent_deployment, language +""" + stats_rows = query_rows(influxdb3_local, stats_sql) + write_points( + influxdb3_local, + db_name=agg_db, + table_name="default_agg_stats", + rows=stats_rows, + time_col="time", + tag_cols=( + "deployment", + "model", + "project_id", + "parent_deployment", + "language", + ), + field_cols=( + "prompt_tokens", + "completion_tokens", + "price", + "number_request_messages", + "deployment_price", + "request_count", + "unique_user_count", + ), + task_id=task_id, + ) + + # 2) default_agg_topic + default_agg_topic_2 (same content, two tables like your current setup) + topic_sql = f""" +SELECT + '{start_s}' AS time, + title, + topic, + model, + COUNT(*) AS topic_count, + SUM(number_request_messages) AS number_request_messages, + SUM(price) AS price, + SUM(prompt_tokens) AS prompt_tokens, + SUM(completion_tokens) AS completion_tokens +FROM {raw_table} +WHERE time >= '{start_s}' AND time < '{end_s}' +GROUP BY title, topic, model +""" + topic_rows = query_rows(influxdb3_local, topic_sql) + + for table in ("default_agg_topic", "default_agg_topic_2"): + write_points( + influxdb3_local, + db_name=agg_db, + table_name=table, + rows=topic_rows, + time_col="time", + tag_cols=("title", "topic", "model"), + field_cols=( + "topic_count", + "number_request_messages", + "price", + "prompt_tokens", + "completion_tokens", + ), + task_id=task_id, + ) + + # 3) token class histogram (kept compatible with your old task; stored in default_agg_topic by default) + token_table = str(args.get("token_class_table", "default_agg_topic")) + token_sql = f""" +SELECT + '{start_s}' AS time, + CASE WHEN user_hash = 'undefined' THEN 'project' ELSE 'user' END AS user_type, + {token_class_case_sql()} AS prompt_token_class, + COUNT(*) AS request_count +FROM {raw_table} +WHERE time >= '{start_s}' AND time < '{end_s}' +GROUP BY user_type, prompt_token_class +""" + token_rows = query_rows(influxdb3_local, token_sql) + write_points( + influxdb3_local, + db_name=agg_db, + table_name=token_table, + rows=token_rows, + time_col="time", + tag_cols=("user_type", "prompt_token_class"), + field_cols=("request_count",), + task_id=task_id, + ) + + # 4) default_agg_kpi + kpi_sql = f""" +SELECT + '{start_s}' AS time, + user_hash, + project_id, + parent_deployment, + title, + COUNT(*) AS request_count, + SUM(completion_tokens) AS completion_tokens, + SUM(prompt_tokens) AS prompt_tokens, + SUM(price) AS cost +FROM {raw_table} +WHERE time >= '{start_s}' AND time < '{end_s}' +GROUP BY user_hash, project_id, parent_deployment, title +""" + kpi_rows = query_rows(influxdb3_local, kpi_sql) + write_points( + influxdb3_local, + db_name=agg_db, + table_name="default_agg_kpi", + rows=kpi_rows, + time_col="time", + tag_cols=("user_hash", "project_id", "parent_deployment", "title"), + field_cols=( + "cost", + "request_count", + "completion_tokens", + "prompt_tokens", + ), + task_id=task_id, + ) + + # 5) default_agg_chatid + chat_sql = f""" +SELECT + '{start_s}' AS time, + chat_id, + COUNT(*) AS request_count +FROM {raw_table} +WHERE time >= '{start_s}' AND time < '{end_s}' +GROUP BY chat_id +""" + chat_rows = query_rows(influxdb3_local, chat_sql) + write_points( + influxdb3_local, + db_name=agg_db, + table_name="default_agg_chatid", + rows=chat_rows, + time_col="time", + tag_cols=("chat_id",), + field_cols=("request_count",), + task_id=task_id, + ) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py new file mode 100644 index 0000000..cabdb1d --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict + +from .utils import query_rows, rfc3339, write_points + + +def _month_start(dt: datetime) -> datetime: + dt = dt.astimezone(timezone.utc) + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + +def run_monthly( + influxdb3_local, call_time: datetime, args: Dict[str, Any], *, task_id: str +) -> None: + agg_db = str(args.get("agg_database", "analytics_agg")) + + # Previous month window: [prev_month_start, this_month_start) + this_month = _month_start(call_time) + prev_month = _month_start(this_month - timedelta(days=32)) + + start_s = rfc3339(prev_month) + end_s = rfc3339(this_month) + stamp_s = start_s # stamp results at first day of prev month + + influxdb3_local.info( + f"[{task_id}] monthly rollup window: {start_s} .. {end_s}" + ) + + # API-level (project_id) rollups from default_agg_stats + api_sql = f""" +SELECT + '{stamp_s}' AS time, + SUM(price) AS total_cost_per_api, + AVG(price) AS avg_cost_per_api, + COUNT(DISTINCT project_id) AS active_apis, + SUM(request_count) AS total_rc_per_api, + AVG(request_count) AS avg_rc_per_api +FROM default_agg_stats +WHERE time >= '{start_s}' AND time < '{end_s}' + AND project_id IS NOT NULL AND project_id <> '' +""" + api_rows = query_rows(influxdb3_local, api_sql) + + # Model-level rollups from default_agg_stats + model_sql = f""" +SELECT + '{stamp_s}' AS time, + SUM(price) AS total_cost_per_model, + AVG(price) AS avg_cost_per_model +FROM default_agg_stats +WHERE time >= '{start_s}' AND time < '{end_s}' +""" + model_rows = query_rows(influxdb3_local, model_sql) + + # User-level rollups from default_agg_kpi (exclude undefined like your Flux) + user_sql = f""" +SELECT + '{stamp_s}' AS time, + SUM(cost) AS total_user_cost, + AVG(cost) AS avg_cost_per_user, + COUNT(DISTINCT user_hash) AS unique_users +FROM default_agg_kpi +WHERE time >= '{start_s}' AND time < '{end_s}' + AND user_hash IS NOT NULL AND user_hash <> 'undefined' +""" + user_rows = query_rows(influxdb3_local, user_sql) + + merged: Dict[str, Any] = {"time": stamp_s} + if api_rows: + merged.update(api_rows[0]) + if model_rows: + merged.update(model_rows[0]) + if user_rows: + merged.update(user_rows[0]) + + write_points( + influxdb3_local, + db_name=agg_db, + table_name="default_agg_month", + rows=[merged], + time_col="time", + tag_cols=(), + field_cols=( + "total_user_cost", + "avg_cost_per_user", + "unique_users", + "total_cost_per_api", + "avg_cost_per_api", + "active_apis", + "total_rc_per_api", + "avg_rc_per_api", + "total_cost_per_model", + "avg_cost_per_model", + ), + task_id=task_id, + ) diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py new file mode 100644 index 0000000..12e9372 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence, Tuple + +if TYPE_CHECKING: + # LineBuilder is available in the processing runtime (as used by official plugins). + class LineBuilder: + def __init__(self, measurement: str) -> None: ... + def time_ns(self, value: int) -> None: ... + def tag(self, key: str, value: str) -> None: ... + def string_field(self, key: str, value: str) -> None: ... + def int64_field(self, key: str, value: int) -> None: ... + def float64_field(self, key: str, value: float) -> None: ... + def bool_field(self, key: str, value: bool) -> None: ... + + +def parse_call_time(call_time: Any) -> datetime: + """ + Best-effort conversion to a timezone-aware UTC datetime. + """ + if isinstance(call_time, datetime): + return ( + call_time + if call_time.tzinfo + else call_time.replace(tzinfo=timezone.utc) + ) + + if isinstance(call_time, str): + s = call_time.strip() + # Accept ...Z + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s).astimezone(timezone.utc) + + return datetime.now(tz=timezone.utc) + + +def parse_rfc3339(s: str) -> datetime: + s = s.strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s).astimezone(timezone.utc) + + +def rfc3339(dt: datetime) -> str: + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def ns(dt: datetime) -> int: + return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) + + +def window_from_args_or_call_time( + call_time: datetime, + args: Dict[str, Any], + *, + window_hours_default: int = 6, + offset_minutes_default: int = 2, +) -> Tuple[datetime, datetime]: + """ + If args contains start_time/end_time => backfill window. + Else uses call_time to compute [end - window, end) with optional offset. + """ + start_arg = args.get("start_time") + end_arg = args.get("end_time") + + if start_arg and end_arg: + start = parse_rfc3339(str(start_arg)) + end = parse_rfc3339(str(end_arg)) + if end <= start: + raise ValueError( + f"end_time must be > start_time (got {start_arg} .. {end_arg})" + ) + return start, end + + window_hours = int(args.get("window_hours", window_hours_default)) + offset_minutes = int(args.get("offset_minutes", offset_minutes_default)) + + end = call_time - timedelta(minutes=offset_minutes) + start = end - timedelta(hours=window_hours) + return start, end + + +def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: + """ + Execute SQL and return list of row dicts. + """ + res = influxdb3_local.query(sql) + return list(res) if res else [] + + +def write_points( + influxdb3_local, + *, + db_name: str, + table_name: str, + rows: Iterable[Dict[str, Any]], + time_col: str, + tag_cols: Sequence[str], + field_cols: Sequence[str], + task_id: str, +) -> int: + """ + Write each row as a point into db_name.table_name. + time_col can be RFC3339 string, datetime, or ns int. + """ + written = 0 + + for r in rows: + b = LineBuilder(table_name) + + t = r.get(time_col) + if isinstance(t, int): + b.time_ns(t) + elif isinstance(t, datetime): + b.time_ns(ns(t)) + elif isinstance(t, str): + b.time_ns(ns(parse_rfc3339(t))) + else: + # fallback + b.time_ns(ns(datetime.now(tz=timezone.utc))) + + for k in tag_cols: + v = r.get(k) + if v is None: + continue + # tags are always strings + b.tag(k, str(v)) + + for k in field_cols: + v = r.get(k) + if v is None: + continue + if isinstance(v, bool): + b.bool_field(k, v) + elif isinstance(v, int): + b.int64_field(k, v) + elif isinstance(v, float): + b.float64_field(k, v) + else: + b.string_field(k, str(v)) + + influxdb3_local.write_to_db(db_name, b) + written += 1 + + influxdb3_local.info( + f"[{task_id}] wrote {written} points -> {db_name}.{table_name}" + ) + return written + + +def token_class_case_sql() -> str: + """ + Same thresholds/order as your Flux task. + """ + return """ +CASE + WHEN prompt_tokens >= 50000 THEN 'class_1' + WHEN prompt_tokens > 10000 THEN 'class_2' + WHEN prompt_tokens > 5000 THEN 'class_3' + WHEN prompt_tokens > 1000 THEN 'class_4' + WHEN prompt_tokens > 100 THEN 'class_5' + ELSE 'class_6' +END +""".strip() From 2dfec490c5bfcd00486a2fb9ddb918cf3fc25ac3 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 27 Feb 2026 14:31:25 +0000 Subject: [PATCH 02/56] fix: reformatting JSON with dashboard --- dashboards/public/dial_analytics.json | 2 +- .../public/dial_analytics_raw_data.json | 384 +++++++++--------- 2 files changed, 193 insertions(+), 193 deletions(-) diff --git a/dashboards/public/dial_analytics.json b/dashboards/public/dial_analytics.json index ba3ca82..92cc4a0 100644 --- a/dashboards/public/dial_analytics.json +++ b/dashboards/public/dial_analytics.json @@ -723,4 +723,4 @@ "uid": "435858b2-4106-4d11-b6e8-9b0ff107b470", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/dashboards/public/dial_analytics_raw_data.json b/dashboards/public/dial_analytics_raw_data.json index e16aa85..d7467d5 100644 --- a/dashboards/public/dial_analytics_raw_data.json +++ b/dashboards/public/dial_analytics_raw_data.json @@ -1,212 +1,212 @@ { - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.3" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.3" }, - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "influxdb", - "uid": "${datasource}" + "type": "grafana", + "uid": "-- Grafana --" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "inspect": false }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "price" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - } - ] - } - ] - }, - "gridPos": { - "h": 24, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true + { + "color": "red", + "value": 80 + } + ] + } }, - "pluginVersion": "10.0.3", - "targets": [ + "overrides": [ { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" + "matcher": { + "id": "byName", + "options": "price" }, - "query": "import \"influxdata/influxdb/schema\"\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\" or r._field == \"price\")\r\n |> keep(columns: [\"_time\", \"_field\", \"_value\", \"response_id\", \"project_id\", \"deployment\", \"model\"])\r\n |> schema.fieldsAsCols()\r\n |> keep(columns: [\"_time\", \"project_id\", \"deployment\", \"model\", \"price\"])\r\n |> group()\r\n |> sort(columns: [\"_time\"], desc: true)", - "refId": "A" - } - ], - "title": "Panel Title", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "_time": 0, - "deployment": 4, - "model": 5, - "price": 6, - "project_id": 1, - "title": 3, - "topic": 2 - }, - "renameByName": { - "_time": "Time (UTC)", - "deployment": "Deployment", - "model": "Model", - "price": "Price", - "project_id": "Project", - "title": "Title", - "topic": "Topic" + "properties": [ + { + "id": "unit", + "value": "currencyUSD" } - } + ] } - ], - "type": "table" - } - ], - "refresh": "1m", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "DIAL", - "ai-dial-analytics-realtime" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, - "description": "InfluxDB datasource", - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "influxdb", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" + ] + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, + "showHeader": true + }, + "pluginVersion": "10.0.3", + "targets": [ { "datasource": { "type": "influxdb", "uid": "${datasource}" }, - "definition": "buckets()", - "description": "A InfluxDB bucket name", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "INFLUX_BUCKET", - "options": [], - "query": "buckets()", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" + "query": "import \"influxdata/influxdb/schema\"\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\" or r._field == \"price\")\r\n |> keep(columns: [\"_time\", \"_field\", \"_value\", \"response_id\", \"project_id\", \"deployment\", \"model\"])\r\n |> schema.fieldsAsCols()\r\n |> keep(columns: [\"_time\", \"project_id\", \"deployment\", \"model\", \"price\"])\r\n |> group()\r\n |> sort(columns: [\"_time\"], desc: true)", + "refId": "A" } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "DIAL Analytics Raw Data", - "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ef", - "version": 1, - "weekStart": "" - } + ], + "title": "Panel Title", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "_time": 0, + "deployment": 4, + "model": 5, + "price": 6, + "project_id": 1, + "title": 3, + "topic": 2 + }, + "renameByName": { + "_time": "Time (UTC)", + "deployment": "Deployment", + "model": "Model", + "price": "Price", + "project_id": "Project", + "title": "Title", + "topic": "Topic" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DIAL", + "ai-dial-analytics-realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "description": "InfluxDB datasource", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "definition": "buckets()", + "description": "A InfluxDB bucket name", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "INFLUX_BUCKET", + "options": [], + "query": "buckets()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIAL Analytics Raw Data", + "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ef", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 8e53955c00fe438a1e903d1d91e7ddc145af888e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 27 Feb 2026 14:59:58 +0000 Subject: [PATCH 03/56] feat: add Influx DB3 basic Grafana dashboards --- .../dial_analytics_raw_data_v3.json | 212 +++++ dashboards/customized/dial_analytics_v3.json | 861 ++++++++++++++++++ .../public/dial_analytics_raw_data_v3.json | 212 +++++ dashboards/public/dial_analytics_v3.json | 726 +++++++++++++++ 4 files changed, 2011 insertions(+) create mode 100644 dashboards/customized/dial_analytics_raw_data_v3.json create mode 100644 dashboards/customized/dial_analytics_v3.json create mode 100644 dashboards/public/dial_analytics_raw_data_v3.json create mode 100644 dashboards/public/dial_analytics_v3.json diff --git a/dashboards/customized/dial_analytics_raw_data_v3.json b/dashboards/customized/dial_analytics_raw_data_v3.json new file mode 100644 index 0000000..39f1e2d --- /dev/null +++ b/dashboards/customized/dial_analytics_raw_data_v3.json @@ -0,0 +1,212 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.3" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "price" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + } + ] + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT\n time AS \"_time\",\n project_id,\n topic,\n title,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", + "refId": "A" + } + ], + "title": "Panel Title", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "_time": 0, + "deployment": 4, + "model": 5, + "price": 6, + "project_id": 1, + "title": 3, + "topic": 2 + }, + "renameByName": { + "_time": "Time (UTC)", + "deployment": "Deployment", + "model": "Model", + "price": "Price", + "project_id": "Project", + "title": "Title", + "topic": "Topic" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DIAL", + "ai-dial-analytics-realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "description": "InfluxDB datasource", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "definition": "buckets()", + "description": "A InfluxDB bucket name", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "INFLUX_BUCKET", + "options": [], + "query": "buckets()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIAL Analytics Raw Data", + "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3df", + "version": 1, + "weekStart": "" +} diff --git a/dashboards/customized/dial_analytics_v3.json b/dashboards/customized/dial_analytics_v3.json new file mode 100644 index 0000000..184c8b5 --- /dev/null +++ b/dashboards/customized/dial_analytics_v3.json @@ -0,0 +1,861 @@ +{ + "__requires": [ + { + "type": "panel", + "id": "ae3e-plotly-panel", + "name": "Plotly panel", + "version": "0.3.3" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.3" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "magnesium-wordcloud-panel", + "name": "Word cloud", + "version": "1.2.4" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "/.*/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT COUNT(DISTINCT user_hash)\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo;", + "refId": "A" + } + ], + "title": "Unique Users", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "gridPos": { + "h": 10, + "w": 22, + "x": 2, + "y": 0 + }, + "id": 12, + "options": { + "datasource_count_field": "_value", + "datasource_tags_field": "topic", + "series_index": 0, + "wordCloudOptions": { + "deterministic": true, + "enableTooltip": true, + "fontFamily": "arial", + "fontSizes": [ + 15, + 80 + ], + "padding": 1, + "rotationAngles": [ + 0, + 0 + ], + "rotations": 2, + "scale": "sqrt", + "spiral": "archimedean", + "transitionDuration": 800 + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT COUNT(*) AS count\nFROM analytics\nWHERE time >= v.timeRangeStart\n AND time < v.timeRangeStop\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY topic", + "refId": "A" + } + ], + "title": "Popular Topics", + "type": "magnesium-wordcloud-panel" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "description": "", + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 14, + "options": { + "config": { + "displayModeBar": false + }, + "data": "", + "layout": { + "font": { + "color": "lightgrey" + }, + "margin": { + "t": 0 + }, + "paper_bgcolor": "rgba(0,0,0,0)", + "plot_bgcolor": "rgba(0,0,0,0)", + "xaxis": { + "automargin": true, + "autorange": true, + "categoryorder": "category ascending", + "range": [ + -0.5, + 0.5 + ], + "showgrid": false, + "showticklabels": true, + "type": "category" + }, + "yaxis": { + "automargin": true, + "autorange": true, + "range": [ + -0.5, + 52.5 + ], + "showgrid": false, + "type": "category" + } + }, + "onclick": "//console.log(data)\n//window.updateVariables({query:{'var-project':'test'}, partial: true})", + "script": "var title = data.series[0].fields.find(x => x.name === \"title\")?.values\nvar topic = data.series[0].fields.find(x => x.name === \"topic\")?.values\nvar count = data.series[0].fields.find(x => x.name === \"_value\")?.values\n\nvar trace = {\n x: title?.buffer || title || [\"No data\"],\n y: topic?.buffer || topic || [\"No data\"],\n z: count?.buffer || count || [NaN],\n type: 'heatmap',\n colorscale: \"YlOrRd\",\n};\n\nreturn {data:[trace]};" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT COUNT(*) AS count\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'", + "refId": "A" + } + ], + "title": "Title-Topic Heatmap", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "topic": " " + } + } + } + ], + "type": "ae3e-plotly-panel" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "Requests", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT\n date_bin(INTERVAL '$__interval', time, TIMESTAMP '1970-01-01T00:00:00Z') AS _time,\n COUNT(*) AS _value\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY 1\nORDER BY 1", + "refId": "A" + } + ], + "title": "System Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [ + "request_count", + "prompt_tokens", + "completion_tokens", + "money" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n project_id,\n deployment,\n model,\n COUNT(DISTINCT user_hash) AS user_count,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id, deployment, model", + "refId": "A" + } + ], + "title": "Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 6, + "deployment": 1, + "model": 2, + "money": 7, + "project_id": 0, + "prompt_tokens": 5, + "request_count": 4, + "user_count": 3 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "deployment": "Deployment", + "model": "Model", + "money": "Money", + "number_request_messages": "", + "project_id": "Project", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count", + "user_count": "Users" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n project_id,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id", + "refId": "A" + } + ], + "title": "Project Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 3, + "money": 4, + "project_id": 0, + "prompt_tokens": 2, + "request_count": 1 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "model": "", + "money": "Money", + "number_request_messages": "", + "project_id": "Project", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n model,\n deployment,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY model, deployment", + "refId": "A" + } + ], + "title": "Deployment/Model Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 4, + "deployment": 0, + "model": 1, + "money": 5, + "prompt_tokens": 3, + "request_count": 2 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "deployment": "Deployment", + "model": "Model", + "money": "Money", + "number_request_messages": "", + "project_id": "", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DIAL", + "ai-dial-analytics-realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "description": "InfluxDB datasource", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "definition": "buckets()", + "description": "A InfluxDB bucket name", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "INFLUX_BUCKET", + "options": [], + "query": "buckets()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIAL Analytics", + "uid": "435858b2-4106-4d11-b6e8-9b0ff107b460", + "version": 1, + "weekStart": "" +} diff --git a/dashboards/public/dial_analytics_raw_data_v3.json b/dashboards/public/dial_analytics_raw_data_v3.json new file mode 100644 index 0000000..73d1b86 --- /dev/null +++ b/dashboards/public/dial_analytics_raw_data_v3.json @@ -0,0 +1,212 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.3" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "price" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + } + ] + } + ] + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT\n time AS _time,\n project_id,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", + "refId": "A" + } + ], + "title": "Panel Title", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "_time": 0, + "deployment": 4, + "model": 5, + "price": 6, + "project_id": 1, + "title": 3, + "topic": 2 + }, + "renameByName": { + "_time": "Time (UTC)", + "deployment": "Deployment", + "model": "Model", + "price": "Price", + "project_id": "Project", + "title": "Title", + "topic": "Topic" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DIAL", + "ai-dial-analytics-realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "description": "InfluxDB datasource", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "definition": "buckets()", + "description": "A InfluxDB bucket name", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "INFLUX_BUCKET", + "options": [], + "query": "buckets()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIAL Analytics Raw Data", + "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ef", + "version": 1, + "weekStart": "" +} diff --git a/dashboards/public/dial_analytics_v3.json b/dashboards/public/dial_analytics_v3.json new file mode 100644 index 0000000..6fb547b --- /dev/null +++ b/dashboards/public/dial_analytics_v3.json @@ -0,0 +1,726 @@ +{ + "__requires": [ + { + "type": "panel", + "id": "ae3e-plotly-panel", + "name": "Plotly panel", + "version": "0.3.3" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.3" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "magnesium-wordcloud-panel", + "name": "Word cloud", + "version": "1.2.4" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "/.*/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT COUNT(DISTINCT user_hash)\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo;", + "refId": "A" + } + ], + "title": "Unique Users", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "Requests", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "query": "SELECT\n time AS _time,\n COUNT(*) AS _value\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY time_bucket(INTERVAL '$__interval', time)\nORDER BY _time", + "refId": "A" + } + ], + "title": "System Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n project_id,\n deployment,\n model,\n COUNT(DISTINCT user_hash) AS user_count,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id, deployment, model", + "refId": "A" + } + ], + "title": "Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 6, + "deployment": 1, + "model": 2, + "money": 7, + "project_id": 0, + "prompt_tokens": 5, + "request_count": 4, + "user_count": 3 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "deployment": "Deployment", + "model": "Model", + "money": "Money", + "number_request_messages": "", + "project_id": "Project", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count", + "user_count": "Users" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n project_id,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id", + "refId": "A" + } + ], + "title": "Project Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 3, + "money": 4, + "project_id": 0, + "prompt_tokens": 2, + "request_count": 1 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "model": "", + "money": "Money", + "number_request_messages": "", + "project_id": "Project", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "money" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "Not configured" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Money" + } + ] + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "hide": false, + "query": "SELECT\n \"model\",\n \"deployment\",\n COUNT(*) AS request_count,\n SUM(COALESCE(\"prompt_tokens\", 0)) AS prompt_tokens,\n SUM(COALESCE(\"completion_tokens\", 0)) AS completion_tokens,\n SUM(COALESCE(\"price\", 0.0)) AS money\nFROM \"default\".\"analytics\"\nWHERE \"time\" >= $__timeFrom AND \"time\" < $__timeTo\nGROUP BY\n \"model\",\n \"deployment\"", + "refId": "A" + } + ], + "title": "Deployment/Model Stats Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_time": true + }, + "indexByName": { + "completion_tokens": 4, + "deployment": 0, + "model": 1, + "money": 5, + "prompt_tokens": 3, + "request_count": 2 + }, + "renameByName": { + "completion_tokens": "Completion tokens", + "deployment": "Deployment", + "model": "Model", + "money": "Money", + "number_request_messages": "", + "project_id": "", + "prompt_tokens": "Prompt tokens", + "request_count": "Request count" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Money" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DIAL", + "ai-dial-analytics-realtime" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "description": "InfluxDB datasource", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "influxdb", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "definition": "buckets()", + "description": "A InfluxDB bucket name", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "INFLUX_BUCKET", + "options": [], + "query": "buckets()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIAL Analytics", + "uid": "435858b2-4106-4d11-b6e8-9b0ff107b470", + "version": 1, + "weekStart": "" +} From da0952e673f79453e8d5ff120953c375f3f67338 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 27 Feb 2026 15:29:17 +0000 Subject: [PATCH 04/56] feat: fix dashboards --- .../dial_analytics_raw_data_v3.json | 21 +------------------ dashboards/customized/dial_analytics_v3.json | 21 +------------------ .../public/dial_analytics_raw_data_v3.json | 21 +------------------ dashboards/public/dial_analytics_v3.json | 21 +------------------ 4 files changed, 4 insertions(+), 80 deletions(-) diff --git a/dashboards/customized/dial_analytics_raw_data_v3.json b/dashboards/customized/dial_analytics_raw_data_v3.json index 39f1e2d..ba1bc27 100644 --- a/dashboards/customized/dial_analytics_raw_data_v3.json +++ b/dashboards/customized/dial_analytics_raw_data_v3.json @@ -171,31 +171,12 @@ "multi": false, "name": "datasource", "options": [], - "query": "influxdb", + "query": "influxdb3", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "definition": "buckets()", - "description": "A InfluxDB bucket name", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "INFLUX_BUCKET", - "options": [], - "query": "buckets()", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, diff --git a/dashboards/customized/dial_analytics_v3.json b/dashboards/customized/dial_analytics_v3.json index 184c8b5..ee26ffb 100644 --- a/dashboards/customized/dial_analytics_v3.json +++ b/dashboards/customized/dial_analytics_v3.json @@ -820,31 +820,12 @@ "multi": false, "name": "datasource", "options": [], - "query": "influxdb", + "query": "influxdb3", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "definition": "buckets()", - "description": "A InfluxDB bucket name", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "INFLUX_BUCKET", - "options": [], - "query": "buckets()", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, diff --git a/dashboards/public/dial_analytics_raw_data_v3.json b/dashboards/public/dial_analytics_raw_data_v3.json index 73d1b86..b4c7ce7 100644 --- a/dashboards/public/dial_analytics_raw_data_v3.json +++ b/dashboards/public/dial_analytics_raw_data_v3.json @@ -171,31 +171,12 @@ "multi": false, "name": "datasource", "options": [], - "query": "influxdb", + "query": "influxdb3", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "definition": "buckets()", - "description": "A InfluxDB bucket name", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "INFLUX_BUCKET", - "options": [], - "query": "buckets()", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, diff --git a/dashboards/public/dial_analytics_v3.json b/dashboards/public/dial_analytics_v3.json index 6fb547b..d900f0c 100644 --- a/dashboards/public/dial_analytics_v3.json +++ b/dashboards/public/dial_analytics_v3.json @@ -685,31 +685,12 @@ "multi": false, "name": "datasource", "options": [], - "query": "influxdb", + "query": "influxdb3", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "definition": "buckets()", - "description": "A InfluxDB bucket name", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "INFLUX_BUCKET", - "options": [], - "query": "buckets()", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, From 64d8a020d144e8fe9d4b9e25bda1c4336c13ca26 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 27 Feb 2026 16:54:56 +0000 Subject: [PATCH 05/56] fix: fix uids --- dashboards/customized/dial_analytics_raw_data_v3.json | 2 +- dashboards/customized/dial_analytics_v3.json | 2 +- dashboards/public/dial_analytics_raw_data_v3.json | 2 +- dashboards/public/dial_analytics_v3.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/customized/dial_analytics_raw_data_v3.json b/dashboards/customized/dial_analytics_raw_data_v3.json index ba1bc27..e58a1db 100644 --- a/dashboards/customized/dial_analytics_raw_data_v3.json +++ b/dashboards/customized/dial_analytics_raw_data_v3.json @@ -187,7 +187,7 @@ "timepicker": {}, "timezone": "", "title": "DIAL Analytics Raw Data", - "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3df", + "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3da", "version": 1, "weekStart": "" } diff --git a/dashboards/customized/dial_analytics_v3.json b/dashboards/customized/dial_analytics_v3.json index ee26ffb..fed7e25 100644 --- a/dashboards/customized/dial_analytics_v3.json +++ b/dashboards/customized/dial_analytics_v3.json @@ -836,7 +836,7 @@ "timepicker": {}, "timezone": "", "title": "DIAL Analytics", - "uid": "435858b2-4106-4d11-b6e8-9b0ff107b460", + "uid": "435858b2-4106-4d11-b6e8-9b0ff107b461", "version": 1, "weekStart": "" } diff --git a/dashboards/public/dial_analytics_raw_data_v3.json b/dashboards/public/dial_analytics_raw_data_v3.json index b4c7ce7..9fde285 100644 --- a/dashboards/public/dial_analytics_raw_data_v3.json +++ b/dashboards/public/dial_analytics_raw_data_v3.json @@ -187,7 +187,7 @@ "timepicker": {}, "timezone": "", "title": "DIAL Analytics Raw Data", - "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ef", + "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ea", "version": 1, "weekStart": "" } diff --git a/dashboards/public/dial_analytics_v3.json b/dashboards/public/dial_analytics_v3.json index d900f0c..ad3d52e 100644 --- a/dashboards/public/dial_analytics_v3.json +++ b/dashboards/public/dial_analytics_v3.json @@ -701,7 +701,7 @@ "timepicker": {}, "timezone": "", "title": "DIAL Analytics", - "uid": "435858b2-4106-4d11-b6e8-9b0ff107b470", + "uid": "435858b2-4106-4d11-b6e8-9b0ff107b471", "version": 1, "weekStart": "" } From 3cd15d9b010218ae971a5254baab0250d9d688f2 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 12:10:47 +0000 Subject: [PATCH 06/56] fix: fix dial_analytics_raw_data_v3 --- .../dial_analytics_raw_data_v3.json | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/dashboards/customized/dial_analytics_raw_data_v3.json b/dashboards/customized/dial_analytics_raw_data_v3.json index e58a1db..0233aa5 100644 --- a/dashboards/customized/dial_analytics_raw_data_v3.json +++ b/dashboards/customized/dial_analytics_raw_data_v3.json @@ -1,24 +1,4 @@ { - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.3" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], "annotations": { "list": [ { @@ -35,12 +15,11 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, + "id": 13, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -64,8 +43,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -108,15 +86,36 @@ }, "showHeader": true }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${datasource}" }, - "query": "SELECT\n time AS \"_time\",\n project_id,\n topic,\n title,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", - "refId": "A" + "editorMode": "code", + "format": "table", + "query": "SELECT\r\n time AS \"_time\",\r\n project_id,\r\n topic,\r\n title,\r\n deployment,\r\n model,\r\n price\r\nFROM analytics\r\nWHERE time >= $__timeFrom AND time < $__timeTo\r\nORDER BY time DESC", + "rawQuery": true, + "rawSql": "SELECT\n time AS \"_time\",\n project_id,\n topic,\n title,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Panel Title", @@ -149,9 +148,9 @@ "type": "table" } ], + "preload": false, "refresh": "1m", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 41, "tags": [ "DIAL", "ai-dial-analytics-realtime" @@ -160,22 +159,17 @@ "list": [ { "current": { - "selected": true, - "text": "default", - "value": "default" + "text": "InfluxDB-3", + "value": "influxdb3" }, "description": "InfluxDB datasource", - "hide": 0, "includeAll": false, "label": "Data source", - "multi": false, "name": "datasource", "options": [], - "query": "influxdb3", - "queryValue": "", + "query": "influxdb", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" } ] @@ -188,6 +182,5 @@ "timezone": "", "title": "DIAL Analytics Raw Data", "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3da", - "version": 1, - "weekStart": "" + "version": 3 } From fa318398090943a4ba92ea98e9bb5684891b440b Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 15:45:25 +0000 Subject: [PATCH 07/56] feat: add SQL to the DIAL Analytics dashboard --- dashboards/customized/dial_analytics.json | 256 ++++-- dashboards/customized/dial_analytics_v3.json | 842 ------------------- 2 files changed, 205 insertions(+), 893 deletions(-) delete mode 100644 dashboards/customized/dial_analytics_v3.json diff --git a/dashboards/customized/dial_analytics.json b/dashboards/customized/dial_analytics.json index f1683dc..2990606 100644 --- a/dashboards/customized/dial_analytics.json +++ b/dashboards/customized/dial_analytics.json @@ -1,16 +1,27 @@ { + "__inputs": [ + { + "name": "DS_INFLUXDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__elements": {}, "__requires": [ { "type": "panel", "id": "ae3e-plotly-panel", "name": "Plotly panel", - "version": "0.3.3" + "version": "0.5.0" }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.0.3" + "version": "12.0.0" }, { "type": "datasource", @@ -22,7 +33,7 @@ "type": "panel", "id": "magnesium-wordcloud-panel", "name": "Word cloud", - "version": "1.2.4" + "version": "1.2.11" }, { "type": "panel", @@ -59,12 +70,11 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -82,8 +92,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -106,6 +115,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -113,17 +123,40 @@ "fields": "/.*/", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\" and r._field == \"user_hash\")\r\n |> group()\r\n |> distinct()\r\n |> count()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_hash)\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Unique Users", @@ -164,14 +197,36 @@ "transitionDuration": 800 } }, + "pluginVersion": "1.2.11", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\")\r\n |> group(columns: [\"topic\"])\r\n |> count()\r\n |> group()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT topic, COUNT(*) AS _value\nFROM analytics\nWHERE time >= $__timeFrom \n AND time < $__timeTo \n AND topic IS NOT NULL\n AND TRIM(topic) <> ''\nGROUP BY topic", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Popular Topics", @@ -230,15 +285,36 @@ "onclick": "//console.log(data)\n//window.updateVariables({query:{'var-project':'test'}, partial: true})", "script": "var title = data.series[0].fields.find(x => x.name === \"title\")?.values\nvar topic = data.series[0].fields.find(x => x.name === \"topic\")?.values\nvar count = data.series[0].fields.find(x => x.name === \"_value\")?.values\n\nvar trace = {\n x: title?.buffer || title || [\"No data\"],\n y: topic?.buffer || topic || [\"No data\"],\n z: count?.buffer || count || [NaN],\n type: 'heatmap',\n colorscale: \"YlOrRd\",\n};\n\nreturn {data:[trace]};" }, - "pluginVersion": "9.2.4", + "pluginVersion": "0.5.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "import \"influxdata/influxdb/schema\"\r\n\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\")\r\n |> group(columns: [\"topic\", \"title\"])\r\n |> count()\r\n |> group()\r\n", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS _value, topic, title\nFROM analytics\nWHERE time >= $__timeFrom \n AND time < $__timeTo\nGROUP BY topic, title", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Title-Topic Heatmap", @@ -267,11 +343,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "hue", @@ -280,6 +358,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, @@ -302,8 +381,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -345,18 +423,41 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\")\r\n |> group()\r\n |> aggregateWindow(every: v.windowPeriod, fn: count)\r\n |> keep(columns: [\"_time\", \"_value\"])", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n $__dateBin(time) AS time,\n COUNT(*) AS count\nFROM analytics\nWHERE time >= $__timeFrom\n AND time < $__timeTo\nGROUP BY 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "System Usage", @@ -385,8 +486,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -455,16 +555,37 @@ } ] }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "hide": false, "query": "import \"influxdata/influxdb/schema\"\r\n\r\ngetOrDefault = (f, d) => if exists f then f else d\r\n\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"prompt_tokens\" or r._field == \"completion_tokens\" or r._field == \"number_request_messages\" or r._field == \"user_hash\" or r._field == \"price\")\r\n |> schema.fieldsAsCols()\r\n |> group(columns: [\"project_id\", \"deployment\", \"model\", \"user_hash\"])\r\n |> reduce(\r\n fn: (r, accumulator) => ({\r\n request_count: accumulator.request_count + 1,\r\n prompt_tokens: getOrDefault(f: r.prompt_tokens, d: 0) + accumulator.prompt_tokens,\r\n completion_tokens: getOrDefault(f: r.completion_tokens, d: 0) + accumulator.completion_tokens,\r\n money: getOrDefault(f: r.price, d: 0.0) + accumulator.money\r\n }),\r\n identity: {request_count: 0, prompt_tokens: 0, completion_tokens: 0, money: 0.0}\r\n )\r\n |> group(columns: [\"project_id\", \"deployment\", \"model\"])\r\n |> reduce(\r\n fn: (r, accumulator) => ({\r\n user_count: accumulator.user_count + 1,\r\n request_count: r.request_count + accumulator.request_count,\r\n prompt_tokens: r.prompt_tokens + accumulator.prompt_tokens,\r\n completion_tokens: r.completion_tokens + accumulator.completion_tokens,\r\n money: r.money + accumulator.money\r\n }),\r\n identity: {user_count: 0, request_count: 0, prompt_tokens: 0, completion_tokens: 0, money: 0.0}\r\n )\r\n |> group()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n project_id,\n deployment,\n model,\n COUNT(DISTINCT user_hash) AS user_count,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id, deployment, model", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Stats Table", @@ -536,8 +657,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -601,16 +721,37 @@ } ] }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "hide": false, "query": "import \"influxdata/influxdb/schema\"\r\n\r\ngetOrDefault = (f, d) => if exists f then f else d\r\n\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"prompt_tokens\" or r._field == \"completion_tokens\" or r._field == \"number_request_messages\" or r._field == \"price\")\r\n |> schema.fieldsAsCols()\r\n |> group(columns: [\"project_id\"])\r\n |> reduce(\r\n fn: (r, accumulator) => ({\r\n request_count: accumulator.request_count + 1,\r\n prompt_tokens: getOrDefault(f: r.prompt_tokens, d: 0) + accumulator.prompt_tokens,\r\n completion_tokens: getOrDefault(f: r.completion_tokens, d: 0) + accumulator.completion_tokens,\r\n money: getOrDefault(f: r.price, d: 0.0) + accumulator.money\r\n }),\r\n identity: {request_count: 0, prompt_tokens: 0, completion_tokens: 0, money: 0.0}\r\n )\r\n |> group()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n project_id,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Project Stats Table", @@ -677,8 +818,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -742,16 +882,37 @@ } ] }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "hide": false, "query": "import \"influxdata/influxdb/schema\"\r\n\r\ngetOrDefault = (f, d) => if exists f then f else d\r\n\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"prompt_tokens\" or r._field == \"completion_tokens\" or r._field == \"number_request_messages\" or r._field == \"price\")\r\n |> schema.fieldsAsCols()\r\n |> group(columns: [\"model\", \"deployment\"])\r\n |> reduce(\r\n fn: (r, accumulator) => ({\r\n request_count: accumulator.request_count + 1,\r\n prompt_tokens: getOrDefault(f: r.prompt_tokens, d: 0) + accumulator.prompt_tokens,\r\n completion_tokens: getOrDefault(f: r.completion_tokens, d: 0) + accumulator.completion_tokens,\r\n money: getOrDefault(f: r.price, d: 0.0) + accumulator.money\r\n }),\r\n identity: {request_count: 0, prompt_tokens: 0, completion_tokens: 0, money: 0.0}\r\n )\r\n |> group()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n model,\n deployment,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY model, deployment", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Deployment/Model Stats Table", @@ -799,8 +960,7 @@ } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 41, "tags": [ "DIAL", "ai-dial-analytics-realtime" @@ -809,41 +969,35 @@ "list": [ { "current": { - "selected": true, - "text": "default", - "value": "default" }, "description": "InfluxDB datasource", - "hide": 0, "includeAll": false, "label": "Data source", - "multi": false, "name": "datasource", "options": [], "query": "influxdb", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, "datasource": { "type": "influxdb", "uid": "${datasource}" }, - "definition": "buckets()", + "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))\n |> keep(columns: [\"name\", \"id\", \"retentionPeriod\"])\n |> sort(columns: [\"name\"])", "description": "A InfluxDB bucket name", - "hide": 0, "includeAll": false, - "multi": false, "name": "INFLUX_BUCKET", "options": [], "query": "buckets()", "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -856,6 +1010,6 @@ "timezone": "", "title": "DIAL Analytics", "uid": "435858b2-4106-4d11-b6e8-9b0ff107b460", - "version": 1, + "version": 19, "weekStart": "" -} \ No newline at end of file +} diff --git a/dashboards/customized/dial_analytics_v3.json b/dashboards/customized/dial_analytics_v3.json deleted file mode 100644 index fed7e25..0000000 --- a/dashboards/customized/dial_analytics_v3.json +++ /dev/null @@ -1,842 +0,0 @@ -{ - "__requires": [ - { - "type": "panel", - "id": "ae3e-plotly-panel", - "name": "Plotly panel", - "version": "0.3.3" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.3" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "magnesium-wordcloud-panel", - "name": "Word cloud", - "version": "1.2.4" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 0, - "y": 0 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT COUNT(DISTINCT user_hash)\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo;", - "refId": "A" - } - ], - "title": "Unique Users", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "gridPos": { - "h": 10, - "w": 22, - "x": 2, - "y": 0 - }, - "id": 12, - "options": { - "datasource_count_field": "_value", - "datasource_tags_field": "topic", - "series_index": 0, - "wordCloudOptions": { - "deterministic": true, - "enableTooltip": true, - "fontFamily": "arial", - "fontSizes": [ - 15, - 80 - ], - "padding": 1, - "rotationAngles": [ - 0, - 0 - ], - "rotations": 2, - "scale": "sqrt", - "spiral": "archimedean", - "transitionDuration": 800 - } - }, - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT COUNT(*) AS count\nFROM analytics\nWHERE time >= v.timeRangeStart\n AND time < v.timeRangeStop\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY topic", - "refId": "A" - } - ], - "title": "Popular Topics", - "type": "magnesium-wordcloud-panel" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "description": "", - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 14, - "options": { - "config": { - "displayModeBar": false - }, - "data": "", - "layout": { - "font": { - "color": "lightgrey" - }, - "margin": { - "t": 0 - }, - "paper_bgcolor": "rgba(0,0,0,0)", - "plot_bgcolor": "rgba(0,0,0,0)", - "xaxis": { - "automargin": true, - "autorange": true, - "categoryorder": "category ascending", - "range": [ - -0.5, - 0.5 - ], - "showgrid": false, - "showticklabels": true, - "type": "category" - }, - "yaxis": { - "automargin": true, - "autorange": true, - "range": [ - -0.5, - 52.5 - ], - "showgrid": false, - "type": "category" - } - }, - "onclick": "//console.log(data)\n//window.updateVariables({query:{'var-project':'test'}, partial: true})", - "script": "var title = data.series[0].fields.find(x => x.name === \"title\")?.values\nvar topic = data.series[0].fields.find(x => x.name === \"topic\")?.values\nvar count = data.series[0].fields.find(x => x.name === \"_value\")?.values\n\nvar trace = {\n x: title?.buffer || title || [\"No data\"],\n y: topic?.buffer || topic || [\"No data\"],\n z: count?.buffer || count || [NaN],\n type: 'heatmap',\n colorscale: \"YlOrRd\",\n};\n\nreturn {data:[trace]};" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT COUNT(*) AS count\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'", - "refId": "A" - } - ], - "title": "Title-Topic Heatmap", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "topic": " " - } - } - } - ], - "type": "ae3e-plotly-panel" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "Requests", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT\n date_bin(INTERVAL '$__interval', time, TIMESTAMP '1970-01-01T00:00:00Z') AS _time,\n COUNT(*) AS _value\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY 1\nORDER BY 1", - "refId": "A" - } - ], - "title": "System Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": [ - "request_count", - "prompt_tokens", - "completion_tokens", - "money" - ], - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n project_id,\n deployment,\n model,\n COUNT(DISTINCT user_hash) AS user_count,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id, deployment, model", - "refId": "A" - } - ], - "title": "Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 6, - "deployment": 1, - "model": 2, - "money": 7, - "project_id": 0, - "prompt_tokens": 5, - "request_count": 4, - "user_count": 3 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "deployment": "Deployment", - "model": "Model", - "money": "Money", - "number_request_messages": "", - "project_id": "Project", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count", - "user_count": "Users" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n project_id,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id", - "refId": "A" - } - ], - "title": "Project Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 3, - "money": 4, - "project_id": 0, - "prompt_tokens": 2, - "request_count": 1 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "model": "", - "money": "Money", - "number_request_messages": "", - "project_id": "Project", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 8, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n model,\n deployment,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY model, deployment", - "refId": "A" - } - ], - "title": "Deployment/Model Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 4, - "deployment": 0, - "model": 1, - "money": 5, - "prompt_tokens": 3, - "request_count": 2 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "deployment": "Deployment", - "model": "Model", - "money": "Money", - "number_request_messages": "", - "project_id": "", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "DIAL", - "ai-dial-analytics-realtime" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, - "description": "InfluxDB datasource", - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "influxdb3", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "DIAL Analytics", - "uid": "435858b2-4106-4d11-b6e8-9b0ff107b461", - "version": 1, - "weekStart": "" -} From dba38c2780aa7b0044602163b05dfca4033819f1 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 15:52:48 +0000 Subject: [PATCH 08/56] feat: update DIAL Analytics Raw Data dashboard --- dashboards/customized/dial_analytics.json | 3 +- .../customized/dial_analytics_raw_data.json | 74 +- .../dial_analytics_raw_data_v3.json | 186 ----- .../public/dial_analytics_raw_data_v3.json | 193 ----- dashboards/public/dial_analytics_v3.json | 707 ------------------ 5 files changed, 49 insertions(+), 1114 deletions(-) delete mode 100644 dashboards/customized/dial_analytics_raw_data_v3.json delete mode 100644 dashboards/public/dial_analytics_raw_data_v3.json delete mode 100644 dashboards/public/dial_analytics_v3.json diff --git a/dashboards/customized/dial_analytics.json b/dashboards/customized/dial_analytics.json index 2990606..97c7e52 100644 --- a/dashboards/customized/dial_analytics.json +++ b/dashboards/customized/dial_analytics.json @@ -968,8 +968,7 @@ "templating": { "list": [ { - "current": { - }, + "current": {}, "description": "InfluxDB datasource", "includeAll": false, "label": "Data source", diff --git a/dashboards/customized/dial_analytics_raw_data.json b/dashboards/customized/dial_analytics_raw_data.json index ebf5bb4..6840052 100644 --- a/dashboards/customized/dial_analytics_raw_data.json +++ b/dashboards/customized/dial_analytics_raw_data.json @@ -1,10 +1,21 @@ { + "__inputs": [ + { + "name": "DS_INFLUXDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.0.3" + "version": "12.0.0" }, { "type": "datasource", @@ -35,12 +46,11 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -64,8 +74,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -108,15 +117,36 @@ }, "showHeader": true }, - "pluginVersion": "10.0.3", + "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "import \"influxdata/influxdb/schema\"\r\nbucket = \"${INFLUX_BUCKET}\"\r\n\r\nfrom(bucket: bucket)\r\n |> range(start: v.timeRangeStart, stop:v.timeRangeStop)\r\n |> filter(fn: (r) => r._measurement == \"analytics\")\r\n |> filter(fn: (r) => r._field == \"number_request_messages\" or r._field == \"price\")\r\n |> keep(columns: [\"_time\", \"_field\", \"_value\", \"response_id\", \"project_id\", \"topic\", \"title\", \"deployment\", \"model\"])\r\n |> schema.fieldsAsCols()\r\n |> keep(columns: [\"_time\", \"project_id\", \"topic\", \"title\", \"deployment\", \"model\", \"price\"])\r\n |> group()\r\n |> sort(columns: [\"_time\"], desc: true)", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n time AS \"_time\",\n project_id,\n topic,\n title,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Panel Title", @@ -150,8 +180,7 @@ } ], "refresh": "1m", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 41, "tags": [ "DIAL", "ai-dial-analytics-realtime" @@ -159,42 +188,35 @@ "templating": { "list": [ { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, + "current": {}, "description": "InfluxDB datasource", - "hide": 0, "includeAll": false, "label": "Data source", - "multi": false, "name": "datasource", "options": [], "query": "influxdb", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, "datasource": { "type": "influxdb", "uid": "${datasource}" }, - "definition": "buckets()", + "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))\n |> keep(columns: [\"name\", \"id\", \"retentionPeriod\"])\n |> sort(columns: [\"name\"])", "description": "A InfluxDB bucket name", - "hide": 0, "includeAll": false, - "multi": false, "name": "INFLUX_BUCKET", "options": [], "query": "buckets()", "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -207,6 +229,6 @@ "timezone": "", "title": "DIAL Analytics Raw Data", "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3df", - "version": 1, + "version": 4, "weekStart": "" -} \ No newline at end of file +} diff --git a/dashboards/customized/dial_analytics_raw_data_v3.json b/dashboards/customized/dial_analytics_raw_data_v3.json deleted file mode 100644 index 0233aa5..0000000 --- a/dashboards/customized/dial_analytics_raw_data_v3.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 13, - "links": [], - "panels": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "price" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - } - ] - } - ] - }, - "gridPos": { - "h": 24, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "dataset": "iox", - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "editorMode": "code", - "format": "table", - "query": "SELECT\r\n time AS \"_time\",\r\n project_id,\r\n topic,\r\n title,\r\n deployment,\r\n model,\r\n price\r\nFROM analytics\r\nWHERE time >= $__timeFrom AND time < $__timeTo\r\nORDER BY time DESC", - "rawQuery": true, - "rawSql": "SELECT\n time AS \"_time\",\n project_id,\n topic,\n title,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ] - } - } - ], - "title": "Panel Title", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "_time": 0, - "deployment": 4, - "model": 5, - "price": 6, - "project_id": 1, - "title": 3, - "topic": 2 - }, - "renameByName": { - "_time": "Time (UTC)", - "deployment": "Deployment", - "model": "Model", - "price": "Price", - "project_id": "Project", - "title": "Title", - "topic": "Topic" - } - } - } - ], - "type": "table" - } - ], - "preload": false, - "refresh": "1m", - "schemaVersion": 41, - "tags": [ - "DIAL", - "ai-dial-analytics-realtime" - ], - "templating": { - "list": [ - { - "current": { - "text": "InfluxDB-3", - "value": "influxdb3" - }, - "description": "InfluxDB datasource", - "includeAll": false, - "label": "Data source", - "name": "datasource", - "options": [], - "query": "influxdb", - "refresh": 1, - "regex": "", - "type": "datasource" - } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "DIAL Analytics Raw Data", - "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3da", - "version": 3 -} diff --git a/dashboards/public/dial_analytics_raw_data_v3.json b/dashboards/public/dial_analytics_raw_data_v3.json deleted file mode 100644 index 9fde285..0000000 --- a/dashboards/public/dial_analytics_raw_data_v3.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.3" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "price" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - } - ] - } - ] - }, - "gridPos": { - "h": 24, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT\n time AS _time,\n project_id,\n deployment,\n model,\n price\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nORDER BY time DESC", - "refId": "A" - } - ], - "title": "Panel Title", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "_time": 0, - "deployment": 4, - "model": 5, - "price": 6, - "project_id": 1, - "title": 3, - "topic": 2 - }, - "renameByName": { - "_time": "Time (UTC)", - "deployment": "Deployment", - "model": "Model", - "price": "Price", - "project_id": "Project", - "title": "Title", - "topic": "Topic" - } - } - } - ], - "type": "table" - } - ], - "refresh": "1m", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "DIAL", - "ai-dial-analytics-realtime" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, - "description": "InfluxDB datasource", - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "influxdb3", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "DIAL Analytics Raw Data", - "uid": "98a34b22-dd32-4e2a-8c4c-81147d2ff3ea", - "version": 1, - "weekStart": "" -} diff --git a/dashboards/public/dial_analytics_v3.json b/dashboards/public/dial_analytics_v3.json deleted file mode 100644 index ad3d52e..0000000 --- a/dashboards/public/dial_analytics_v3.json +++ /dev/null @@ -1,707 +0,0 @@ -{ - "__requires": [ - { - "type": "panel", - "id": "ae3e-plotly-panel", - "name": "Plotly panel", - "version": "0.3.3" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.3" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "magnesium-wordcloud-panel", - "name": "Word cloud", - "version": "1.2.4" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 2, - "x": 0, - "y": 0 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT COUNT(DISTINCT user_hash)\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo;", - "refId": "A" - } - ], - "title": "Unique Users", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "Requests", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "query": "SELECT\n time AS _time,\n COUNT(*) AS _value\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND _measurement = 'analytics'\n AND _field = 'number_request_messages'\nGROUP BY time_bucket(INTERVAL '$__interval', time)\nORDER BY _time", - "refId": "A" - } - ], - "title": "System Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n project_id,\n deployment,\n model,\n COUNT(DISTINCT user_hash) AS user_count,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id, deployment, model", - "refId": "A" - } - ], - "title": "Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 6, - "deployment": 1, - "model": 2, - "money": 7, - "project_id": 0, - "prompt_tokens": 5, - "request_count": 4, - "user_count": 3 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "deployment": "Deployment", - "model": "Model", - "money": "Money", - "number_request_messages": "", - "project_id": "Project", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count", - "user_count": "Users" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n project_id,\n COUNT(*) AS request_count,\n SUM(COALESCE(prompt_tokens, 0)) AS prompt_tokens,\n SUM(COALESCE(completion_tokens, 0)) AS completion_tokens,\n SUM(COALESCE(price, 0.0)) AS money\nFROM analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY project_id", - "refId": "A" - } - ], - "title": "Project Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 3, - "money": 4, - "project_id": 0, - "prompt_tokens": 2, - "request_count": 1 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "model": "", - "money": "Money", - "number_request_messages": "", - "project_id": "Project", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "money" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "NaN": { - "index": 0, - "text": "Not configured" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 8, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Money" - } - ] - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "${datasource}" - }, - "hide": false, - "query": "SELECT\n \"model\",\n \"deployment\",\n COUNT(*) AS request_count,\n SUM(COALESCE(\"prompt_tokens\", 0)) AS prompt_tokens,\n SUM(COALESCE(\"completion_tokens\", 0)) AS completion_tokens,\n SUM(COALESCE(\"price\", 0.0)) AS money\nFROM \"default\".\"analytics\"\nWHERE \"time\" >= $__timeFrom AND \"time\" < $__timeTo\nGROUP BY\n \"model\",\n \"deployment\"", - "refId": "A" - } - ], - "title": "Deployment/Model Stats Table", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "_time": true - }, - "indexByName": { - "completion_tokens": 4, - "deployment": 0, - "model": 1, - "money": 5, - "prompt_tokens": 3, - "request_count": 2 - }, - "renameByName": { - "completion_tokens": "Completion tokens", - "deployment": "Deployment", - "model": "Model", - "money": "Money", - "number_request_messages": "", - "project_id": "", - "prompt_tokens": "Prompt tokens", - "request_count": "Request count" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Money" - } - ] - } - } - ], - "type": "table" - } - ], - "refresh": "1m", - "schemaVersion": 38, - "style": "dark", - "tags": [ - "DIAL", - "ai-dial-analytics-realtime" - ], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, - "description": "InfluxDB datasource", - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [], - "query": "influxdb3", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "DIAL Analytics", - "uid": "435858b2-4106-4d11-b6e8-9b0ff107b471", - "version": 1, - "weekStart": "" -} From 0300613205da175b282607df13e7935c7a1267f0 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 15:56:41 +0000 Subject: [PATCH 09/56] fix: minor fix --- dashboards/customized/dial_analytics.json | 2 +- dashboards/customized/dial_analytics_raw_data.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/dial_analytics.json b/dashboards/customized/dial_analytics.json index 97c7e52..6a22771 100644 --- a/dashboards/customized/dial_analytics.json +++ b/dashboards/customized/dial_analytics.json @@ -989,7 +989,7 @@ "type": "influxdb", "uid": "${datasource}" }, - "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))\n |> keep(columns: [\"name\", \"id\", \"retentionPeriod\"])\n |> sort(columns: [\"name\"])", + "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))", "description": "A InfluxDB bucket name", "includeAll": false, "name": "INFLUX_BUCKET", diff --git a/dashboards/customized/dial_analytics_raw_data.json b/dashboards/customized/dial_analytics_raw_data.json index 6840052..69c6217 100644 --- a/dashboards/customized/dial_analytics_raw_data.json +++ b/dashboards/customized/dial_analytics_raw_data.json @@ -209,7 +209,7 @@ "type": "influxdb", "uid": "${datasource}" }, - "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))\n |> keep(columns: [\"name\", \"id\", \"retentionPeriod\"])\n |> sort(columns: [\"name\"])", + "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))", "description": "A InfluxDB bucket name", "includeAll": false, "name": "INFLUX_BUCKET", From 2f15665c1a8371f8295a8edea7ee1a7bc27de015 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 16:04:20 +0000 Subject: [PATCH 10/56] feat: bump DIAL Analytics MCP dashboard --- dashboards/customized/dial_analytics_mcp.json | 196 ++++++++++-------- 1 file changed, 115 insertions(+), 81 deletions(-) diff --git a/dashboards/customized/dial_analytics_mcp.json b/dashboards/customized/dial_analytics_mcp.json index ee08f01..af419d5 100644 --- a/dashboards/customized/dial_analytics_mcp.json +++ b/dashboards/customized/dial_analytics_mcp.json @@ -1,4 +1,47 @@ { + "__inputs": [ + { + "name": "DS_INFLUXDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.0.0" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -36,8 +79,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -65,6 +107,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -81,7 +124,7 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> count()\n |> group()\n |> sum()", "refId": "A" @@ -105,8 +148,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -134,6 +176,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -150,7 +193,7 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"user_hash\")\n |> keep(columns: [\"_value\"])\n |> distinct(column: \"_value\")\n |> count()", "refId": "A" @@ -174,8 +217,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -203,6 +245,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -219,7 +262,7 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r.mcp_method == \"tools/call\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> count()\n |> group()\n |> sum()", "refId": "A" @@ -245,13 +288,14 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { + "legend": false, "tooltip": false, - "viz": false, - "legend": false + "viz": false }, "insertNulls": false, "lineInterpolation": "linear", @@ -275,8 +319,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -299,6 +342,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -308,7 +352,7 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\" and r._field == \"upstream\")\n |> map(fn: (r) => ({\n r with\n project_id: if exists r.project_id then r.project_id else \"undefined\"\n }))\n |> group(columns: [\"project_id\"])\n |> aggregateWindow(every: v.windowPeriod, fn: count)", "refId": "A" @@ -334,13 +378,14 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { + "legend": false, "tooltip": false, - "viz": false, - "legend": false + "viz": false }, "insertNulls": false, "lineInterpolation": "linear", @@ -364,8 +409,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -388,6 +432,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -397,7 +442,7 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\" and r._field == \"upstream\")\n |> group(columns: [\"mcp_method\"])\n |> aggregateWindow(every: v.windowPeriod, fn: count)", "refId": "A" @@ -425,8 +470,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -437,24 +481,24 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 12 }, "id": 7, "options": { - "showHeader": true, "cellHeight": "sm", "footer": { - "show": false, + "countRows": false, + "fields": "", "reducer": [ "sum" ], - "countRows": false, - "fields": "" + "show": false }, + "showHeader": true, "sortBy": [ { - "displayName": "_value", - "desc": true + "desc": true, + "displayName": "_value" } ] }, @@ -463,30 +507,30 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> group(columns: [\"deployment\"])\n |> count()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)\n |> keep(columns: [\"deployment\", \"_value\"])", "refId": "A" } ], "title": "Calls by Deployment", - "type": "table", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": { - "deployment": 0, - "_value": 1 + "_value": 1, + "deployment": 0 }, "renameByName": { - "deployment": "Deployment", - "_value": "Calls" + "_value": "Calls", + "deployment": "Deployment" } } } - ] + ], + "type": "table" }, { "datasource": { @@ -507,8 +551,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -519,24 +562,24 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 12 }, "id": 9, "options": { - "showHeader": true, "cellHeight": "sm", "footer": { - "show": false, + "countRows": false, + "fields": "", "reducer": [ "sum" ], - "countRows": false, - "fields": "" + "show": false }, + "showHeader": true, "sortBy": [ { - "displayName": "_value", - "desc": true + "desc": true, + "displayName": "_value" } ] }, @@ -545,30 +588,30 @@ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"mcp_tool_call_name\")\n |> filter(fn: (r) => r._value != \"undefined\" and r._value != \"\")\n |> map(fn: (r) => ({ r with tool_name: r._value }))\n |> group(columns: [\"tool_name\"])\n |> count()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)", "refId": "A" } ], "title": "Most Called Tools", - "type": "table", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": { - "tool_name": 0, - "_value": 1 + "_value": 1, + "tool_name": 0 }, "renameByName": { - "tool_name": "Tool Name", - "_value": "Calls" + "_value": "Calls", + "tool_name": "Tool Name" } } } - ] + ], + "type": "table" }, { "datasource": { @@ -589,8 +632,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -638,40 +680,39 @@ "h": 11, "w": 24, "x": 0, - "y": 28 + "y": 20 }, "id": 10, "options": { - "showHeader": true, "cellHeight": "sm", "footer": { - "show": false, - "reducer": [], "countRows": false, + "enablePagination": true, "fields": "", - "enablePagination": true + "reducer": [], + "show": false }, + "frameIndex": 0, + "showHeader": true, "sortBy": [ { - "displayName": "_time", - "desc": true + "desc": true, + "displayName": "_time" } - ], - "frameIndex": 0 + ] }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "influxdb", - "uid": "${datasource}" + "uid": "${DS_INFLUXDB}" }, "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _time: r._time,\n project_id: if exists r.project_id then r.project_id else \"undefined\",\n deployment: r.deployment,\n mcp_method: r.mcp_method,\n mcp_tool_call_name: if exists r.mcp_tool_call_name then r.mcp_tool_call_name else \"undefined\",\n trace_id: if exists r.trace_id then r.trace_id else \"\"\n }))\n |> group()\n |> sort(columns: [\"_time\"], desc: true)\n |> limit(n: 100)", "refId": "A" } ], "title": "Recent Activity Log", - "type": "table", "transformations": [ { "id": "organize", @@ -679,27 +720,28 @@ "excludeByName": {}, "indexByName": { "_time": 0, - "project_id": 1, "deployment": 2, "mcp_method": 3, "mcp_tool_call_name": 4, + "project_id": 1, "trace_id": 5 }, "renameByName": { "_time": "Time", - "project_id": "Project", "deployment": "Deployment", "mcp_method": "Method", "mcp_tool_call_name": "Tool Name", + "project_id": "Project", "trace_id": "Trace ID" } } } - ] + ], + "type": "table" } ], "refresh": "", - "schemaVersion": 39, + "schemaVersion": 41, "tags": [ "DIAL", "ai-dial-analytics-realtime", @@ -708,15 +750,9 @@ "templating": { "list": [ { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, + "current": {}, "includeAll": false, "label": "Data Source", - "multi": false, "name": "datasource", "options": [], "query": "influxdb", @@ -725,10 +761,7 @@ "type": "datasource" }, { - "current": { - "text": "default", - "value": "default" - }, + "current": {}, "datasource": { "type": "influxdb", "uid": "${datasource}" @@ -754,5 +787,6 @@ "timezone": "", "title": "DIAL Analytics MCP", "uid": "c935f785-18d2-4754-b95f-354d0cdd149f", - "version": 1 -} \ No newline at end of file + "version": 1, + "weekStart": "" +} From 0effa55a133e6f3cb39843fa4721bb6b4c48513f Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 16:17:28 +0000 Subject: [PATCH 11/56] feat: update buckets in MCP --- dashboards/customized/dial_analytics_mcp.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/dial_analytics_mcp.json b/dashboards/customized/dial_analytics_mcp.json index af419d5..b47c8e9 100644 --- a/dashboards/customized/dial_analytics_mcp.json +++ b/dashboards/customized/dial_analytics_mcp.json @@ -766,7 +766,7 @@ "type": "influxdb", "uid": "${datasource}" }, - "definition": "buckets()", + "definition": "import \"strings\"\n\nbuckets()\n |> filter(fn: (r) => not strings.hasPrefix(v: r.name, prefix: \"_\"))", "description": "InfluxDB bucket name", "includeAll": false, "label": "Bucket", @@ -787,6 +787,6 @@ "timezone": "", "title": "DIAL Analytics MCP", "uid": "c935f785-18d2-4754-b95f-354d0cdd149f", - "version": 1, + "version": 2, "weekStart": "" } From bd6803d12e0b05da09fa351f74a8e03b773de9bf Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 18:02:37 +0000 Subject: [PATCH 12/56] feat: migrate MCP dashboard to InfluxDB 3 --- dashboards/customized/dial_analytics_mcp.json | 186 +++++++++++++++++- 1 file changed, 177 insertions(+), 9 deletions(-) diff --git a/dashboards/customized/dial_analytics_mcp.json b/dashboards/customized/dial_analytics_mcp.json index b47c8e9..fc62e76 100644 --- a/dashboards/customized/dial_analytics_mcp.json +++ b/dashboards/customized/dial_analytics_mcp.json @@ -122,12 +122,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> count()\n |> group()\n |> sum()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT COUNT(*)\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Total MCP Calls", @@ -191,12 +212,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"user_hash\")\n |> keep(columns: [\"_value\"])\n |> distinct(column: \"_value\")\n |> count()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_hash)\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Unique Users", @@ -260,12 +302,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r.mcp_method == \"tools/call\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> count()\n |> group()\n |> sum()", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT COUNT(*)\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND mcp_method = 'tools/call'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Tool Calls", @@ -350,12 +413,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "time_series", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\" and r._field == \"upstream\")\n |> map(fn: (r) => ({\n r with\n project_id: if exists r.project_id then r.project_id else \"undefined\"\n }))\n |> group(columns: [\"project_id\"])\n |> aggregateWindow(every: v.windowPeriod, fn: count)", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(project_id, 'undefined') AS project_id,\n date_bin_gapfill($__interval, time) AS time,\n COUNT(*) AS requests\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY 1, 2\nORDER BY 2 ASC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Requests per DIAL Project", @@ -440,12 +524,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "time_series", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\" and r._field == \"upstream\")\n |> group(columns: [\"mcp_method\"])\n |> aggregateWindow(every: v.windowPeriod, fn: count)", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n mcp_method,\n date_bin_gapfill($__interval, time) AS time,\n COUNT(*) AS method\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY 1, 2\nORDER BY 2 ASC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Requests per MCP Method", @@ -505,12 +610,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"upstream\")\n |> group(columns: [\"deployment\"])\n |> count()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)\n |> keep(columns: [\"deployment\", \"_value\"])", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT deployment, COUNT(*) AS count\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\nGROUP BY deployment", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Calls by Deployment", @@ -586,12 +712,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> filter(fn: (r) => r._field == \"mcp_tool_call_name\")\n |> filter(fn: (r) => r._value != \"undefined\" and r._value != \"\")\n |> map(fn: (r) => ({ r with tool_name: r._value }))\n |> group(columns: [\"tool_name\"])\n |> count()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT mcp_tool_call_name, COUNT(*) AS count\nFROM mcp_analytics\nWHERE time >= $__timeFrom AND time < $__timeTo\n AND mcp_method == 'tools/call'\nGROUP BY mcp_tool_call_name\nORDER BY 2 DESC, 1 DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Most Called Tools", @@ -704,12 +851,33 @@ "pluginVersion": "12.0.0", "targets": [ { + "dataset": "iox", "datasource": { "type": "influxdb", "uid": "${DS_INFLUXDB}" }, + "editorMode": "code", + "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _time: r._time,\n project_id: if exists r.project_id then r.project_id else \"undefined\",\n deployment: r.deployment,\n mcp_method: r.mcp_method,\n mcp_tool_call_name: if exists r.mcp_tool_call_name then r.mcp_tool_call_name else \"undefined\",\n trace_id: if exists r.trace_id then r.trace_id else \"\"\n }))\n |> group()\n |> sort(columns: [\"_time\"], desc: true)\n |> limit(n: 100)", - "refId": "A" + "rawQuery": true, + "rawSql": "SELECT\n time,\n COALESCE(project_id, 'undefined') AS project_id,\n deployment,\n mcp_method,\n mcp_tool_call_name,\n trace_id\nFROM mcp_analytics\nORDER BY time DESC\nLIMIT 100", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ] + } } ], "title": "Recent Activity Log", @@ -787,6 +955,6 @@ "timezone": "", "title": "DIAL Analytics MCP", "uid": "c935f785-18d2-4754-b95f-354d0cdd149f", - "version": 2, + "version": 11, "weekStart": "" } From 96a973183760f199145ceaf7258595144409cd81 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 3 Mar 2026 18:08:45 +0000 Subject: [PATCH 13/56] feat: fix MCP dashboard --- dashboards/customized/dial_analytics_mcp.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/dial_analytics_mcp.json b/dashboards/customized/dial_analytics_mcp.json index fc62e76..9eb399e 100644 --- a/dashboards/customized/dial_analytics_mcp.json +++ b/dashboards/customized/dial_analytics_mcp.json @@ -860,7 +860,7 @@ "format": "table", "query": "bucket = \"${INFLUX_BUCKET}\"\nfrom(bucket: bucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"mcp_analytics\")\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _time: r._time,\n project_id: if exists r.project_id then r.project_id else \"undefined\",\n deployment: r.deployment,\n mcp_method: r.mcp_method,\n mcp_tool_call_name: if exists r.mcp_tool_call_name then r.mcp_tool_call_name else \"undefined\",\n trace_id: if exists r.trace_id then r.trace_id else \"\"\n }))\n |> group()\n |> sort(columns: [\"_time\"], desc: true)\n |> limit(n: 100)", "rawQuery": true, - "rawSql": "SELECT\n time,\n COALESCE(project_id, 'undefined') AS project_id,\n deployment,\n mcp_method,\n mcp_tool_call_name,\n trace_id\nFROM mcp_analytics\nORDER BY time DESC\nLIMIT 100", + "rawSql": "SELECT\n time as _time,\n COALESCE(project_id, 'undefined') AS project_id,\n deployment,\n mcp_method,\n mcp_tool_call_name,\n trace_id\nFROM mcp_analytics\nORDER BY time DESC\nLIMIT 100", "refId": "A", "sql": { "columns": [ @@ -955,6 +955,6 @@ "timezone": "", "title": "DIAL Analytics MCP", "uid": "c935f785-18d2-4754-b95f-354d0cdd149f", - "version": 11, + "version": 12, "weekStart": "" } From 1262f3704a02dae78e82d3fea90bd040cca22a44 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 11:41:39 +0000 Subject: [PATCH 14/56] feat: simplify plugin --- .../customized/influxdb_v3/plugin/__init__.py | 18 +-- .../influxdb_v3/plugin/rollup_6h.py | 107 +++++++++------ .../influxdb_v3/plugin/rollup_monthly.py | 36 ++--- .../customized/influxdb_v3/plugin/utils.py | 124 +++++++----------- 4 files changed, 141 insertions(+), 144 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index e4e037d..7ecb3ec 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -50,34 +50,34 @@ } """ -from __future__ import annotations - import uuid +from datetime import datetime, timezone from .rollup_6h import run_6h from .rollup_monthly import run_monthly -from .utils import parse_call_time -def process_scheduled_call(influxdb3_local, call_time, args=None): +def process_scheduled_call(influxdb3_local, call_time: datetime, args=None): """ InfluxDB 3 scheduled plugin entrypoint. """ task_id = str(uuid.uuid4()) args = args or {} - ct = parse_call_time(call_time) + call_time = call_time.replace(tzinfo=timezone.utc) mode = str(args.get("mode", "6h")).strip().lower() influxdb3_local.info( - f"[{task_id}] scheduled call mode={mode} call_time={ct.isoformat()} args={args}" + f"[{task_id}] scheduled call mode={mode} call_time={call_time.isoformat()} args={args}" ) try: - if mode in ("month", "monthly"): - run_monthly(influxdb3_local, ct, args, task_id=task_id) + if mode == "monthly": + run_monthly(influxdb3_local, call_time, args, task_id=task_id) + elif mode == "6h": + run_6h(influxdb3_local, call_time, args, task_id=task_id) else: - run_6h(influxdb3_local, ct, args, task_id=task_id) + raise ValueError(f"unsupported mode: {mode}") influxdb3_local.info(f"[{task_id}] completed OK") except Exception as e: diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_6h.py b/dashboards/customized/influxdb_v3/plugin/rollup_6h.py index 1106590..0dbf7a9 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_6h.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_6h.py @@ -4,8 +4,9 @@ from typing import Any, Dict from .utils import ( + parse_iso_datetime, query_rows, - rfc3339, + to_iso, token_class_case_sql, window_from_args_or_call_time, write_points, @@ -18,30 +19,46 @@ def run_6h( raw_table = str(args.get("raw_table", "analytics")) agg_db = str(args.get("agg_database", "analytics_agg")) + start_arg: str | None = args.get("start_time") + end_arg: str | None = args.get("end_time") + start_time: datetime | None = ( + parse_iso_datetime("start_time", start_arg) if start_arg else None + ) + end_time: datetime | None = ( + parse_iso_datetime("end_time", end_arg) if end_arg else None + ) + + window_hours = int(args.get("window_hours") or 6) + offset_minutes = int(args.get("offset_minutes") or 2) + start, end = window_from_args_or_call_time( - call_time, args, window_hours_default=6, offset_minutes_default=2 + call_time, + start_time, + end_time, + window_hours=window_hours, + offset_minutes=offset_minutes, ) - start_s = rfc3339(start) - end_s = rfc3339(end) + + start_s, end_s = to_iso(start), to_iso(end) influxdb3_local.info(f"[{task_id}] 6h rollup window: {start_s} .. {end_s}") # 1) default_agg_stats stats_sql = f""" SELECT - '{start_s}' AS time, - deployment, - model, - project_id, - parent_deployment, - language, - SUM(prompt_tokens) AS prompt_tokens, - SUM(completion_tokens) AS completion_tokens, - SUM(price) AS price, - SUM(number_request_messages) AS number_request_messages, - SUM(deployment_price) AS deployment_price, - COUNT(*) AS request_count, - COUNT(DISTINCT user_hash) AS unique_user_count + '{start_s}' AS time, + deployment, + model, + project_id, + parent_deployment, + language, + SUM(prompt_tokens) AS prompt_tokens, + SUM(completion_tokens) AS completion_tokens, + SUM(price) AS price, + SUM(number_request_messages) AS number_request_messages, + SUM(deployment_price) AS deployment_price, + COUNT(*) AS request_count, + COUNT(DISTINCT user_hash) AS unique_user_count FROM {raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY deployment, model, project_id, parent_deployment, language @@ -72,18 +89,18 @@ def run_6h( task_id=task_id, ) - # 2) default_agg_topic + default_agg_topic_2 (same content, two tables like your current setup) + # 2) default_agg_topic + default_agg_topic_2 (same content, but different retention/purpose - FIXME????) topic_sql = f""" SELECT - '{start_s}' AS time, - title, - topic, - model, - COUNT(*) AS topic_count, - SUM(number_request_messages) AS number_request_messages, - SUM(price) AS price, - SUM(prompt_tokens) AS prompt_tokens, - SUM(completion_tokens) AS completion_tokens + '{start_s}' AS time, + title, + topic, + model, + COUNT(*) AS topic_count, + SUM(number_request_messages) AS number_request_messages, + SUM(price) AS price, + SUM(prompt_tokens) AS prompt_tokens, + SUM(completion_tokens) AS completion_tokens FROM {raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY title, topic, model @@ -112,10 +129,14 @@ def run_6h( token_table = str(args.get("token_class_table", "default_agg_topic")) token_sql = f""" SELECT - '{start_s}' AS time, - CASE WHEN user_hash = 'undefined' THEN 'project' ELSE 'user' END AS user_type, - {token_class_case_sql()} AS prompt_token_class, - COUNT(*) AS request_count + '{start_s}' AS time, + CASE + WHEN user_hash = 'undefined' + THEN 'project' + ELSE 'user' + END AS user_type, + {token_class_case_sql()} AS prompt_token_class, + COUNT(*) AS request_count FROM {raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY user_type, prompt_token_class @@ -135,15 +156,15 @@ def run_6h( # 4) default_agg_kpi kpi_sql = f""" SELECT - '{start_s}' AS time, - user_hash, - project_id, - parent_deployment, - title, - COUNT(*) AS request_count, - SUM(completion_tokens) AS completion_tokens, - SUM(prompt_tokens) AS prompt_tokens, - SUM(price) AS cost + '{start_s}' AS time, + user_hash, + project_id, + parent_deployment, + title, + COUNT(*) AS request_count, + SUM(completion_tokens) AS completion_tokens, + SUM(prompt_tokens) AS prompt_tokens, + SUM(price) AS cost FROM {raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY user_hash, project_id, parent_deployment, title @@ -168,9 +189,9 @@ def run_6h( # 5) default_agg_chatid chat_sql = f""" SELECT - '{start_s}' AS time, - chat_id, - COUNT(*) AS request_count + '{start_s}' AS time, + chat_id, + COUNT(*) AS request_count FROM {raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY chat_id diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index cabdb1d..dc62c97 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict -from .utils import query_rows, rfc3339, write_points +from .utils import query_rows, to_iso, write_points def _month_start(dt: datetime) -> datetime: @@ -20,8 +20,8 @@ def run_monthly( this_month = _month_start(call_time) prev_month = _month_start(this_month - timedelta(days=32)) - start_s = rfc3339(prev_month) - end_s = rfc3339(this_month) + start_s = to_iso(prev_month) + end_s = to_iso(this_month) stamp_s = start_s # stamp results at first day of prev month influxdb3_local.info( @@ -31,24 +31,24 @@ def run_monthly( # API-level (project_id) rollups from default_agg_stats api_sql = f""" SELECT - '{stamp_s}' AS time, - SUM(price) AS total_cost_per_api, - AVG(price) AS avg_cost_per_api, - COUNT(DISTINCT project_id) AS active_apis, - SUM(request_count) AS total_rc_per_api, - AVG(request_count) AS avg_rc_per_api + '{stamp_s}' AS time, + SUM(price) AS total_cost_per_api, + AVG(price) AS avg_cost_per_api, + COUNT(DISTINCT project_id) AS active_apis, + SUM(request_count) AS total_rc_per_api, + AVG(request_count) AS avg_rc_per_api FROM default_agg_stats WHERE time >= '{start_s}' AND time < '{end_s}' - AND project_id IS NOT NULL AND project_id <> '' + AND project_id IS NOT NULL AND project_id <> '' """ api_rows = query_rows(influxdb3_local, api_sql) # Model-level rollups from default_agg_stats model_sql = f""" SELECT - '{stamp_s}' AS time, - SUM(price) AS total_cost_per_model, - AVG(price) AS avg_cost_per_model + '{stamp_s}' AS time, + SUM(price) AS total_cost_per_model, + AVG(price) AS avg_cost_per_model FROM default_agg_stats WHERE time >= '{start_s}' AND time < '{end_s}' """ @@ -57,13 +57,13 @@ def run_monthly( # User-level rollups from default_agg_kpi (exclude undefined like your Flux) user_sql = f""" SELECT - '{stamp_s}' AS time, - SUM(cost) AS total_user_cost, - AVG(cost) AS avg_cost_per_user, - COUNT(DISTINCT user_hash) AS unique_users + '{stamp_s}' AS time, + SUM(cost) AS total_user_cost, + AVG(cost) AS avg_cost_per_user, + COUNT(DISTINCT user_hash) AS unique_users FROM default_agg_kpi WHERE time >= '{start_s}' AND time < '{end_s}' - AND user_hash IS NOT NULL AND user_hash <> 'undefined' + AND user_hash IS NOT NULL AND user_hash <> 'undefined' """ user_rows = query_rows(influxdb3_local, user_sql) diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index 12e9372..ec229d2 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -5,6 +5,7 @@ if TYPE_CHECKING: # LineBuilder is available in the processing runtime (as used by official plugins). + # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data class LineBuilder: def __init__(self, measurement: str) -> None: ... def time_ns(self, value: int) -> None: ... @@ -15,71 +16,51 @@ def float64_field(self, key: str, value: float) -> None: ... def bool_field(self, key: str, value: bool) -> None: ... -def parse_call_time(call_time: Any) -> datetime: - """ - Best-effort conversion to a timezone-aware UTC datetime. - """ - if isinstance(call_time, datetime): - return ( - call_time - if call_time.tzinfo - else call_time.replace(tzinfo=timezone.utc) +def parse_iso_datetime(name: str, value: str) -> datetime: + try: + dt: datetime = datetime.fromisoformat(value) + except ValueError: + raise ValueError( + f"Invalid ISO 8601 datetime from the {name!r} column: {value!r}." ) - - if isinstance(call_time, str): - s = call_time.strip() - # Accept ...Z - if s.endswith("Z"): - s = s[:-1] + "+00:00" - return datetime.fromisoformat(s).astimezone(timezone.utc) - - return datetime.now(tz=timezone.utc) - - -def parse_rfc3339(s: str) -> datetime: - s = s.strip() - if s.endswith("Z"): - s = s[:-1] + "+00:00" - return datetime.fromisoformat(s).astimezone(timezone.utc) + if dt.tzinfo is None: + raise ValueError( + f"Date from the {name!r} column must include timezone info (e.g., '+00:00'): {value!r}" + ) + return dt.astimezone(timezone.utc) -def rfc3339(dt: datetime) -> str: - return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") +def to_iso(dt: datetime) -> str: + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def ns(dt: datetime) -> int: +def _ns(dt: datetime) -> int: return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) def window_from_args_or_call_time( call_time: datetime, - args: Dict[str, Any], + end_time: datetime | None, + start_time: datetime | None, *, - window_hours_default: int = 6, - offset_minutes_default: int = 2, + window_hours: int, + offset_minutes: int, ) -> Tuple[datetime, datetime]: """ If args contains start_time/end_time => backfill window. Else uses call_time to compute [end - window, end) with optional offset. """ - start_arg = args.get("start_time") - end_arg = args.get("end_time") - if start_arg and end_arg: - start = parse_rfc3339(str(start_arg)) - end = parse_rfc3339(str(end_arg)) - if end <= start: + if start_time and end_time: + if end_time <= start_time: raise ValueError( - f"end_time must be > start_time (got {start_arg} .. {end_arg})" + f"end_time must be > start_time (got {start_time} .. {end_time})" ) - return start, end - - window_hours = int(args.get("window_hours", window_hours_default)) - offset_minutes = int(args.get("offset_minutes", offset_minutes_default)) + return start_time, end_time - end = call_time - timedelta(minutes=offset_minutes) - start = end - timedelta(hours=window_hours) - return start, end + end_time = call_time - timedelta(minutes=offset_minutes) + start_time = end_time - timedelta(hours=window_hours) + return start_time, end_time def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: @@ -114,32 +95,30 @@ def write_points( if isinstance(t, int): b.time_ns(t) elif isinstance(t, datetime): - b.time_ns(ns(t)) + b.time_ns(_ns(t)) elif isinstance(t, str): - b.time_ns(ns(parse_rfc3339(t))) + b.time_ns(_ns(parse_iso_datetime(f"{time_col!r} column", t))) else: - # fallback - b.time_ns(ns(datetime.now(tz=timezone.utc))) + raise ValueError( + f"Unexpected time value in the column '{time_col!r}': {t} of type {type(t)}" + ) for k in tag_cols: - v = r.get(k) - if v is None: - continue - # tags are always strings - b.tag(k, str(v)) + if (v := r.get(k)) is not None: + b.tag(k, str(v)) for k in field_cols: - v = r.get(k) - if v is None: - continue - if isinstance(v, bool): - b.bool_field(k, v) - elif isinstance(v, int): - b.int64_field(k, v) - elif isinstance(v, float): - b.float64_field(k, v) - else: - b.string_field(k, str(v)) + if (v := r.get(k)) is not None: + if isinstance(v, bool): + b.bool_field(k, v) + elif isinstance(v, int): + b.int64_field(k, v) + elif isinstance(v, float): + b.float64_field(k, v) + else: + raise ValueError( + f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" + ) influxdb3_local.write_to_db(db_name, b) written += 1 @@ -151,16 +130,13 @@ def write_points( def token_class_case_sql() -> str: - """ - Same thresholds/order as your Flux task. - """ return """ CASE - WHEN prompt_tokens >= 50000 THEN 'class_1' - WHEN prompt_tokens > 10000 THEN 'class_2' - WHEN prompt_tokens > 5000 THEN 'class_3' - WHEN prompt_tokens > 1000 THEN 'class_4' - WHEN prompt_tokens > 100 THEN 'class_5' - ELSE 'class_6' + WHEN prompt_tokens >= 50000 THEN 'class_1' + WHEN prompt_tokens > 10000 THEN 'class_2' + WHEN prompt_tokens > 5000 THEN 'class_3' + WHEN prompt_tokens > 1000 THEN 'class_4' + WHEN prompt_tokens > 100 THEN 'class_5' + ELSE 'class_6' END """.strip() From 5b7a8e0fd02fbbd02ec025846897360990a17031 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 12:04:57 +0000 Subject: [PATCH 15/56] feat: introduce Config class for plugin --- .../customized/influxdb_v3/plugin/__init__.py | 31 +++++----- .../customized/influxdb_v3/plugin/config.py | 58 +++++++++++++++++++ .../plugin/{rollup_6h.py => rollup_hourly.py} | 54 ++++++----------- .../influxdb_v3/plugin/rollup_monthly.py | 15 ++--- .../customized/influxdb_v3/plugin/utils.py | 3 +- 5 files changed, 103 insertions(+), 58 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/config.py rename dashboards/customized/influxdb_v3/plugin/{rollup_6h.py => rollup_hourly.py} (79%) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 7ecb3ec..3a3ce32 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -6,8 +6,8 @@ "scheduled_args_config": [ { "name": "mode", - "example": "6h", - "description": "Which rollup to run: '6h' (default) or 'monthly'.", + "example": "hourly", + "description": "Which rollup to run: 'hourly' (default) or 'monthly'.", "required": false }, { @@ -25,25 +25,25 @@ { "name": "window_hours", "example": "6", - "description": "For mode=6h: window size in hours.", + "description": "For mode=hourly: window size in hours.", "required": false }, { "name": "offset_minutes", "example": "2", - "description": "For mode=6h: shift end of window backward to avoid late data.", + "description": "For mode=hourly: shift end of window backward to avoid late data.", "required": false }, { "name": "start_time", "example": "2026-01-01T00:00:00Z", - "description": "Optional backfill start (RFC3339). If set with end_time, overrides call_time-derived window.", + "description": "Optional backfill start (ISO 8601). If set with end_time, overrides call_time-derived window.", "required": false }, { "name": "end_time", "example": "2026-01-01T06:00:00Z", - "description": "Optional backfill end (RFC3339). If set with start_time, overrides call_time-derived window.", + "description": "Optional backfill end (ISO 8601)). If set with start_time, overrides call_time-derived window.", "required": false } ] @@ -52,8 +52,10 @@ import uuid from datetime import datetime, timezone +from typing import assert_never -from .rollup_6h import run_6h +from .config import Config, Mode +from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly @@ -62,22 +64,21 @@ def process_scheduled_call(influxdb3_local, call_time: datetime, args=None): InfluxDB 3 scheduled plugin entrypoint. """ task_id = str(uuid.uuid4()) - args = args or {} + config = Config.parse(args) call_time = call_time.replace(tzinfo=timezone.utc) - mode = str(args.get("mode", "6h")).strip().lower() influxdb3_local.info( - f"[{task_id}] scheduled call mode={mode} call_time={call_time.isoformat()} args={args}" + f"[{task_id}] scheduled call mode={config.mode} call_time={call_time.isoformat()} args={args}" ) try: - if mode == "monthly": - run_monthly(influxdb3_local, call_time, args, task_id=task_id) - elif mode == "6h": - run_6h(influxdb3_local, call_time, args, task_id=task_id) + if config.mode == Mode.MONTHLY: + run_monthly(influxdb3_local, call_time, config, task_id=task_id) + elif config.mode == Mode.HOURLY: + run_hourly(influxdb3_local, call_time, config, task_id=task_id) else: - raise ValueError(f"unsupported mode: {mode}") + assert_never(config.mode) influxdb3_local.info(f"[{task_id}] completed OK") except Exception as e: diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py new file mode 100644 index 0000000..b6b6a76 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -0,0 +1,58 @@ +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Dict + +from .utils import parse_iso_datetime + + +class Mode(Enum): + HOURLY = "hourly" + MONTHLY = "monthly" + + +@dataclass +class Config: + mode: Mode + agg_database: str + raw_table: str + + window_hours: int + offset_minutes: int + + start_time: datetime | None + end_time: datetime | None + + @classmethod + def parse(cls, d: Dict[str, str] | None) -> "Config": + d = d or {} + + mode_s = str(d.get("mode", "hourly")).strip().lower() + if mode_s == "hourly": + mode = Mode.HOURLY + elif mode_s == "monthly": + mode = Mode.MONTHLY + else: + raise ValueError(f"unsupported mode: {mode_s}") + + agg_database = d.get("agg_database") + raw_table = d.get("raw_table") + + start_arg = d.get("start_time") + end_arg = d.get("end_time") + start_time: datetime | None = ( + parse_iso_datetime("start_time", start_arg) if start_arg else None + ) + end_time: datetime | None = ( + parse_iso_datetime("end_time", end_arg) if end_arg else None + ) + + return cls( + mode=mode, + agg_database=agg_database or "analytics_agg", + raw_table=raw_table or "analytics", + start_time=start_time, + end_time=end_time, + window_hours=int(d.get("window_hours") or 6), + offset_minutes=int(d.get("offset_minutes") or 2), + ) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_6h.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py similarity index 79% rename from dashboards/customized/influxdb_v3/plugin/rollup_6h.py rename to dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 0dbf7a9..d6457da 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_6h.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -1,10 +1,9 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict +from .config import Config from .utils import ( - parse_iso_datetime, query_rows, to_iso, token_class_case_sql, @@ -13,30 +12,16 @@ ) -def run_6h( - influxdb3_local, call_time: datetime, args: Dict[str, Any], *, task_id: str +def run_hourly( + influxdb3_local, call_time: datetime, config: Config, *, task_id: str ) -> None: - raw_table = str(args.get("raw_table", "analytics")) - agg_db = str(args.get("agg_database", "analytics_agg")) - - start_arg: str | None = args.get("start_time") - end_arg: str | None = args.get("end_time") - start_time: datetime | None = ( - parse_iso_datetime("start_time", start_arg) if start_arg else None - ) - end_time: datetime | None = ( - parse_iso_datetime("end_time", end_arg) if end_arg else None - ) - - window_hours = int(args.get("window_hours") or 6) - offset_minutes = int(args.get("offset_minutes") or 2) start, end = window_from_args_or_call_time( call_time, - start_time, - end_time, - window_hours=window_hours, - offset_minutes=offset_minutes, + config.start_time, + config.end_time, + window_hours=config.window_hours, + offset_minutes=config.offset_minutes, ) start_s, end_s = to_iso(start), to_iso(end) @@ -59,14 +44,14 @@ def run_6h( SUM(deployment_price) AS deployment_price, COUNT(*) AS request_count, COUNT(DISTINCT user_hash) AS unique_user_count -FROM {raw_table} +FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY deployment, model, project_id, parent_deployment, language """ stats_rows = query_rows(influxdb3_local, stats_sql) write_points( influxdb3_local, - db_name=agg_db, + db_name=config.agg_database, table_name="default_agg_stats", rows=stats_rows, time_col="time", @@ -101,7 +86,7 @@ def run_6h( SUM(price) AS price, SUM(prompt_tokens) AS prompt_tokens, SUM(completion_tokens) AS completion_tokens -FROM {raw_table} +FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY title, topic, model """ @@ -110,7 +95,7 @@ def run_6h( for table in ("default_agg_topic", "default_agg_topic_2"): write_points( influxdb3_local, - db_name=agg_db, + db_name=config.agg_database, table_name=table, rows=topic_rows, time_col="time", @@ -125,8 +110,7 @@ def run_6h( task_id=task_id, ) - # 3) token class histogram (kept compatible with your old task; stored in default_agg_topic by default) - token_table = str(args.get("token_class_table", "default_agg_topic")) + # 3) token class histogram token_sql = f""" SELECT '{start_s}' AS time, @@ -137,15 +121,15 @@ def run_6h( END AS user_type, {token_class_case_sql()} AS prompt_token_class, COUNT(*) AS request_count -FROM {raw_table} +FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY user_type, prompt_token_class """ token_rows = query_rows(influxdb3_local, token_sql) write_points( influxdb3_local, - db_name=agg_db, - table_name=token_table, + db_name=config.agg_database, + table_name="default_agg_topic", rows=token_rows, time_col="time", tag_cols=("user_type", "prompt_token_class"), @@ -165,14 +149,14 @@ def run_6h( SUM(completion_tokens) AS completion_tokens, SUM(prompt_tokens) AS prompt_tokens, SUM(price) AS cost -FROM {raw_table} +FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY user_hash, project_id, parent_deployment, title """ kpi_rows = query_rows(influxdb3_local, kpi_sql) write_points( influxdb3_local, - db_name=agg_db, + db_name=config.agg_database, table_name="default_agg_kpi", rows=kpi_rows, time_col="time", @@ -192,14 +176,14 @@ def run_6h( '{start_s}' AS time, chat_id, COUNT(*) AS request_count -FROM {raw_table} +FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY chat_id """ chat_rows = query_rows(influxdb3_local, chat_sql) write_points( influxdb3_local, - db_name=agg_db, + db_name=config.agg_database, table_name="default_agg_chatid", rows=chat_rows, time_col="time", diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index dc62c97..b904bda 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict +from .config import Config from .utils import query_rows, to_iso, write_points @@ -12,10 +13,8 @@ def _month_start(dt: datetime) -> datetime: def run_monthly( - influxdb3_local, call_time: datetime, args: Dict[str, Any], *, task_id: str + influxdb3_local, call_time: datetime, config: Config, *, task_id: str ) -> None: - agg_db = str(args.get("agg_database", "analytics_agg")) - # Previous month window: [prev_month_start, this_month_start) this_month = _month_start(call_time) prev_month = _month_start(this_month - timedelta(days=32)) @@ -39,7 +38,8 @@ def run_monthly( AVG(request_count) AS avg_rc_per_api FROM default_agg_stats WHERE time >= '{start_s}' AND time < '{end_s}' - AND project_id IS NOT NULL AND project_id <> '' + AND project_id IS NOT NULL + AND project_id <> '' """ api_rows = query_rows(influxdb3_local, api_sql) @@ -54,7 +54,7 @@ def run_monthly( """ model_rows = query_rows(influxdb3_local, model_sql) - # User-level rollups from default_agg_kpi (exclude undefined like your Flux) + # User-level rollups from default_agg_kpi user_sql = f""" SELECT '{stamp_s}' AS time, @@ -63,7 +63,8 @@ def run_monthly( COUNT(DISTINCT user_hash) AS unique_users FROM default_agg_kpi WHERE time >= '{start_s}' AND time < '{end_s}' - AND user_hash IS NOT NULL AND user_hash <> 'undefined' + AND user_hash IS NOT NULL + AND user_hash <> 'undefined' """ user_rows = query_rows(influxdb3_local, user_sql) @@ -77,7 +78,7 @@ def run_monthly( write_points( influxdb3_local, - db_name=agg_db, + db_name=config.agg_database, table_name="default_agg_month", rows=[merged], time_col="time", diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index ec229d2..5989169 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -18,7 +18,8 @@ def bool_field(self, key: str, value: bool) -> None: ... def parse_iso_datetime(name: str, value: str) -> datetime: try: - dt: datetime = datetime.fromisoformat(value) + # In Python ≤3.10, "Z" was not supported by fromisoformat() + dt: datetime = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: raise ValueError( f"Invalid ISO 8601 datetime from the {name!r} column: {value!r}." From 90e8180099aa898f489536d1bf95c64f5274e879 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 12:08:01 +0000 Subject: [PATCH 16/56] feat: simplify Config class --- dashboards/customized/influxdb_v3/plugin/config.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index b6b6a76..3892f29 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -27,13 +27,11 @@ class Config: def parse(cls, d: Dict[str, str] | None) -> "Config": d = d or {} - mode_s = str(d.get("mode", "hourly")).strip().lower() - if mode_s == "hourly": - mode = Mode.HOURLY - elif mode_s == "monthly": - mode = Mode.MONTHLY - else: - raise ValueError(f"unsupported mode: {mode_s}") + mode_s = d.get("mode") or "hourly" + try: + mode = Mode(mode_s.strip().lower()) + except Exception as e: + raise ValueError(f"Unsupported mode: {mode_s!r}") from e agg_database = d.get("agg_database") raw_table = d.get("raw_table") From 0db0dd76d9f4503928ea257599db4ab300ba3191 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 12:23:08 +0000 Subject: [PATCH 17/56] chore: update README --- dashboards/customized/influxdb_v3/README.md | 22 ++++++------------- .../customized/influxdb_v3/plugin/utils.py | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md index e47f983..770f94a 100644 --- a/dashboards/customized/influxdb_v3/README.md +++ b/dashboards/customized/influxdb_v3/README.md @@ -35,20 +35,12 @@ Aggregated data is written into the following **tables** (tables are created aut ## Aggregation Logic Overview -All aggregation logic lives in a **single Python scheduled plugin**, structured as a small package: - -```txt -influx_v3/plugin/ - __init__.py # Scheduled plugin entrypoint - rollup_6h.py # 6-hour aggregations - rollup_monthly.py # Monthly aggregations - utils.py # Shared helpers (time windows, SQL, writing) -``` +All aggregation logic lives in a **single Python scheduled plugin**, structured as a small [package](./plugin/). This plugin: -* queries raw data from `default` -* writes roll-ups into `analytics_agg` +* queries raw data from the `default.analytics` table +* writes hourly and monthly roll-ups into `analytics_agg.*` tables * supports **scheduled execution** and **manual backfills** using the same code paths --- @@ -93,7 +85,7 @@ influxdb3 create trigger \ --database default \ --path analytics_rollups \ --trigger-spec "cron:0 0 */6 * * *" \ - --trigger-arguments mode=6h,raw_table=analytics,agg_database=analytics_agg,window_hours=6,offset_minutes=2 \ + --trigger-arguments mode=hourly,raw_table=analytics,agg_database=analytics_agg,window_hours=6,offset_minutes=2 \ analytics_6h_rollups ``` @@ -123,7 +115,7 @@ Example trigger creation: ```sh influxdb3 create trigger \ - --database default \ + --database analytics_agg \ --path analytics_rollups \ --trigger-spec "cron:0 0 0 1 * *" \ --trigger-arguments mode=monthly,agg_database=analytics_agg \ @@ -141,8 +133,8 @@ If you already have existing raw data, you **must backfill aggregates explicitly The plugin supports two optional arguments: -* `start_time` — RFC3339 timestamp (inclusive) -* `end_time` — RFC3339 timestamp (exclusive) +* `start_time` — ISO 8601 timestamp (inclusive) +* `end_time` — ISO 8601 timestamp (exclusive) When provided, these **override the scheduled window**. diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index 5989169..46db030 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -18,7 +18,7 @@ def bool_field(self, key: str, value: bool) -> None: ... def parse_iso_datetime(name: str, value: str) -> datetime: try: - # In Python ≤3.10, "Z" was not supported by fromisoformat() + # In Python<=3.10, "Z" was not supported by fromisoformat() dt: datetime = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: raise ValueError( From 2fc117e3234eeaf3f8c126b30805dafd32753975 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 12:31:03 +0000 Subject: [PATCH 18/56] chore: fix yml formatting --- .../influxdb/tasks/tasks_template.yml | 606 +++++++++--------- 1 file changed, 303 insertions(+), 303 deletions(-) diff --git a/dashboards/customized/influxdb/tasks/tasks_template.yml b/dashboards/customized/influxdb/tasks/tasks_template.yml index 1c8221b..1875d0b 100644 --- a/dashboards/customized/influxdb/tasks/tasks_template.yml +++ b/dashboards/customized/influxdb/tasks/tasks_template.yml @@ -624,306 +624,306 @@ spec: cron: 0 0 1 * * name: monthly_agg query: |- - import "influxdata/influxdb/schema" - import "experimental" import "date" - import "influxdata/influxdb/secrets" - - - option org = secrets.get(key: "org") - - getOrDefault = (f, d) => if exists f then f else d - - - now_time = now() - - - first_day_current_month = date.truncate(t: now_time, unit: 1mo) - - - start_time = experimental.addDuration(d: -1mo, to: first_day_current_month) - - - end_time = experimental.addDuration(d: -1s, to: first_day_current_month) - - - toBucket_monthly = "default_agg_month" - - - project_api_data = - from(bucket: "default_agg_stats") - |> range(start: start_time, stop: end_time) - |> filter(fn: (r) => r._field == "request_count" or r._field == "price") - |> filter(fn: (r) => r["project_id"] != "") - |> schema.fieldsAsCols() - |> group() - - monthly_api_data = - project_api_data - |> map( - fn: (r) => - ({r with month: date.truncate(t: r._time, unit: 1mo), - // Truncate time to the month - - }), - ) - |> group(columns: ["month", "project_id"]) - // Group by month and user_hash - |> reduce( - fn: (r, accumulator) => - ({ - request_count: getOrDefault(f: r.request_count, d: 0) + accumulator.request_count, - price: getOrDefault(f: r.price, d: 0.0) + accumulator.price, - }), - identity: {request_count: 0, price: 0.0}, - ) - - monthly_model_data = - project_api_data - |> map( - fn: (r) => - ({r with month: date.truncate(t: r._time, unit: 1mo), - // Truncate time to the month - - }), - ) - |> group(columns: ["month", "model"]) - // Group by month and user_hash - |> reduce( - fn: (r, accumulator) => - ({ - request_count: getOrDefault(f: r.request_count, d: 0) + accumulator.request_count, - price: getOrDefault(f: r.price, d: 0.0) + accumulator.price, - }), - identity: {request_count: 0, price: 0.0}, - ) - - total_api_cost = - monthly_api_data - |> group(columns: ["month"]) - // Group by month only - |> sum(column: "price") - // Sum all user costs per month - |> rename(columns: {price: "Total_Cost_Per_Api"}) - - avg_api_cost = - monthly_api_data - |> group(columns: ["month"]) - // Group by month only - |> mean(column: "price") - // Compute the average cost per user - |> rename(columns: {price: "Avg_Cost_Per_Api"}) - - unique_api = - monthly_api_data - |> group(columns: ["month"]) - // Group by month only - |> count(column: "project_id") - // Compute the average cost per user - |> rename(columns: {project_id: "Active_Apis"}) - - total_api_request = - monthly_api_data - |> group(columns: ["month"]) - // Group by month only - |> sum(column: "request_count") - // Compute the average cost per user - |> rename(columns: {request_count: "Total_RC_Per_Api"}) - - avg_api_request = - monthly_api_data - |> group(columns: ["month"]) - // Group by month only - |> mean(column: "request_count") - // Compute the average cost per user - |> rename(columns: {request_count: "Avg_RC_Per_Api"}) - - total_model_cost = - monthly_model_data - |> group(columns: ["month"]) - // Group by month only - |> sum(column: "price") - // Sum all user costs per month - |> rename(columns: {price: "Total_Cost_Per_Model"}) - - avg_model_cost = - monthly_model_data - |> group(columns: ["month"]) - // Group by month only - |> mean(column: "price") - // Compute the average cost per user - |> rename(columns: {price: "Avg_Cost_Per_Model"}) - - kpi_data = - from(bucket: "default_agg_kpi") - |> range(start: start_time, stop: end_time) - |> filter(fn: (r) => r["_measurement"] == "analytics") - |> filter(fn: (r) => r["user_hash"] != "undefined") - |> schema.fieldsAsCols() - |> group() - - user_cost_per_month = - kpi_data - |> map( - fn: (r) => - ({r with month: date.truncate(t: r._time, unit: 1mo), - // Truncate time to the month - - }), - ) - |> group(columns: ["month", "user_hash"]) - // Group by month and user_hash - |> sum(column: "cost") - - total_user_cost = - user_cost_per_month - |> group(columns: ["month"]) - // Group by month only - |> sum(column: "cost") - // Sum all user costs per month - |> rename(columns: {cost: "total_user_cost"}) - - avg_per_user_per_month = - user_cost_per_month - |> group(columns: ["month"]) - // Group by month only - |> mean(column: "cost") - // Compute the average cost per user - |> rename(columns: {cost: "Avg_Cost_Per_User"}) - - unique_user_per_month = - user_cost_per_month - |> group(columns: ["month"]) - // Group by month only - |> count(column: "user_hash") - // Compute the average cost per user - |> rename(columns: {user_hash: "Unique_Users"}) - - final_transformed = - union( - tables: [ - total_user_cost - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "total_user_cost", - _value: float(v: r.total_user_cost), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - avg_per_user_per_month - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Avg_Cost_Per_User", - _value: float(v: r.Avg_Cost_Per_User), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - unique_user_per_month - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Unique_Users", - _value: float(v: r.Unique_Users), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - total_api_cost - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Total_Cost_Per_Api", - _value: float(v: r.Total_Cost_Per_Api), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - avg_api_cost - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Avg_Cost_Per_Api", - _value: float(v: r.Avg_Cost_Per_Api), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - unique_api - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Active_Apis", - _value: float(v: r.Active_Apis), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - total_api_request - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Total_RC_Per_Api", - _value: float(v: r.Total_RC_Per_Api), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - avg_api_request - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Avg_RC_Per_Api", - _value: float(v: r.Avg_RC_Per_Api), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - total_model_cost - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Total_Cost_Per_Model", - _value: float(v: r.Total_Cost_Per_Model), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - avg_model_cost - |> rename(columns: {month: "_time"}) - |> map( - fn: (r) => - ({ - _time: r._time, - _measurement: "analytics", - _field: "Avg_Cost_Per_Model", - _value: float(v: r.Avg_Cost_Per_Model), - }), - ) - |> keep(columns: ["_time", "_measurement", "_field", "_value"]), - ], - ) - - final_transformed - |> group() - |> to(bucket: toBucket_monthly, org: org) + import "influxdata/influxdb/schema" + import "experimental" import "date" + import "influxdata/influxdb/secrets" + + + option org = secrets.get(key: "org") + + getOrDefault = (f, d) => if exists f then f else d + + + now_time = now() + + + first_day_current_month = date.truncate(t: now_time, unit: 1mo) + + + start_time = experimental.addDuration(d: -1mo, to: first_day_current_month) + + + end_time = experimental.addDuration(d: -1s, to: first_day_current_month) + + + toBucket_monthly = "default_agg_month" + + + project_api_data = + from(bucket: "default_agg_stats") + |> range(start: start_time, stop: end_time) + |> filter(fn: (r) => r._field == "request_count" or r._field == "price") + |> filter(fn: (r) => r["project_id"] != "") + |> schema.fieldsAsCols() + |> group() + + monthly_api_data = + project_api_data + |> map( + fn: (r) => + ({r with month: date.truncate(t: r._time, unit: 1mo), + // Truncate time to the month + + }), + ) + |> group(columns: ["month", "project_id"]) + // Group by month and user_hash + |> reduce( + fn: (r, accumulator) => + ({ + request_count: getOrDefault(f: r.request_count, d: 0) + accumulator.request_count, + price: getOrDefault(f: r.price, d: 0.0) + accumulator.price, + }), + identity: {request_count: 0, price: 0.0}, + ) + + monthly_model_data = + project_api_data + |> map( + fn: (r) => + ({r with month: date.truncate(t: r._time, unit: 1mo), + // Truncate time to the month + + }), + ) + |> group(columns: ["month", "model"]) + // Group by month and user_hash + |> reduce( + fn: (r, accumulator) => + ({ + request_count: getOrDefault(f: r.request_count, d: 0) + accumulator.request_count, + price: getOrDefault(f: r.price, d: 0.0) + accumulator.price, + }), + identity: {request_count: 0, price: 0.0}, + ) + + total_api_cost = + monthly_api_data + |> group(columns: ["month"]) + // Group by month only + |> sum(column: "price") + // Sum all user costs per month + |> rename(columns: {price: "Total_Cost_Per_Api"}) + + avg_api_cost = + monthly_api_data + |> group(columns: ["month"]) + // Group by month only + |> mean(column: "price") + // Compute the average cost per user + |> rename(columns: {price: "Avg_Cost_Per_Api"}) + + unique_api = + monthly_api_data + |> group(columns: ["month"]) + // Group by month only + |> count(column: "project_id") + // Compute the average cost per user + |> rename(columns: {project_id: "Active_Apis"}) + + total_api_request = + monthly_api_data + |> group(columns: ["month"]) + // Group by month only + |> sum(column: "request_count") + // Compute the average cost per user + |> rename(columns: {request_count: "Total_RC_Per_Api"}) + + avg_api_request = + monthly_api_data + |> group(columns: ["month"]) + // Group by month only + |> mean(column: "request_count") + // Compute the average cost per user + |> rename(columns: {request_count: "Avg_RC_Per_Api"}) + + total_model_cost = + monthly_model_data + |> group(columns: ["month"]) + // Group by month only + |> sum(column: "price") + // Sum all user costs per month + |> rename(columns: {price: "Total_Cost_Per_Model"}) + + avg_model_cost = + monthly_model_data + |> group(columns: ["month"]) + // Group by month only + |> mean(column: "price") + // Compute the average cost per user + |> rename(columns: {price: "Avg_Cost_Per_Model"}) + + kpi_data = + from(bucket: "default_agg_kpi") + |> range(start: start_time, stop: end_time) + |> filter(fn: (r) => r["_measurement"] == "analytics") + |> filter(fn: (r) => r["user_hash"] != "undefined") + |> schema.fieldsAsCols() + |> group() + + user_cost_per_month = + kpi_data + |> map( + fn: (r) => + ({r with month: date.truncate(t: r._time, unit: 1mo), + // Truncate time to the month + + }), + ) + |> group(columns: ["month", "user_hash"]) + // Group by month and user_hash + |> sum(column: "cost") + + total_user_cost = + user_cost_per_month + |> group(columns: ["month"]) + // Group by month only + |> sum(column: "cost") + // Sum all user costs per month + |> rename(columns: {cost: "total_user_cost"}) + + avg_per_user_per_month = + user_cost_per_month + |> group(columns: ["month"]) + // Group by month only + |> mean(column: "cost") + // Compute the average cost per user + |> rename(columns: {cost: "Avg_Cost_Per_User"}) + + unique_user_per_month = + user_cost_per_month + |> group(columns: ["month"]) + // Group by month only + |> count(column: "user_hash") + // Compute the average cost per user + |> rename(columns: {user_hash: "Unique_Users"}) + + final_transformed = + union( + tables: [ + total_user_cost + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "total_user_cost", + _value: float(v: r.total_user_cost), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + avg_per_user_per_month + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Avg_Cost_Per_User", + _value: float(v: r.Avg_Cost_Per_User), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + unique_user_per_month + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Unique_Users", + _value: float(v: r.Unique_Users), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + total_api_cost + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Total_Cost_Per_Api", + _value: float(v: r.Total_Cost_Per_Api), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + avg_api_cost + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Avg_Cost_Per_Api", + _value: float(v: r.Avg_Cost_Per_Api), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + unique_api + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Active_Apis", + _value: float(v: r.Active_Apis), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + total_api_request + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Total_RC_Per_Api", + _value: float(v: r.Total_RC_Per_Api), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + avg_api_request + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Avg_RC_Per_Api", + _value: float(v: r.Avg_RC_Per_Api), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + total_model_cost + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Total_Cost_Per_Model", + _value: float(v: r.Total_Cost_Per_Model), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + avg_model_cost + |> rename(columns: {month: "_time"}) + |> map( + fn: (r) => + ({ + _time: r._time, + _measurement: "analytics", + _field: "Avg_Cost_Per_Model", + _value: float(v: r.Avg_Cost_Per_Model), + }), + ) + |> keep(columns: ["_time", "_measurement", "_field", "_value"]), + ], + ) + + final_transformed + |> group() + |> to(bucket: toBucket_monthly, org: org) From 3064233de603e44da46c71e78b3f97e9c18a4e96 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 14:35:12 +0000 Subject: [PATCH 19/56] feat: fix rollup monthly script --- .../influxdb_v3/plugin/rollup_monthly.py | 72 ++++++++++++------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index b904bda..d7f84e7 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -7,17 +7,11 @@ from .utils import query_rows, to_iso, write_points -def _month_start(dt: datetime) -> datetime: - dt = dt.astimezone(timezone.utc) - return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - - def run_monthly( influxdb3_local, call_time: datetime, config: Config, *, task_id: str ) -> None: - # Previous month window: [prev_month_start, this_month_start) this_month = _month_start(call_time) - prev_month = _month_start(this_month - timedelta(days=32)) + prev_month = _month_start(this_month - timedelta(days=2)) start_s = to_iso(prev_month) end_s = to_iso(this_month) @@ -31,17 +25,13 @@ def run_monthly( api_sql = f""" SELECT '{stamp_s}' AS time, - SUM(price) AS total_cost_per_api, - AVG(price) AS avg_cost_per_api, - COUNT(DISTINCT project_id) AS active_apis, - SUM(request_count) AS total_rc_per_api, - AVG(request_count) AS avg_rc_per_api -FROM default_agg_stats -WHERE time >= '{start_s}' AND time < '{end_s}' - AND project_id IS NOT NULL - AND project_id <> '' + SUM(price) AS total_cost_per_api, + AVG(price) AS avg_cost_per_api, + SUM(request_count) AS total_rc_per_api, + AVG(request_count) AS avg_rc_per_api, + COUNT(*) AS active_apis +FROM ({_get_stats_sub_table("project_id", start_s, end_s)}) """ - api_rows = query_rows(influxdb3_local, api_sql) # Model-level rollups from default_agg_stats model_sql = f""" @@ -49,23 +39,21 @@ def run_monthly( '{stamp_s}' AS time, SUM(price) AS total_cost_per_model, AVG(price) AS avg_cost_per_model -FROM default_agg_stats -WHERE time >= '{start_s}' AND time < '{end_s}' +FROM ({_get_stats_sub_table("model", start_s, end_s)}) """ - model_rows = query_rows(influxdb3_local, model_sql) # User-level rollups from default_agg_kpi user_sql = f""" SELECT '{stamp_s}' AS time, - SUM(cost) AS total_user_cost, - AVG(cost) AS avg_cost_per_user, - COUNT(DISTINCT user_hash) AS unique_users -FROM default_agg_kpi -WHERE time >= '{start_s}' AND time < '{end_s}' - AND user_hash IS NOT NULL - AND user_hash <> 'undefined' + SUM(cost) AS total_user_cost, + AVG(cost) AS avg_cost_per_user, + COUNT(*) AS unique_users +FROM ({_get_kpi_sub_table(start_s, end_s)}) """ + + api_rows = query_rows(influxdb3_local, api_sql) + model_rows = query_rows(influxdb3_local, model_sql) user_rows = query_rows(influxdb3_local, user_sql) merged: Dict[str, Any] = {"time": stamp_s} @@ -97,3 +85,33 @@ def run_monthly( ), task_id=task_id, ) + + +def _month_start(dt: datetime) -> datetime: + dt = dt.astimezone(timezone.utc) + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + +def _get_stats_sub_table(group_by: str, start_s: str, end_s: str) -> str: + return f""" +SELECT + SUM(price) AS price, + SUM(request_count) AS request_count +FROM default_agg_stats +WHERE time >= '{start_s}' AND time < '{end_s}' + AND project_id IS NOT NULL + AND project_id <> '' +GROUP BY {group_by} +""".strip() + + +def _get_kpi_sub_table(start_s: str, end_s: str) -> str: + return f""" +SELECT + SUM(cost) AS cost +FROM default_agg_kpi +WHERE time >= '{start_s}' AND time < '{end_s}' + AND user_hash IS NOT NULL + AND user_hash <> 'undefined' +GROUP BY user_hash +""".strip() From 121f1396521f27326ba5e34335ce676343ab7d36 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 4 Mar 2026 17:38:25 +0000 Subject: [PATCH 20/56] feat: fix hourly script --- .../influxdb_v3/plugin/rollup_hourly.py | 72 ++++++++++++------- .../influxdb_v3/plugin/rollup_monthly.py | 5 ++ 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index d6457da..eb5f68a 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -15,6 +15,10 @@ def run_hourly( influxdb3_local, call_time: datetime, config: Config, *, task_id: str ) -> None: + """ + Reads: raw_table + Writes: default_agg_stats, default_agg_topic, default_agg_topic_2, default_agg_kpi, default_agg_chatid + """ start, end = window_from_args_or_call_time( call_time, @@ -26,7 +30,9 @@ def run_hourly( start_s, end_s = to_iso(start), to_iso(end) - influxdb3_local.info(f"[{task_id}] 6h rollup window: {start_s} .. {end_s}") + influxdb3_local.info( + f"[{task_id}] {config.window_hours}-hours rollup window: {start_s} .. {end_s}" + ) # 1) default_agg_stats stats_sql = f""" @@ -48,6 +54,7 @@ def run_hourly( WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY deployment, model, project_id, parent_deployment, language """ + stats_rows = query_rows(influxdb3_local, stats_sql) write_points( influxdb3_local, @@ -74,7 +81,7 @@ def run_hourly( task_id=task_id, ) - # 2) default_agg_topic + default_agg_topic_2 (same content, but different retention/purpose - FIXME????) + # 2) default_agg_topic_2 topic_sql = f""" SELECT '{start_s}' AS time, @@ -90,27 +97,26 @@ def run_hourly( WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY title, topic, model """ + topic_rows = query_rows(influxdb3_local, topic_sql) + write_points( + influxdb3_local, + db_name=config.agg_database, + table_name="default_agg_topic_2", + rows=topic_rows, + time_col="time", + tag_cols=("title", "topic", "model"), + field_cols=( + "topic_count", + "number_request_messages", + "price", + "prompt_tokens", + "completion_tokens", + ), + task_id=task_id, + ) - for table in ("default_agg_topic", "default_agg_topic_2"): - write_points( - influxdb3_local, - db_name=config.agg_database, - table_name=table, - rows=topic_rows, - time_col="time", - tag_cols=("title", "topic", "model"), - field_cols=( - "topic_count", - "number_request_messages", - "price", - "prompt_tokens", - "completion_tokens", - ), - task_id=task_id, - ) - - # 3) token class histogram + # 3) default_agg_topic - token class histogram token_sql = f""" SELECT '{start_s}' AS time, @@ -119,12 +125,17 @@ def run_hourly( THEN 'project' ELSE 'user' END AS user_type, - {token_class_case_sql()} AS prompt_token_class, - COUNT(*) AS request_count + SUM(CAST(50000 <= prompt_tokens AS INT)) AS class_1, + SUM(CAST(10000 < prompt_tokens AND prompt_tokens < 50000 AS INT)) AS class_2, + SUM(CAST( 5000 < prompt_tokens AND prompt_tokens < 10000 AS INT)) AS class_3, + SUM(CAST( 1000 < prompt_tokens AND prompt_tokens < 5000 AS INT)) AS class_4, + SUM(CAST( 100 < prompt_tokens AND prompt_tokens < 1000 AS INT)) AS class_5, + SUM(CAST( prompt_tokens <= 100 AS INT)) AS class_6 FROM {config.raw_table} WHERE time >= '{start_s}' AND time < '{end_s}' -GROUP BY user_type, prompt_token_class +GROUP BY user_type """ + token_rows = query_rows(influxdb3_local, token_sql) write_points( influxdb3_local, @@ -132,8 +143,15 @@ def run_hourly( table_name="default_agg_topic", rows=token_rows, time_col="time", - tag_cols=("user_type", "prompt_token_class"), - field_cols=("request_count",), + tag_cols=("user_type"), + field_cols=( + "class_1", + "class_2", + "class_3", + "class_4", + "class_5", + "class_6", + ), task_id=task_id, ) @@ -153,6 +171,7 @@ def run_hourly( WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY user_hash, project_id, parent_deployment, title """ + kpi_rows = query_rows(influxdb3_local, kpi_sql) write_points( influxdb3_local, @@ -180,6 +199,7 @@ def run_hourly( WHERE time >= '{start_s}' AND time < '{end_s}' GROUP BY chat_id """ + chat_rows = query_rows(influxdb3_local, chat_sql) write_points( influxdb3_local, diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index d7f84e7..37ebf45 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -10,6 +10,11 @@ def run_monthly( influxdb3_local, call_time: datetime, config: Config, *, task_id: str ) -> None: + """ + Reads: default_agg_stats, default_agg_kpi + Writes: default_agg_month + """ + this_month = _month_start(call_time) prev_month = _month_start(this_month - timedelta(days=2)) From c696079f4845691036470e6da64045b2fa579bff Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 10:20:57 +0000 Subject: [PATCH 21/56] feat: fix default_agg_topic table --- .../customized/influxdb_v3/plugin/rollup_hourly.py | 1 - dashboards/customized/influxdb_v3/plugin/utils.py | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index eb5f68a..f853fa4 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -6,7 +6,6 @@ from .utils import ( query_rows, to_iso, - token_class_case_sql, window_from_args_or_call_time, write_points, ) diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index 46db030..31d4476 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -128,16 +128,3 @@ def write_points( f"[{task_id}] wrote {written} points -> {db_name}.{table_name}" ) return written - - -def token_class_case_sql() -> str: - return """ -CASE - WHEN prompt_tokens >= 50000 THEN 'class_1' - WHEN prompt_tokens > 10000 THEN 'class_2' - WHEN prompt_tokens > 5000 THEN 'class_3' - WHEN prompt_tokens > 1000 THEN 'class_4' - WHEN prompt_tokens > 100 THEN 'class_5' - ELSE 'class_6' -END -""".strip() From a942303b89a08a48e4b1d249039180a40090a31c Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 11:44:33 +0000 Subject: [PATCH 22/56] feat: minor refactoring --- .../customized/influxdb_v3/plugin/__init__.py | 2 +- .../customized/influxdb_v3/plugin/config.py | 41 +++++++++++++++---- .../influxdb_v3/plugin/rollup_hourly.py | 16 +------- .../customized/influxdb_v3/plugin/utils.py | 27 +----------- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 3a3ce32..8b45814 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -25,7 +25,7 @@ { "name": "window_hours", "example": "6", - "description": "For mode=hourly: window size in hours.", + "description": "For mode=hourly: window size in hours. Must be a divisor of 24. (default: 6 => windows are [00,06), [06,12), [12,18), [18,24)).", "required": false }, { diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 3892f29..5fef1b2 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from enum import Enum -from typing import Dict +from typing import Dict, Tuple from .utils import parse_iso_datetime @@ -33,8 +33,8 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": except Exception as e: raise ValueError(f"Unsupported mode: {mode_s!r}") from e - agg_database = d.get("agg_database") - raw_table = d.get("raw_table") + agg_database = d.get("agg_database") or "analytics_agg" + raw_table = d.get("raw_table") or "analytics" start_arg = d.get("start_time") end_arg = d.get("end_time") @@ -45,12 +45,37 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": parse_iso_datetime("end_time", end_arg) if end_arg else None ) + offset_minutes = int(d.get("offset_minutes") or 2) + window_hours = int(d.get("window_hours") or 6) + + if 24 % window_hours: + raise ValueError( + f"window_hours must divide 24 evenly (got {window_hours})" + ) + return cls( mode=mode, - agg_database=agg_database or "analytics_agg", - raw_table=raw_table or "analytics", + agg_database=agg_database, + raw_table=raw_table, start_time=start_time, end_time=end_time, - window_hours=int(d.get("window_hours") or 6), - offset_minutes=int(d.get("offset_minutes") or 2), + window_hours=window_hours, + offset_minutes=offset_minutes, ) + + def get_window(self, call_time: datetime) -> Tuple[datetime, datetime]: + """ + If args contains start_time/end_time => backfill window. + Else uses call_time to compute [end - window, end) with optional offset. + """ + + if self.start_time and self.end_time: + if self.end_time <= self.start_time: + raise ValueError( + f"end_time must be > start_time (got {self.start_time} .. {self.end_time})" + ) + return self.start_time, self.end_time + + end_time = call_time - timedelta(minutes=self.offset_minutes) + start_time = end_time - timedelta(hours=self.window_hours) + return start_time, end_time diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index f853fa4..168c3ae 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -3,12 +3,7 @@ from datetime import datetime from .config import Config -from .utils import ( - query_rows, - to_iso, - window_from_args_or_call_time, - write_points, -) +from .utils import query_rows, to_iso, write_points def run_hourly( @@ -19,14 +14,7 @@ def run_hourly( Writes: default_agg_stats, default_agg_topic, default_agg_topic_2, default_agg_kpi, default_agg_chatid """ - start, end = window_from_args_or_call_time( - call_time, - config.start_time, - config.end_time, - window_hours=config.window_hours, - offset_minutes=config.offset_minutes, - ) - + start, end = config.get_window(call_time) start_s, end_s = to_iso(start), to_iso(end) influxdb3_local.info( diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index 31d4476..f8e02af 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -3,6 +3,8 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence, Tuple +from .config import Config + if TYPE_CHECKING: # LineBuilder is available in the processing runtime (as used by official plugins). # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data @@ -39,31 +41,6 @@ def _ns(dt: datetime) -> int: return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) -def window_from_args_or_call_time( - call_time: datetime, - end_time: datetime | None, - start_time: datetime | None, - *, - window_hours: int, - offset_minutes: int, -) -> Tuple[datetime, datetime]: - """ - If args contains start_time/end_time => backfill window. - Else uses call_time to compute [end - window, end) with optional offset. - """ - - if start_time and end_time: - if end_time <= start_time: - raise ValueError( - f"end_time must be > start_time (got {start_time} .. {end_time})" - ) - return start_time, end_time - - end_time = call_time - timedelta(minutes=offset_minutes) - start_time = end_time - timedelta(hours=window_hours) - return start_time, end_time - - def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: """ Execute SQL and return list of row dicts. From ad398673aec53ccf213271545fcd25974665eb2a Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 11:53:52 +0000 Subject: [PATCH 23/56] fix: remove offset_minutes parameter --- dashboards/customized/influxdb_v3/README.md | 25 ++++++++++++------- .../customized/influxdb_v3/plugin/__init__.py | 6 ----- .../customized/influxdb_v3/plugin/config.py | 5 +--- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md index 770f94a..0799153 100644 --- a/dashboards/customized/influxdb_v3/README.md +++ b/dashboards/customized/influxdb_v3/README.md @@ -55,21 +55,24 @@ We define **two triggers**, both referencing the same plugin package. ### 6-Hour Aggregation Trigger -Runs **exactly at**: +Runs at the following UTC times every day: ```txt -00:00, 06:00, 12:00, 18:00 (UTC) +00:02, 06:02, 12:02, 18:02 (UTC) ``` +The 2-minute offset (`02` instead of `00`) ensures that all raw data for the previous window is ingested before aggregation runs. + Trigger spec: ```txt -cron:0 0 */6 * * * +cron:2 0 */6 * * * ``` What it does: * aggregates the previous 6-hour window of raw data +* window bins are aligned to 6-hour boundaries (00:00, 06:00, 12:00, 18:00) which makes the query **idempotent** and backfill straightforward * writes results into: * `default_agg_stats` @@ -84,8 +87,8 @@ Example trigger creation: influxdb3 create trigger \ --database default \ --path analytics_rollups \ - --trigger-spec "cron:0 0 */6 * * *" \ - --trigger-arguments mode=hourly,raw_table=analytics,agg_database=analytics_agg,window_hours=6,offset_minutes=2 \ + --trigger-spec "cron:2 0 */6 * * *" \ + --trigger-arguments mode=hourly,raw_table=analytics,agg_database=analytics_agg,window_hours=6 \ analytics_6h_rollups ``` @@ -96,18 +99,21 @@ influxdb3 create trigger \ Runs at: ```txt -00:00:00 UTC on the 1st day of each month +00:00:02 UTC on the 1st day of each month ``` +The 2-minute offset (`02` instead of `00`) ensures that all raw data for the previous window is ingested before aggregation runs. + Trigger spec: ```txt -cron:0 0 0 1 * * +cron:2 0 0 1 * * ``` What it does: * reads from 6-hour aggregate tables +* window bins are aligned to monthly boundaries which makes the query **idempotent** and backfill straightforward * computes monthly totals, averages, and uniques * writes results into `default_agg_month` @@ -117,7 +123,7 @@ Example trigger creation: influxdb3 create trigger \ --database analytics_agg \ --path analytics_rollups \ - --trigger-spec "cron:0 0 0 1 * *" \ + --trigger-spec "cron:2 0 0 1 * *" \ --trigger-arguments mode=monthly,agg_database=analytics_agg \ analytics_monthly_rollups ``` @@ -141,7 +147,8 @@ When provided, these **override the scheduled window**. Example (single 6-hour window): ```text -mode=6h, +mode=hourly, +window_hours=6, raw_table=analytics, agg_database=analytics_agg, start_time=2026-01-01T00:00:00Z, diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 8b45814..25afe62 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -28,12 +28,6 @@ "description": "For mode=hourly: window size in hours. Must be a divisor of 24. (default: 6 => windows are [00,06), [06,12), [12,18), [18,24)).", "required": false }, - { - "name": "offset_minutes", - "example": "2", - "description": "For mode=hourly: shift end of window backward to avoid late data.", - "required": false - }, { "name": "start_time", "example": "2026-01-01T00:00:00Z", diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 5fef1b2..dd20ea9 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -18,7 +18,6 @@ class Config: raw_table: str window_hours: int - offset_minutes: int start_time: datetime | None end_time: datetime | None @@ -45,7 +44,6 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": parse_iso_datetime("end_time", end_arg) if end_arg else None ) - offset_minutes = int(d.get("offset_minutes") or 2) window_hours = int(d.get("window_hours") or 6) if 24 % window_hours: @@ -60,7 +58,6 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": start_time=start_time, end_time=end_time, window_hours=window_hours, - offset_minutes=offset_minutes, ) def get_window(self, call_time: datetime) -> Tuple[datetime, datetime]: @@ -76,6 +73,6 @@ def get_window(self, call_time: datetime) -> Tuple[datetime, datetime]: ) return self.start_time, self.end_time - end_time = call_time - timedelta(minutes=self.offset_minutes) + end_time = call_time start_time = end_time - timedelta(hours=self.window_hours) return start_time, end_time From 5ff36151519cfacfea1edf143a37b3f7faf23853 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 12:46:45 +0000 Subject: [PATCH 24/56] feat: support mode with many windows --- .../customized/influxdb_v3/plugin/__init__.py | 99 ++++++++++--------- .../customized/influxdb_v3/plugin/config.py | 58 +++++++---- .../customized/influxdb_v3/plugin/influx.py | 87 ++++++++++++++++ .../influxdb_v3/plugin/rollup_hourly.py | 28 ++++-- .../influxdb_v3/plugin/rollup_monthly.py | 48 ++++----- .../customized/influxdb_v3/plugin/utils.py | 91 +---------------- .../customized/influxdb_v3/plugin/window.py | 24 +++++ 7 files changed, 245 insertions(+), 190 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/influx.py create mode 100644 dashboards/customized/influxdb_v3/plugin/window.py diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 25afe62..b0b6434 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -1,52 +1,53 @@ """ { - "plugin_type": [ - "scheduled" - ], - "scheduled_args_config": [ - { - "name": "mode", - "example": "hourly", - "description": "Which rollup to run: 'hourly' (default) or 'monthly'.", - "required": false - }, - { - "name": "raw_table", - "example": "analytics", - "description": "Raw source table name (in the trigger database).", - "required": false - }, - { - "name": "agg_database", - "example": "analytics_agg", - "description": "Database to write aggregates into.", - "required": false - }, - { - "name": "window_hours", - "example": "6", - "description": "For mode=hourly: window size in hours. Must be a divisor of 24. (default: 6 => windows are [00,06), [06,12), [12,18), [18,24)).", - "required": false - }, - { - "name": "start_time", - "example": "2026-01-01T00:00:00Z", - "description": "Optional backfill start (ISO 8601). If set with end_time, overrides call_time-derived window.", - "required": false - }, - { - "name": "end_time", - "example": "2026-01-01T06:00:00Z", - "description": "Optional backfill end (ISO 8601)). If set with start_time, overrides call_time-derived window.", - "required": false - } - ] + "plugin_type": [ + "scheduled" + ], + "scheduled_args_config": [ + { + "name": "mode", + "example": "hourly", + "description": "Which rollup to run: 'hourly' (default) or 'monthly'.", + "required": false + }, + { + "name": "raw_table", + "example": "analytics", + "description": "Raw source table name (in the trigger database).", + "required": false + }, + { + "name": "agg_database", + "example": "analytics_agg", + "description": "Database to write aggregates into.", + "required": false + }, + { + "name": "window_hours", + "example": "6", + "description": "For mode=hourly: window size in hours. Must be a divisor of 24. (default: 6 => windows are [00,06), [06,12), [12,18), [18,24)).", + "required": false + }, + { + "name": "start_time", + "example": "2026-01-01T00:00:00Z", + "description": "Optional backfill start (ISO 8601). If set with end_time, overrides call_time-derived window.", + "required": false + }, + { + "name": "end_time", + "example": "2026-01-01T06:00:00Z", + "description": "Optional backfill end (ISO 8601)). If set with start_time, overrides call_time-derived window.", + "required": false + } + ] } """ import uuid from datetime import datetime, timezone -from typing import assert_never + +from typing_extensions import assert_never from .config import Config, Mode from .rollup_hourly import run_hourly @@ -67,14 +68,16 @@ def process_scheduled_call(influxdb3_local, call_time: datetime, args=None): ) try: - if config.mode == Mode.MONTHLY: - run_monthly(influxdb3_local, call_time, config, task_id=task_id) - elif config.mode == Mode.HOURLY: - run_hourly(influxdb3_local, call_time, config, task_id=task_id) - else: - assert_never(config.mode) + match config.mode: + case Mode.MONTHLY: + run_monthly(influxdb3_local, call_time, config, task_id=task_id) + case Mode.HOURLY: + run_hourly(influxdb3_local, call_time, config, task_id=task_id) + case _: + assert_never(config.mode) influxdb3_local.info(f"[{task_id}] completed OK") + except Exception as e: influxdb3_local.error(f"[{task_id}] failed: {e}") raise diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index dd20ea9..962b738 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -1,9 +1,12 @@ from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from enum import Enum -from typing import Dict, Tuple +from typing import Dict, List -from .utils import parse_iso_datetime +from typing_extensions import assert_never + +from .utils import parse_iso_date +from .window import Window class Mode(Enum): @@ -38,10 +41,10 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": start_arg = d.get("start_time") end_arg = d.get("end_time") start_time: datetime | None = ( - parse_iso_datetime("start_time", start_arg) if start_arg else None + parse_iso_date("start_time", start_arg) if start_arg else None ) end_time: datetime | None = ( - parse_iso_datetime("end_time", end_arg) if end_arg else None + parse_iso_date("end_time", end_arg) if end_arg else None ) window_hours = int(d.get("window_hours") or 6) @@ -60,19 +63,32 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": window_hours=window_hours, ) - def get_window(self, call_time: datetime) -> Tuple[datetime, datetime]: - """ - If args contains start_time/end_time => backfill window. - Else uses call_time to compute [end - window, end) with optional offset. - """ - - if self.start_time and self.end_time: - if self.end_time <= self.start_time: - raise ValueError( - f"end_time must be > start_time (got {self.start_time} .. {self.end_time})" - ) - return self.start_time, self.end_time - - end_time = call_time - start_time = end_time - timedelta(hours=self.window_hours) - return start_time, end_time + def get_windows(self, call_time: datetime) -> List[Window]: + match self.mode: + + case Mode.HOURLY: + if self.start_time and self.end_time: + if self.end_time <= self.start_time: + raise ValueError( + f"end_time must be > start_time (got {self.start_time} .. {self.end_time})" + ) + window = Window(start=self.start_time, end=self.end_time) + else: + end_time = call_time + start_time = end_time - timedelta(hours=self.window_hours) + window = Window(start=start_time, end=end_time) + + case Mode.MONTHLY: + this_month = _month_start(call_time) + prev_month = _month_start(this_month - timedelta(days=2)) + window = Window(start=prev_month, end=this_month) + + case _: + assert_never(self.mode) + + return [window] + + +def _month_start(dt: datetime) -> datetime: + dt = dt.astimezone(timezone.utc) + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx.py new file mode 100644 index 0000000..91fe46a --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/influx.py @@ -0,0 +1,87 @@ +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, Iterable, List + +from .utils import parse_iso_date + +if TYPE_CHECKING: + # LineBuilder is available in the processing runtime (as used by official plugins). + # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data + class LineBuilder: + def __init__(self, measurement: str) -> None: ... + def time_ns(self, value: int) -> None: ... + def tag(self, key: str, value: str) -> None: ... + def string_field(self, key: str, value: str) -> None: ... + def int64_field(self, key: str, value: int) -> None: ... + def float64_field(self, key: str, value: float) -> None: ... + def bool_field(self, key: str, value: bool) -> None: ... + + +def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: + """ + Execute SQL and return list of row dicts. + """ + res = influxdb3_local.query(sql) + return list(res) if res else [] + + +def write_points( + influxdb3_local, + *, + db_name: str, + table_name: str, + rows: Iterable[Dict[str, Any]], + time_col: str, + tag_cols: Sequence[str], + field_cols: Sequence[str], + task_id: str, +) -> int: + """ + Write each row as a point into db_name.table_name. + time_col can be RFC3339 string, datetime, or ns int. + """ + written = 0 + + for r in rows: + b = LineBuilder(table_name) + + t = r.get(time_col) + if isinstance(t, int): + b.time_ns(t) + elif isinstance(t, datetime): + b.time_ns(_ns(t)) + elif isinstance(t, str): + b.time_ns(_ns(parse_iso_date(f"{time_col!r} column", t))) + else: + raise ValueError( + f"Unexpected time value in the column '{time_col!r}': {t} of type {type(t)}" + ) + + for k in tag_cols: + if (v := r.get(k)) is not None: + b.tag(k, str(v)) + + for k in field_cols: + if (v := r.get(k)) is not None: + if isinstance(v, bool): + b.bool_field(k, v) + elif isinstance(v, int): + b.int64_field(k, v) + elif isinstance(v, float): + b.float64_field(k, v) + else: + raise ValueError( + f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" + ) + + influxdb3_local.write_to_db(db_name, b) + written += 1 + + influxdb3_local.info( + f"[{task_id}] wrote {written} points -> {db_name}.{table_name}" + ) + return written + + +def _ns(dt: datetime) -> int: + return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 168c3ae..ec2dc60 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -3,7 +3,8 @@ from datetime import datetime from .config import Config -from .utils import query_rows, to_iso, write_points +from .influx import query_rows, write_points +from .window import Window def run_hourly( @@ -14,11 +15,20 @@ def run_hourly( Writes: default_agg_stats, default_agg_topic, default_agg_topic_2, default_agg_kpi, default_agg_chatid """ - start, end = config.get_window(call_time) - start_s, end_s = to_iso(start), to_iso(end) + windows = config.get_windows(call_time) + + for window in windows: + run_hourly_window(influxdb3_local, config, window, task_id) + + +def run_hourly_window( + influxdb3_local, config: Config, window: Window, task_id: str +) -> None: + start_s = window.start_s + in_window = window.in_window_sql() influxdb3_local.info( - f"[{task_id}] {config.window_hours}-hours rollup window: {start_s} .. {end_s}" + f"[{task_id}] {config.window_hours}-hours rollup window: {window.display()}" ) # 1) default_agg_stats @@ -38,7 +48,7 @@ def run_hourly( COUNT(*) AS request_count, COUNT(DISTINCT user_hash) AS unique_user_count FROM {config.raw_table} -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} GROUP BY deployment, model, project_id, parent_deployment, language """ @@ -81,7 +91,7 @@ def run_hourly( SUM(prompt_tokens) AS prompt_tokens, SUM(completion_tokens) AS completion_tokens FROM {config.raw_table} -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} GROUP BY title, topic, model """ @@ -119,7 +129,7 @@ def run_hourly( SUM(CAST( 100 < prompt_tokens AND prompt_tokens < 1000 AS INT)) AS class_5, SUM(CAST( prompt_tokens <= 100 AS INT)) AS class_6 FROM {config.raw_table} -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} GROUP BY user_type """ @@ -155,7 +165,7 @@ def run_hourly( SUM(prompt_tokens) AS prompt_tokens, SUM(price) AS cost FROM {config.raw_table} -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} GROUP BY user_hash, project_id, parent_deployment, title """ @@ -183,7 +193,7 @@ def run_hourly( chat_id, COUNT(*) AS request_count FROM {config.raw_table} -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} GROUP BY chat_id """ diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index 37ebf45..1b38bea 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -1,10 +1,11 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone +from datetime import datetime from typing import Any, Dict from .config import Config -from .utils import query_rows, to_iso, write_points +from .influx import query_rows, write_points +from .window import Window def run_monthly( @@ -14,54 +15,58 @@ def run_monthly( Reads: default_agg_stats, default_agg_kpi Writes: default_agg_month """ + windows = config.get_windows(call_time) - this_month = _month_start(call_time) - prev_month = _month_start(this_month - timedelta(days=2)) + for window in windows: + run_monthly_window(influxdb3_local, config, window, task_id) - start_s = to_iso(prev_month) - end_s = to_iso(this_month) - stamp_s = start_s # stamp results at first day of prev month + +def run_monthly_window( + influxdb3_local, config: Config, window: Window, task_id: str +) -> None: + start_s = window.start_s + in_window = window.in_window_sql() influxdb3_local.info( - f"[{task_id}] monthly rollup window: {start_s} .. {end_s}" + f"[{task_id}] monthly rollup window: {window.display()}" ) # API-level (project_id) rollups from default_agg_stats api_sql = f""" SELECT - '{stamp_s}' AS time, + '{start_s}' AS time, SUM(price) AS total_cost_per_api, AVG(price) AS avg_cost_per_api, SUM(request_count) AS total_rc_per_api, AVG(request_count) AS avg_rc_per_api, COUNT(*) AS active_apis -FROM ({_get_stats_sub_table("project_id", start_s, end_s)}) +FROM ({_get_stats_sub_table("project_id", in_window)}) """ # Model-level rollups from default_agg_stats model_sql = f""" SELECT - '{stamp_s}' AS time, + '{start_s}' AS time, SUM(price) AS total_cost_per_model, AVG(price) AS avg_cost_per_model -FROM ({_get_stats_sub_table("model", start_s, end_s)}) +FROM ({_get_stats_sub_table("model", in_window)}) """ # User-level rollups from default_agg_kpi user_sql = f""" SELECT - '{stamp_s}' AS time, + '{start_s}' AS time, SUM(cost) AS total_user_cost, AVG(cost) AS avg_cost_per_user, COUNT(*) AS unique_users -FROM ({_get_kpi_sub_table(start_s, end_s)}) +FROM ({_get_kpi_sub_table(in_window)}) """ api_rows = query_rows(influxdb3_local, api_sql) model_rows = query_rows(influxdb3_local, model_sql) user_rows = query_rows(influxdb3_local, user_sql) - merged: Dict[str, Any] = {"time": stamp_s} + merged: Dict[str, Any] = {"time": start_s} if api_rows: merged.update(api_rows[0]) if model_rows: @@ -92,30 +97,25 @@ def run_monthly( ) -def _month_start(dt: datetime) -> datetime: - dt = dt.astimezone(timezone.utc) - return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - - -def _get_stats_sub_table(group_by: str, start_s: str, end_s: str) -> str: +def _get_stats_sub_table(group_by: str, in_window: str) -> str: return f""" SELECT SUM(price) AS price, SUM(request_count) AS request_count FROM default_agg_stats -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} AND project_id IS NOT NULL AND project_id <> '' GROUP BY {group_by} """.strip() -def _get_kpi_sub_table(start_s: str, end_s: str) -> str: +def _get_kpi_sub_table(in_window: str) -> str: return f""" SELECT SUM(cost) AS cost FROM default_agg_kpi -WHERE time >= '{start_s}' AND time < '{end_s}' +WHERE {in_window} AND user_hash IS NOT NULL AND user_hash <> 'undefined' GROUP BY user_hash diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/utils.py index f8e02af..4f0127b 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/utils.py @@ -1,24 +1,9 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence, Tuple +from datetime import datetime, timezone -from .config import Config -if TYPE_CHECKING: - # LineBuilder is available in the processing runtime (as used by official plugins). - # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data - class LineBuilder: - def __init__(self, measurement: str) -> None: ... - def time_ns(self, value: int) -> None: ... - def tag(self, key: str, value: str) -> None: ... - def string_field(self, key: str, value: str) -> None: ... - def int64_field(self, key: str, value: int) -> None: ... - def float64_field(self, key: str, value: float) -> None: ... - def bool_field(self, key: str, value: bool) -> None: ... - - -def parse_iso_datetime(name: str, value: str) -> datetime: +def parse_iso_date(name: str, value: str) -> datetime: try: # In Python<=3.10, "Z" was not supported by fromisoformat() dt: datetime = datetime.fromisoformat(value.replace("Z", "+00:00")) @@ -33,75 +18,5 @@ def parse_iso_datetime(name: str, value: str) -> datetime: return dt.astimezone(timezone.utc) -def to_iso(dt: datetime) -> str: +def to_iso_date(dt: datetime) -> str: return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _ns(dt: datetime) -> int: - return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) - - -def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: - """ - Execute SQL and return list of row dicts. - """ - res = influxdb3_local.query(sql) - return list(res) if res else [] - - -def write_points( - influxdb3_local, - *, - db_name: str, - table_name: str, - rows: Iterable[Dict[str, Any]], - time_col: str, - tag_cols: Sequence[str], - field_cols: Sequence[str], - task_id: str, -) -> int: - """ - Write each row as a point into db_name.table_name. - time_col can be RFC3339 string, datetime, or ns int. - """ - written = 0 - - for r in rows: - b = LineBuilder(table_name) - - t = r.get(time_col) - if isinstance(t, int): - b.time_ns(t) - elif isinstance(t, datetime): - b.time_ns(_ns(t)) - elif isinstance(t, str): - b.time_ns(_ns(parse_iso_datetime(f"{time_col!r} column", t))) - else: - raise ValueError( - f"Unexpected time value in the column '{time_col!r}': {t} of type {type(t)}" - ) - - for k in tag_cols: - if (v := r.get(k)) is not None: - b.tag(k, str(v)) - - for k in field_cols: - if (v := r.get(k)) is not None: - if isinstance(v, bool): - b.bool_field(k, v) - elif isinstance(v, int): - b.int64_field(k, v) - elif isinstance(v, float): - b.float64_field(k, v) - else: - raise ValueError( - f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" - ) - - influxdb3_local.write_to_db(db_name, b) - written += 1 - - influxdb3_local.info( - f"[{task_id}] wrote {written} points -> {db_name}.{table_name}" - ) - return written diff --git a/dashboards/customized/influxdb_v3/plugin/window.py b/dashboards/customized/influxdb_v3/plugin/window.py new file mode 100644 index 0000000..bd95816 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/window.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass +from datetime import datetime + +from .utils import to_iso_date + + +@dataclass +class Window: + start: datetime + end: datetime + + def display(self) -> str: + return f"[{self.start_s} .. {self.end_s})" + + def in_window_sql(self) -> str: + return f"'{self.start_s}' <= time AND time < '{self.end_s}'" + + @property + def start_s(self) -> str: + return to_iso_date(self.start) + + @property + def end_s(self) -> str: + return to_iso_date(self.end) From bb69fa5503e7934dd94301c6a0dd332d6dc1521f Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 12:48:56 +0000 Subject: [PATCH 25/56] fix: simplify code --- dashboards/customized/influxdb_v3/plugin/influx.py | 10 +--------- .../customized/influxdb_v3/plugin/rollup_hourly.py | 12 ++++++------ .../customized/influxdb_v3/plugin/rollup_monthly.py | 8 ++++---- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx.py index 91fe46a..1515731 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx.py +++ b/dashboards/customized/influxdb_v3/plugin/influx.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, Iterable, List +from typing import TYPE_CHECKING, Any, Dict, Iterable from .utils import parse_iso_date @@ -17,14 +17,6 @@ def float64_field(self, key: str, value: float) -> None: ... def bool_field(self, key: str, value: bool) -> None: ... -def query_rows(influxdb3_local, sql: str) -> List[Dict[str, Any]]: - """ - Execute SQL and return list of row dicts. - """ - res = influxdb3_local.query(sql) - return list(res) if res else [] - - def write_points( influxdb3_local, *, diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index ec2dc60..ce267e3 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -3,7 +3,7 @@ from datetime import datetime from .config import Config -from .influx import query_rows, write_points +from .influx import write_points from .window import Window @@ -52,7 +52,7 @@ def run_hourly_window( GROUP BY deployment, model, project_id, parent_deployment, language """ - stats_rows = query_rows(influxdb3_local, stats_sql) + stats_rows = influxdb3_local.query(stats_sql) write_points( influxdb3_local, db_name=config.agg_database, @@ -95,7 +95,7 @@ def run_hourly_window( GROUP BY title, topic, model """ - topic_rows = query_rows(influxdb3_local, topic_sql) + topic_rows = influxdb3_local.query(topic_sql) write_points( influxdb3_local, db_name=config.agg_database, @@ -133,7 +133,7 @@ def run_hourly_window( GROUP BY user_type """ - token_rows = query_rows(influxdb3_local, token_sql) + token_rows = influxdb3_local.query(token_sql) write_points( influxdb3_local, db_name=config.agg_database, @@ -169,7 +169,7 @@ def run_hourly_window( GROUP BY user_hash, project_id, parent_deployment, title """ - kpi_rows = query_rows(influxdb3_local, kpi_sql) + kpi_rows = influxdb3_local.query(kpi_sql) write_points( influxdb3_local, db_name=config.agg_database, @@ -197,7 +197,7 @@ def run_hourly_window( GROUP BY chat_id """ - chat_rows = query_rows(influxdb3_local, chat_sql) + chat_rows = influxdb3_local.query(chat_sql) write_points( influxdb3_local, db_name=config.agg_database, diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index 1b38bea..dd2b025 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -4,7 +4,7 @@ from typing import Any, Dict from .config import Config -from .influx import query_rows, write_points +from .influx import write_points from .window import Window @@ -62,9 +62,9 @@ def run_monthly_window( FROM ({_get_kpi_sub_table(in_window)}) """ - api_rows = query_rows(influxdb3_local, api_sql) - model_rows = query_rows(influxdb3_local, model_sql) - user_rows = query_rows(influxdb3_local, user_sql) + api_rows = influxdb3_local.query(api_sql) + model_rows = influxdb3_local.query(model_sql) + user_rows = influxdb3_local.query(user_sql) merged: Dict[str, Any] = {"time": start_s} if api_rows: From 431619f146c3db1cde6b2f08cc6319e89092ae86 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 13:04:36 +0000 Subject: [PATCH 26/56] feat: add InfluxDBClient class --- .../customized/influxdb_v3/plugin/__init__.py | 7 ++++- .../customized/influxdb_v3/plugin/config.py | 3 +- .../customized/influxdb_v3/plugin/influx.py | 14 +++++---- .../influxdb_v3/plugin/rollup_hourly.py | 30 +++++++++---------- .../influxdb_v3/plugin/rollup_monthly.py | 20 ++++++------- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index b0b6434..5f88a18 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -50,11 +50,16 @@ from typing_extensions import assert_never from .config import Config, Mode +from .influx import InfluxDBClient from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly -def process_scheduled_call(influxdb3_local, call_time: datetime, args=None): +def process_scheduled_call( + influxdb3_local: InfluxDBClient, + call_time: datetime, + args: dict | None = None, +): """ InfluxDB 3 scheduled plugin entrypoint. """ diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 962b738..c6dec92 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -65,7 +65,6 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": def get_windows(self, call_time: datetime) -> List[Window]: match self.mode: - case Mode.HOURLY: if self.start_time and self.end_time: if self.end_time <= self.start_time: @@ -80,7 +79,7 @@ def get_windows(self, call_time: datetime) -> List[Window]: case Mode.MONTHLY: this_month = _month_start(call_time) - prev_month = _month_start(this_month - timedelta(days=2)) + prev_month = _month_start(this_month - timedelta(minutes=1)) window = Window(start=prev_month, end=this_month) case _: diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx.py index 1515731..383a54c 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx.py +++ b/dashboards/customized/influxdb_v3/plugin/influx.py @@ -16,9 +16,15 @@ def int64_field(self, key: str, value: int) -> None: ... def float64_field(self, key: str, value: float) -> None: ... def bool_field(self, key: str, value: bool) -> None: ... + class InfluxDBClient: + def query(self, query: str) -> list[Dict[str, Any]]: ... + def write_to_db(self, db_name: str, line: LineBuilder) -> None: ... + def info(self, msg: str) -> None: ... + def error(self, msg: str) -> None: ... + def write_points( - influxdb3_local, + client: InfluxDBClient, *, db_name: str, table_name: str, @@ -66,12 +72,10 @@ def write_points( f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" ) - influxdb3_local.write_to_db(db_name, b) + client.write_to_db(db_name, b) written += 1 - influxdb3_local.info( - f"[{task_id}] wrote {written} points -> {db_name}.{table_name}" - ) + client.info(f"[{task_id}] wrote {written} points -> {db_name}.{table_name}") return written diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index ce267e3..8aa3d7c 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -3,12 +3,12 @@ from datetime import datetime from .config import Config -from .influx import write_points +from .influx import InfluxDBClient, write_points from .window import Window def run_hourly( - influxdb3_local, call_time: datetime, config: Config, *, task_id: str + client: InfluxDBClient, call_time: datetime, config: Config, *, task_id: str ) -> None: """ Reads: raw_table @@ -18,16 +18,16 @@ def run_hourly( windows = config.get_windows(call_time) for window in windows: - run_hourly_window(influxdb3_local, config, window, task_id) + run_hourly_window(client, config, window, task_id) def run_hourly_window( - influxdb3_local, config: Config, window: Window, task_id: str + client: InfluxDBClient, config: Config, window: Window, task_id: str ) -> None: start_s = window.start_s in_window = window.in_window_sql() - influxdb3_local.info( + client.info( f"[{task_id}] {config.window_hours}-hours rollup window: {window.display()}" ) @@ -52,9 +52,9 @@ def run_hourly_window( GROUP BY deployment, model, project_id, parent_deployment, language """ - stats_rows = influxdb3_local.query(stats_sql) + stats_rows = client.query(stats_sql) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_stats", rows=stats_rows, @@ -95,9 +95,9 @@ def run_hourly_window( GROUP BY title, topic, model """ - topic_rows = influxdb3_local.query(topic_sql) + topic_rows = client.query(topic_sql) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_topic_2", rows=topic_rows, @@ -133,9 +133,9 @@ def run_hourly_window( GROUP BY user_type """ - token_rows = influxdb3_local.query(token_sql) + token_rows = client.query(token_sql) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_topic", rows=token_rows, @@ -169,9 +169,9 @@ def run_hourly_window( GROUP BY user_hash, project_id, parent_deployment, title """ - kpi_rows = influxdb3_local.query(kpi_sql) + kpi_rows = client.query(kpi_sql) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_kpi", rows=kpi_rows, @@ -197,9 +197,9 @@ def run_hourly_window( GROUP BY chat_id """ - chat_rows = influxdb3_local.query(chat_sql) + chat_rows = client.query(chat_sql) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_chatid", rows=chat_rows, diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index dd2b025..0a07c3c 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -4,12 +4,12 @@ from typing import Any, Dict from .config import Config -from .influx import write_points +from .influx import InfluxDBClient, write_points from .window import Window def run_monthly( - influxdb3_local, call_time: datetime, config: Config, *, task_id: str + client: InfluxDBClient, call_time: datetime, config: Config, *, task_id: str ) -> None: """ Reads: default_agg_stats, default_agg_kpi @@ -18,18 +18,16 @@ def run_monthly( windows = config.get_windows(call_time) for window in windows: - run_monthly_window(influxdb3_local, config, window, task_id) + run_monthly_window(client, config, window, task_id) def run_monthly_window( - influxdb3_local, config: Config, window: Window, task_id: str + client: InfluxDBClient, config: Config, window: Window, task_id: str ) -> None: start_s = window.start_s in_window = window.in_window_sql() - influxdb3_local.info( - f"[{task_id}] monthly rollup window: {window.display()}" - ) + client.info(f"[{task_id}] monthly rollup window: {window.display()}") # API-level (project_id) rollups from default_agg_stats api_sql = f""" @@ -62,9 +60,9 @@ def run_monthly_window( FROM ({_get_kpi_sub_table(in_window)}) """ - api_rows = influxdb3_local.query(api_sql) - model_rows = influxdb3_local.query(model_sql) - user_rows = influxdb3_local.query(user_sql) + api_rows = client.query(api_sql) + model_rows = client.query(model_sql) + user_rows = client.query(user_sql) merged: Dict[str, Any] = {"time": start_s} if api_rows: @@ -75,7 +73,7 @@ def run_monthly_window( merged.update(user_rows[0]) write_points( - influxdb3_local, + client, db_name=config.agg_database, table_name="default_agg_month", rows=[merged], From 179561a01b571b0b20226e4aebf2cdd0641bc30e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 13:37:28 +0000 Subject: [PATCH 27/56] feat: support epoch-aligned bins --- .../customized/influxdb_v3/plugin/config.py | 33 ++++------- .../influxdb_v3/plugin/window_roller.py | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/window_roller.py diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index c6dec92..be2a639 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from enum import Enum from typing import Dict, List @@ -7,6 +7,7 @@ from .utils import parse_iso_date from .window import Window +from .window_roller import HourlyRoller, MonthlyRoller, roll_windows class Mode(Enum): @@ -64,30 +65,18 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": ) def get_windows(self, call_time: datetime) -> List[Window]: + call_time = call_time.astimezone(timezone.utc) + if self.end_time: + call_time = min(call_time, self.end_time) + match self.mode: case Mode.HOURLY: - if self.start_time and self.end_time: - if self.end_time <= self.start_time: - raise ValueError( - f"end_time must be > start_time (got {self.start_time} .. {self.end_time})" - ) - window = Window(start=self.start_time, end=self.end_time) - else: - end_time = call_time - start_time = end_time - timedelta(hours=self.window_hours) - window = Window(start=start_time, end=end_time) - + roller = HourlyRoller(window_hours=self.window_hours) case Mode.MONTHLY: - this_month = _month_start(call_time) - prev_month = _month_start(this_month - timedelta(minutes=1)) - window = Window(start=prev_month, end=this_month) - + roller = MonthlyRoller() case _: assert_never(self.mode) - return [window] - - -def _month_start(dt: datetime) -> datetime: - dt = dt.astimezone(timezone.utc) - return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + return roll_windows( + roller, start_time=self.start_time, call_time=call_time + ) diff --git a/dashboards/customized/influxdb_v3/plugin/window_roller.py b/dashboards/customized/influxdb_v3/plugin/window_roller.py new file mode 100644 index 0000000..afdbab0 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/window_roller.py @@ -0,0 +1,56 @@ +from datetime import datetime, timedelta +from typing import List, Protocol + +from .window import Window + + +class WindowRoller(Protocol): + def align(self, call_time: datetime) -> datetime: ... + def prev(self, time: datetime) -> datetime: ... + + +class HourlyRoller: + def __init__(self, window_hours: int): + if 24 % window_hours: + raise ValueError("window_hours must divide 24 evenly") + self.window_hours = window_hours + + def align(self, call_time: datetime) -> datetime: + hour = call_time.hour - (call_time.hour % self.window_hours) + return call_time.replace(hour=hour, minute=0, second=0, microsecond=0) + + def prev(self, time: datetime) -> datetime: + return time - timedelta(hours=self.window_hours) + + +class MonthlyRoller: + def align(self, call_time: datetime) -> datetime: + return self._month_start(call_time) + + def prev(self, time: datetime) -> datetime: + return self._month_start(time - timedelta(minutes=1)) + + @staticmethod + def _month_start(dt: datetime) -> datetime: + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + +def roll_windows( + roller: WindowRoller, *, start_time: datetime | None, call_time: datetime +) -> List[Window]: + + end = roller.align(call_time) + start = roller.prev(end) + + if start_time is None: + return [Window(start=start, end=end)] + + ret: List[Window] = [] + while start_time <= start: + ret.append(Window(start=start, end=end)) + end = start + start = roller.prev(end) + + ret = ret[::-1] + + return ret From 497daa407c6e4caa2a813de7513bfa410ad9742f Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 13:38:39 +0000 Subject: [PATCH 28/56] chore: minor refactoring --- dashboards/customized/influxdb_v3/plugin/config.py | 2 +- dashboards/customized/influxdb_v3/plugin/{utils.py => dates.py} | 2 -- dashboards/customized/influxdb_v3/plugin/influx.py | 2 +- dashboards/customized/influxdb_v3/plugin/window.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) rename dashboards/customized/influxdb_v3/plugin/{utils.py => dates.py} (95%) diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index be2a639..910916b 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -5,7 +5,7 @@ from typing_extensions import assert_never -from .utils import parse_iso_date +from .dates import parse_iso_date from .window import Window from .window_roller import HourlyRoller, MonthlyRoller, roll_windows diff --git a/dashboards/customized/influxdb_v3/plugin/utils.py b/dashboards/customized/influxdb_v3/plugin/dates.py similarity index 95% rename from dashboards/customized/influxdb_v3/plugin/utils.py rename to dashboards/customized/influxdb_v3/plugin/dates.py index 4f0127b..85d21e1 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils.py +++ b/dashboards/customized/influxdb_v3/plugin/dates.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from datetime import datetime, timezone diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx.py index 383a54c..6203ad4 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx.py +++ b/dashboards/customized/influxdb_v3/plugin/influx.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict, Iterable -from .utils import parse_iso_date +from .dates import parse_iso_date if TYPE_CHECKING: # LineBuilder is available in the processing runtime (as used by official plugins). diff --git a/dashboards/customized/influxdb_v3/plugin/window.py b/dashboards/customized/influxdb_v3/plugin/window.py index bd95816..cd60f52 100644 --- a/dashboards/customized/influxdb_v3/plugin/window.py +++ b/dashboards/customized/influxdb_v3/plugin/window.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime -from .utils import to_iso_date +from .dates import to_iso_date @dataclass From 8c27fc6ce1cf24c6fdacc28f7321b3d85ec3205e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 14:08:37 +0000 Subject: [PATCH 29/56] feat: add mocks for InfluxDB --- .../customized/influxdb_v3/plugin/__init__.py | 16 +++++ .../customized/influxdb_v3/plugin/influx.py | 39 ++++++----- .../customized/influxdb_v3/plugin/mocks.py | 67 +++++++++++++++++++ 3 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/mocks.py diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 5f88a18..86b244e 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -44,13 +44,17 @@ } """ +import sys +import tomllib import uuid from datetime import datetime, timezone +from pathlib import Path from typing_extensions import assert_never from .config import Config, Mode from .influx import InfluxDBClient +from .mocks import MockInfluxDBClient from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly @@ -86,3 +90,15 @@ def process_scheduled_call( except Exception as e: influxdb3_local.error(f"[{task_id}] failed: {e}") raise + + +if __name__ == "__main__": + config_file = sys.argv[1] if len(sys.argv) > 1 else "config.toml" + args = tomllib.loads(Path(config_file).read_text()) + + # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. + process_scheduled_call( + influxdb3_local=MockInfluxDBClient(), + call_time=datetime.now(timezone.utc), + args=args, + ) diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx.py index 6203ad4..2b3eecc 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx.py +++ b/dashboards/customized/influxdb_v3/plugin/influx.py @@ -1,26 +1,29 @@ from collections.abc import Sequence from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, Iterable +from typing import Any, Dict, Iterable, Protocol, runtime_checkable from .dates import parse_iso_date +from .mocks import create_line_builder -if TYPE_CHECKING: - # LineBuilder is available in the processing runtime (as used by official plugins). - # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data - class LineBuilder: - def __init__(self, measurement: str) -> None: ... - def time_ns(self, value: int) -> None: ... - def tag(self, key: str, value: str) -> None: ... - def string_field(self, key: str, value: str) -> None: ... - def int64_field(self, key: str, value: int) -> None: ... - def float64_field(self, key: str, value: float) -> None: ... - def bool_field(self, key: str, value: bool) -> None: ... - class InfluxDBClient: - def query(self, query: str) -> list[Dict[str, Any]]: ... - def write_to_db(self, db_name: str, line: LineBuilder) -> None: ... - def info(self, msg: str) -> None: ... - def error(self, msg: str) -> None: ... +@runtime_checkable +class LineBuilderProtocol(Protocol): + def __init__(self, measurement: str) -> None: ... + def time_ns(self, value: int) -> None: ... + def tag(self, key: str, value: str) -> None: ... + def string_field(self, key: str, value: str) -> None: ... + def int64_field(self, key: str, value: int) -> None: ... + def float64_field(self, key: str, value: float) -> None: ... + def bool_field(self, key: str, value: bool) -> None: ... + def build(self) -> str: ... + + +@runtime_checkable +class InfluxDBClient(Protocol): + def query(self, query: str) -> list[Dict[str, Any]]: ... + def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: ... + def info(self, msg: str) -> None: ... + def error(self, msg: str) -> None: ... def write_points( @@ -41,7 +44,7 @@ def write_points( written = 0 for r in rows: - b = LineBuilder(table_name) + b = create_line_builder(client, table_name) t = r.get(time_col) if isinstance(t, int): diff --git a/dashboards/customized/influxdb_v3/plugin/mocks.py b/dashboards/customized/influxdb_v3/plugin/mocks.py new file mode 100644 index 0000000..fc9277b --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/mocks.py @@ -0,0 +1,67 @@ +from typing import Any, Dict + +from .influx import InfluxDBClient, LineBuilderProtocol + + +class MockInfluxDBClient(InfluxDBClient): + def query(self, query: str) -> list[Dict[str, Any]]: + print(f"MockInfluxDBClient.query:\n{query}\n") + return [] + + def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: + print( + f"MockInfluxDBClient.write_to_db:\ndb_name={db_name}\nline={line.build()}" + ) + + def info(self, msg: str) -> None: + print(f"INFO: {msg}") + + def error(self, msg: str) -> None: + print(f"ERROR: {msg}") + + +class MockLineBuilder(LineBuilderProtocol): + _measurement: str + _time: int + _tags: Dict[str, str] + _fields: Dict[str, Any] + + def __init__(self, measurement: str) -> None: + self._measurement = measurement + self._time = 0 + self._tags = {} + self._fields = {} + + def time_ns(self, value: int) -> None: + self._time = value + + def tag(self, key: str, value: str) -> None: + self._tags[key] = value + + def string_field(self, key: str, value: str) -> None: + self._fields[key] = value + + def int64_field(self, key: str, value: int) -> None: + self._fields[key] = value + + def float64_field(self, key: str, value: float) -> None: + self._fields[key] = value + + def bool_field(self, key: str, value: bool) -> None: + self._fields[key] = value + + def build(self) -> str: + return f"MockLine(measurement={self._measurement!r}, time_ns={self._time}, tags={self._tags}, fields={self._fields})" + + +def create_line_builder( + client: InfluxDBClient, table_name: str +) -> LineBuilderProtocol: + if isinstance(client, MockInfluxDBClient): + return MockLineBuilder(table_name) + else: + # LineBuilder is available in the runtime: + # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data + # The actual LineBuilder code: + # https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L508-L613 + return LineBuilder(table_name) # type: ignore # noqa: F821 From 92530c535d5a8bb8d2babc962e4b05edfc15d99e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 14:31:21 +0000 Subject: [PATCH 30/56] chore: moving code around --- .../customized/influxdb_v3/plugin/__init__.py | 20 ++----------- .../customized/influxdb_v3/plugin/__main__.py | 29 +++++++++++++++++++ .../plugin/{influx.py => influx/client.py} | 27 +++-------------- .../influxdb_v3/plugin/{ => influx}/mocks.py | 17 ++++++----- .../influxdb_v3/plugin/influx/types.py | 21 ++++++++++++++ .../influxdb_v3/plugin/rollup_hourly.py | 12 ++++---- .../influxdb_v3/plugin/rollup_monthly.py | 10 ++++--- 7 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/__main__.py rename dashboards/customized/influxdb_v3/plugin/{influx.py => influx/client.py} (65%) rename dashboards/customized/influxdb_v3/plugin/{ => influx}/mocks.py (81%) create mode 100644 dashboards/customized/influxdb_v3/plugin/influx/types.py diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 86b244e..489ef66 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -44,17 +44,13 @@ } """ -import sys -import tomllib import uuid from datetime import datetime, timezone -from pathlib import Path from typing_extensions import assert_never from .config import Config, Mode -from .influx import InfluxDBClient -from .mocks import MockInfluxDBClient +from .influx.client import InfluxDBClient from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly @@ -73,7 +69,7 @@ def process_scheduled_call( call_time = call_time.replace(tzinfo=timezone.utc) influxdb3_local.info( - f"[{task_id}] scheduled call mode={config.mode} call_time={call_time.isoformat()} args={args}" + f"[{task_id}] scheduled call mode={config.mode.value} call_time={call_time.isoformat()} args={args}" ) try: @@ -90,15 +86,3 @@ def process_scheduled_call( except Exception as e: influxdb3_local.error(f"[{task_id}] failed: {e}") raise - - -if __name__ == "__main__": - config_file = sys.argv[1] if len(sys.argv) > 1 else "config.toml" - args = tomllib.loads(Path(config_file).read_text()) - - # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. - process_scheduled_call( - influxdb3_local=MockInfluxDBClient(), - call_time=datetime.now(timezone.utc), - args=args, - ) diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py new file mode 100644 index 0000000..a508edd --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import sys +import tomllib +from datetime import datetime, timezone +from pathlib import Path + +from . import process_scheduled_call +from .influx.mocks import MockInfluxDBClient + + +def main() -> None: + config_file = sys.argv[1] if len(sys.argv) > 1 else "config.toml" + config_file = Path(config_file) + if not config_file.exists(): + args = {} + else: + args = tomllib.loads(Path(config_file).read_text()) + + # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. + process_scheduled_call( + influxdb3_local=MockInfluxDBClient(), + call_time=datetime.now(timezone.utc), + args=args, + ) + + +if __name__ == "__main__": + main() diff --git a/dashboards/customized/influxdb_v3/plugin/influx.py b/dashboards/customized/influxdb_v3/plugin/influx/client.py similarity index 65% rename from dashboards/customized/influxdb_v3/plugin/influx.py rename to dashboards/customized/influxdb_v3/plugin/influx/client.py index 2b3eecc..f901234 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client.py @@ -1,29 +1,10 @@ from collections.abc import Sequence from datetime import datetime, timezone -from typing import Any, Dict, Iterable, Protocol, runtime_checkable +from typing import Any, Dict, Iterable -from .dates import parse_iso_date +from ..dates import parse_iso_date from .mocks import create_line_builder - - -@runtime_checkable -class LineBuilderProtocol(Protocol): - def __init__(self, measurement: str) -> None: ... - def time_ns(self, value: int) -> None: ... - def tag(self, key: str, value: str) -> None: ... - def string_field(self, key: str, value: str) -> None: ... - def int64_field(self, key: str, value: int) -> None: ... - def float64_field(self, key: str, value: float) -> None: ... - def bool_field(self, key: str, value: bool) -> None: ... - def build(self) -> str: ... - - -@runtime_checkable -class InfluxDBClient(Protocol): - def query(self, query: str) -> list[Dict[str, Any]]: ... - def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: ... - def info(self, msg: str) -> None: ... - def error(self, msg: str) -> None: ... +from .types import InfluxDBClient def write_points( @@ -78,7 +59,7 @@ def write_points( client.write_to_db(db_name, b) written += 1 - client.info(f"[{task_id}] wrote {written} points -> {db_name}.{table_name}") + client.info(f"[{task_id}] wrote {written} points to {db_name}.{table_name}") return written diff --git a/dashboards/customized/influxdb_v3/plugin/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py similarity index 81% rename from dashboards/customized/influxdb_v3/plugin/mocks.py rename to dashboards/customized/influxdb_v3/plugin/influx/mocks.py index fc9277b..f163160 100644 --- a/dashboards/customized/influxdb_v3/plugin/mocks.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py @@ -1,23 +1,26 @@ from typing import Any, Dict -from .influx import InfluxDBClient, LineBuilderProtocol +from .types import InfluxDBClient, LineBuilderProtocol class MockInfluxDBClient(InfluxDBClient): def query(self, query: str) -> list[Dict[str, Any]]: - print(f"MockInfluxDBClient.query:\n{query}\n") + query = _line_prefix(" | ", query.strip()) + print(f"[INFLUX QUERY]\n{query}") return [] def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: - print( - f"MockInfluxDBClient.write_to_db:\ndb_name={db_name}\nline={line.build()}" - ) + print(f"[INFLUX WRITE](db_name={db_name})\n{line.build()}") def info(self, msg: str) -> None: - print(f"INFO: {msg}") + print(f"[PLUGIN INFO] {msg}") def error(self, msg: str) -> None: - print(f"ERROR: {msg}") + print(f"[PLUGIN ERROR] {msg}") + + +def _line_prefix(prefix: str, text: str) -> str: + return "\n".join(prefix + line for line in text.splitlines()) class MockLineBuilder(LineBuilderProtocol): diff --git a/dashboards/customized/influxdb_v3/plugin/influx/types.py b/dashboards/customized/influxdb_v3/plugin/influx/types.py new file mode 100644 index 0000000..885f474 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/influx/types.py @@ -0,0 +1,21 @@ +from typing import Any, Dict, Protocol, runtime_checkable + + +@runtime_checkable +class LineBuilderProtocol(Protocol): + def __init__(self, measurement: str) -> None: ... + def time_ns(self, value: int) -> None: ... + def tag(self, key: str, value: str) -> None: ... + def string_field(self, key: str, value: str) -> None: ... + def int64_field(self, key: str, value: int) -> None: ... + def float64_field(self, key: str, value: float) -> None: ... + def bool_field(self, key: str, value: bool) -> None: ... + def build(self) -> str: ... + + +@runtime_checkable +class InfluxDBClient(Protocol): + def query(self, query: str) -> list[Dict[str, Any]]: ... + def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: ... + def info(self, msg: str) -> None: ... + def error(self, msg: str) -> None: ... diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 8aa3d7c..d79796b 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -3,7 +3,7 @@ from datetime import datetime from .config import Config -from .influx import InfluxDBClient, write_points +from .influx.client import InfluxDBClient, write_points from .window import Window @@ -17,7 +17,11 @@ def run_hourly( windows = config.get_windows(call_time) - for window in windows: + n = len(windows) + for idx, window in enumerate(windows, start=1): + client.info( + f"[{task_id}] [{idx}/{n}] {config.window_hours}-hours rollup window: {window.display()}" + ) run_hourly_window(client, config, window, task_id) @@ -27,10 +31,6 @@ def run_hourly_window( start_s = window.start_s in_window = window.in_window_sql() - client.info( - f"[{task_id}] {config.window_hours}-hours rollup window: {window.display()}" - ) - # 1) default_agg_stats stats_sql = f""" SELECT diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index 0a07c3c..950b7e5 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -4,7 +4,7 @@ from typing import Any, Dict from .config import Config -from .influx import InfluxDBClient, write_points +from .influx.client import InfluxDBClient, write_points from .window import Window @@ -17,7 +17,11 @@ def run_monthly( """ windows = config.get_windows(call_time) - for window in windows: + n = len(windows) + for idx, window in enumerate(windows, start=1): + client.info( + f"[{task_id}] [{idx}/{n}] monthly rollup window: {window.display()}" + ) run_monthly_window(client, config, window, task_id) @@ -27,8 +31,6 @@ def run_monthly_window( start_s = window.start_s in_window = window.in_window_sql() - client.info(f"[{task_id}] monthly rollup window: {window.display()}") - # API-level (project_id) rollups from default_agg_stats api_sql = f""" SELECT From 48b2d07dd2f5a7226a829d61a607004fa5822470 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 15:10:41 +0000 Subject: [PATCH 31/56] feat: add more validations --- dashboards/customized/influxdb_v3/.gitignore | 3 +++ .../customized/influxdb_v3/plugin/__main__.py | 9 ++++----- .../customized/influxdb_v3/plugin/config.py | 16 +++++++++------- 3 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/.gitignore diff --git a/dashboards/customized/influxdb_v3/.gitignore b/dashboards/customized/influxdb_v3/.gitignore new file mode 100644 index 0000000..9f7ea6d --- /dev/null +++ b/dashboards/customized/influxdb_v3/.gitignore @@ -0,0 +1,3 @@ +*.txt +*.log +*.toml diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index a508edd..d9a6324 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -10,12 +10,11 @@ def main() -> None: - config_file = sys.argv[1] if len(sys.argv) > 1 else "config.toml" - config_file = Path(config_file) - if not config_file.exists(): - args = {} - else: + if len(sys.argv) > 1: + config_file = Path(sys.argv[1]) args = tomllib.loads(Path(config_file).read_text()) + else: + args = None # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. process_scheduled_call( diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 910916b..53e48cd 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -30,17 +30,17 @@ class Config: def parse(cls, d: Dict[str, str] | None) -> "Config": d = d or {} - mode_s = d.get("mode") or "hourly" + mode_s = d.pop("mode", None) or "hourly" try: mode = Mode(mode_s.strip().lower()) except Exception as e: raise ValueError(f"Unsupported mode: {mode_s!r}") from e - agg_database = d.get("agg_database") or "analytics_agg" - raw_table = d.get("raw_table") or "analytics" + agg_database = d.pop("agg_database", None) or "analytics_agg" + raw_table = d.pop("raw_table", None) or "analytics" - start_arg = d.get("start_time") - end_arg = d.get("end_time") + start_arg = d.pop("start_time", None) + end_arg = d.pop("end_time", None) start_time: datetime | None = ( parse_iso_date("start_time", start_arg) if start_arg else None ) @@ -48,13 +48,15 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": parse_iso_date("end_time", end_arg) if end_arg else None ) - window_hours = int(d.get("window_hours") or 6) - + window_hours = int(d.pop("window_hours", None) or 6) if 24 % window_hours: raise ValueError( f"window_hours must divide 24 evenly (got {window_hours})" ) + if d: + raise ValueError(f"Unexpected config keys: {', '.join(d.keys())}") + return cls( mode=mode, agg_database=agg_database, From 85655cbcb9e99d2e2a0bee481343b11f7270e29c Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 17:20:20 +0000 Subject: [PATCH 32/56] feat: make actual queries form DryRunInfluxDBClient --- dashboards/customized/influxdb_v3/.gitignore | 1 + .../customized/influxdb_v3/plugin/__main__.py | 17 ++++++- .../customized/influxdb_v3/plugin/config.py | 10 ++++ .../influxdb_v3/plugin/influx/mocks.py | 46 +++++++++++++++++-- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/dashboards/customized/influxdb_v3/.gitignore b/dashboards/customized/influxdb_v3/.gitignore index 9f7ea6d..e33f88a 100644 --- a/dashboards/customized/influxdb_v3/.gitignore +++ b/dashboards/customized/influxdb_v3/.gitignore @@ -1,3 +1,4 @@ *.txt *.log *.toml +*.sh diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index d9a6324..0965893 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -1,12 +1,20 @@ from __future__ import annotations +import os import sys import tomllib from datetime import datetime, timezone from pathlib import Path from . import process_scheduled_call -from .influx.mocks import MockInfluxDBClient +from .config import Config +from .influx.mocks import DryRunInfluxDBClient + + +def _get_env(name: str) -> str: + if (v := os.getenv(name)) is None: + raise ValueError(f"Env variable {name!r} is unset") + return v def main() -> None: @@ -16,9 +24,14 @@ def main() -> None: else: args = None + url = _get_env("INFLUX_URL") + token = _get_env("INFLUX_API_TOKEN") + database = Config.parse(args or {}).input_database + client = DryRunInfluxDBClient(url=url, token=token, database=database) + # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. process_scheduled_call( - influxdb3_local=MockInfluxDBClient(), + influxdb3_local=client, call_time=datetime.now(timezone.utc), args=args, ) diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 53e48cd..734ed02 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -66,6 +66,16 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": window_hours=window_hours, ) + @property + def input_database(self) -> str: + match self.mode: + case Mode.HOURLY: + return "default" + case Mode.MONTHLY: + return self.agg_database + case _: + assert_never(self.mode) + def get_windows(self, call_time: datetime) -> List[Window]: call_time = call_time.astimezone(timezone.utc) if self.end_time: diff --git a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py index f163160..3a211af 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py @@ -1,13 +1,49 @@ +import json from typing import Any, Dict +from urllib.request import Request, urlopen from .types import InfluxDBClient, LineBuilderProtocol -class MockInfluxDBClient(InfluxDBClient): +class DryRunInfluxDBClient(InfluxDBClient): + _influxdb_url: str + _influxdb_token: str + + _database: str + + def __init__(self, *, url: str, token: str, database: str): + self._database = database + self._influxdb_url = url + self._influxdb_token = token + def query(self, query: str) -> list[Dict[str, Any]]: - query = _line_prefix(" | ", query.strip()) - print(f"[INFLUX QUERY]\n{query}") - return [] + query = query.strip() + print(f"[INFLUX QUERY]\n{_line_prefix(" | ", query)}") + + endpoint = f"{self._influxdb_url}/api/v3/query_sql" + + payload = json.dumps( + {"formats": "json", "db": self._database, "q": query} + ).encode("utf-8") + + req = Request( + endpoint, + data=payload, + method="POST", + headers={ + "Authorization": f"Token {self._influxdb_token}", + "Content-Type": "application/json", + }, + ) + + with urlopen(req) as resp: + rows = json.load(resp) + + prefix = json.dumps(rows[:3], indent=2) + prefix = _line_prefix(" | ", prefix.strip()) + print(f"[INFLUX QUERY RESPONSE](rows={len(rows)}):\n{prefix}") + + return rows def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: print(f"[INFLUX WRITE](db_name={db_name})\n{line.build()}") @@ -60,7 +96,7 @@ def build(self) -> str: def create_line_builder( client: InfluxDBClient, table_name: str ) -> LineBuilderProtocol: - if isinstance(client, MockInfluxDBClient): + if isinstance(client, DryRunInfluxDBClient): return MockLineBuilder(table_name) else: # LineBuilder is available in the runtime: From 552b6896b9598131d679120c5e53ef651449c85b Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 17:30:56 +0000 Subject: [PATCH 33/56] feat: borrow actual implementation of the LineBuilder --- .../customized/influxdb_v3/plugin/__main__.py | 4 +- .../influxdb_v3/plugin/influx/client.py | 20 ++- .../influxdb_v3/plugin/influx/line_builder.py | 157 ++++++++++++++++++ .../influxdb_v3/plugin/influx/mocks.py | 61 +------ .../influxdb_v3/plugin/influx/types.py | 14 +- 5 files changed, 192 insertions(+), 64 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/influx/line_builder.py diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index 0965893..c62650b 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -8,7 +8,7 @@ from . import process_scheduled_call from .config import Config -from .influx.mocks import DryRunInfluxDBClient +from .influx.mocks import ReadOnlyInfluxDBClient def _get_env(name: str) -> str: @@ -27,7 +27,7 @@ def main() -> None: url = _get_env("INFLUX_URL") token = _get_env("INFLUX_API_TOKEN") database = Config.parse(args or {}).input_database - client = DryRunInfluxDBClient(url=url, token=token, database=database) + client = ReadOnlyInfluxDBClient(url=url, token=token, database=database) # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. process_scheduled_call( diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client.py b/dashboards/customized/influxdb_v3/plugin/influx/client.py index f901234..a203734 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client.py @@ -3,8 +3,9 @@ from typing import Any, Dict, Iterable from ..dates import parse_iso_date -from .mocks import create_line_builder -from .types import InfluxDBClient +from .line_builder import LineBuilder as LineBuilderImpl +from .mocks import ReadOnlyInfluxDBClient +from .types import InfluxDBClient, LineBuilderProtocol def write_points( @@ -25,7 +26,7 @@ def write_points( written = 0 for r in rows: - b = create_line_builder(client, table_name) + b = _create_line_builder(client, table_name) t = r.get(time_col) if isinstance(t, int): @@ -65,3 +66,16 @@ def write_points( def _ns(dt: datetime) -> int: return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) + + +def _create_line_builder( + client: InfluxDBClient, table_name: str +) -> LineBuilderProtocol: + if isinstance(client, ReadOnlyInfluxDBClient): + return LineBuilderImpl(table_name) + else: + # LineBuilder is available in the runtime: + # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data + # The actual LineBuilder code: + # https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L508-L613 + return LineBuilder(table_name) # type: ignore # noqa: F821 diff --git a/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py b/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py new file mode 100644 index 0000000..cfa8750 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py @@ -0,0 +1,157 @@ +""" +Copied as is from the actual LineBuilder: +https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L488-L613 +""" + +from collections import OrderedDict +from typing import Optional + + +class InfluxDBError(Exception): + """Base exception for InfluxDB-related errors""" + + pass + + +class InvalidMeasurementError(InfluxDBError): + """Raised when measurement name is invalid""" + + pass + + +class InvalidKeyError(InfluxDBError): + """Raised when a tag or field key is invalid""" + + pass + + +class InvalidLineError(InfluxDBError): + """Raised when a line protocol string is invalid""" + + pass + + +class LineBuilder: + def __init__(self, measurement: str): + if " " in measurement: + raise InvalidMeasurementError( + "Measurement name cannot contain spaces" + ) + self.measurement = measurement + self.tags: OrderedDict[str, str] = OrderedDict() + self.fields: OrderedDict[str, str] = OrderedDict() + self._timestamp_ns: Optional[int] = None + + def _validate_key(self, key: str, key_type: str) -> None: + """Validate that a key does not contain spaces, commas, or equals signs.""" + if not key: + raise InvalidKeyError(f"{key_type} key cannot be empty") + if " " in key: + raise InvalidKeyError( + f"{key_type} key '{key}' cannot contain spaces" + ) + if "," in key: + raise InvalidKeyError( + f"{key_type} key '{key}' cannot contain commas" + ) + if "=" in key: + raise InvalidKeyError( + f"{key_type} key '{key}' cannot contain equals signs" + ) + + def _escape_measurement(self, value: str) -> str: + """Escape characters in measurement names according to line protocol.""" + return value.replace(",", "\\,").replace(" ", "\\ ") + + def _escape_tag_value(self, value: str) -> str: + """Escape characters in tag values according to line protocol.""" + return ( + value.replace("\\", "\\\\") + .replace(",", "\\,") + .replace("=", "\\=") + .replace(" ", "\\ ") + ) + + def _escape_field_key(self, value: str) -> str: + """Escape characters in field keys according to line protocol.""" + return ( + value.replace("\\", "\\\\") + .replace(",", "\\,") + .replace("=", "\\=") + .replace(" ", "\\ ") + ) + + def tag(self, key: str, value: str) -> "LineBuilder": + """Add a tag to the line protocol.""" + self._validate_key(key, "tag") + self.tags[key] = str(value) + return self + + def uint64_field(self, key: str, value: int) -> "LineBuilder": + """Add an unsigned integer field to the line protocol.""" + self._validate_key(key, "field") + if value < 0: + raise ValueError(f"uint64 field '{key}' cannot be negative") + self.fields[key] = f"{value}u" + return self + + def int64_field(self, key: str, value: int) -> "LineBuilder": + """Add an integer field to the line protocol.""" + self._validate_key(key, "field") + self.fields[key] = f"{value}i" + return self + + def float64_field(self, key: str, value: float) -> "LineBuilder": + """Add a float field to the line protocol.""" + self._validate_key(key, "field") + # Check if value has no decimal component + self.fields[key] = f"{int(value)}.0" if value % 1 == 0 else str(value) + return self + + def string_field(self, key: str, value: str) -> "LineBuilder": + """Add a string field to the line protocol.""" + self._validate_key(key, "field") + # Escape quotes and backslashes in string values + escaped_value = value.replace("\\", "\\\\").replace('"', '\\"') + self.fields[key] = f'"{escaped_value}"' + return self + + def bool_field(self, key: str, value: bool) -> "LineBuilder": + """Add a boolean field to the line protocol.""" + self._validate_key(key, "field") + self.fields[key] = "t" if value else "f" + return self + + def time_ns(self, timestamp_ns: int) -> "LineBuilder": + """Set the timestamp in nanoseconds.""" + self._timestamp_ns = timestamp_ns + return self + + def build(self) -> str: + """Build the line protocol string.""" + # Start with measurement name (escape commas and spaces) + line = self._escape_measurement(self.measurement) + + # Add tags if present + if self.tags: + tags_str = ",".join( + f"{key}={self._escape_tag_value(value)}" + for key, value in self.tags.items() + ) + line += f",{tags_str}" + + # Add fields (required) + if not self.fields: + raise InvalidLineError(f"At least one field is required: {line}") + + fields_str = ",".join( + f"{self._escape_field_key(key)}={value}" + for key, value in self.fields.items() + ) + line += f" {fields_str}" + + # Add timestamp if present + if self._timestamp_ns is not None: + line += f" {self._timestamp_ns}" + + return line diff --git a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py index 3a211af..b5809cb 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py @@ -5,7 +5,7 @@ from .types import InfluxDBClient, LineBuilderProtocol -class DryRunInfluxDBClient(InfluxDBClient): +class ReadOnlyInfluxDBClient(InfluxDBClient): _influxdb_url: str _influxdb_token: str @@ -17,8 +17,7 @@ def __init__(self, *, url: str, token: str, database: str): self._influxdb_token = token def query(self, query: str) -> list[Dict[str, Any]]: - query = query.strip() - print(f"[INFLUX QUERY]\n{_line_prefix(" | ", query)}") + print(f"[INFLUX QUERY]\n{_prettify(query)}") endpoint = f"{self._influxdb_url}/api/v3/query_sql" @@ -40,8 +39,9 @@ def query(self, query: str) -> list[Dict[str, Any]]: rows = json.load(resp) prefix = json.dumps(rows[:3], indent=2) - prefix = _line_prefix(" | ", prefix.strip()) - print(f"[INFLUX QUERY RESPONSE](rows={len(rows)}):\n{prefix}") + print( + f"[INFLUX QUERY RESPONSE](rows={len(rows)}):\n{_prettify(prefix)}" + ) return rows @@ -55,52 +55,9 @@ def error(self, msg: str) -> None: print(f"[PLUGIN ERROR] {msg}") -def _line_prefix(prefix: str, text: str) -> str: - return "\n".join(prefix + line for line in text.splitlines()) - - -class MockLineBuilder(LineBuilderProtocol): - _measurement: str - _time: int - _tags: Dict[str, str] - _fields: Dict[str, Any] - - def __init__(self, measurement: str) -> None: - self._measurement = measurement - self._time = 0 - self._tags = {} - self._fields = {} - - def time_ns(self, value: int) -> None: - self._time = value - - def tag(self, key: str, value: str) -> None: - self._tags[key] = value +def _prettify(text: str) -> str: + return _line_prefix(" | ", text.strip()) - def string_field(self, key: str, value: str) -> None: - self._fields[key] = value - def int64_field(self, key: str, value: int) -> None: - self._fields[key] = value - - def float64_field(self, key: str, value: float) -> None: - self._fields[key] = value - - def bool_field(self, key: str, value: bool) -> None: - self._fields[key] = value - - def build(self) -> str: - return f"MockLine(measurement={self._measurement!r}, time_ns={self._time}, tags={self._tags}, fields={self._fields})" - - -def create_line_builder( - client: InfluxDBClient, table_name: str -) -> LineBuilderProtocol: - if isinstance(client, DryRunInfluxDBClient): - return MockLineBuilder(table_name) - else: - # LineBuilder is available in the runtime: - # https://docs.influxdata.com/influxdb3/enterprise/plugins/extend-plugin/#write-data - # The actual LineBuilder code: - # https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L508-L613 - return LineBuilder(table_name) # type: ignore # noqa: F821 +def _line_prefix(prefix: str, text: str) -> str: + return "\n".join(prefix + line for line in text.splitlines()) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/types.py b/dashboards/customized/influxdb_v3/plugin/influx/types.py index 885f474..d4ac234 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/types.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/types.py @@ -1,15 +1,15 @@ -from typing import Any, Dict, Protocol, runtime_checkable +from typing import Any, Dict, Protocol, Self, runtime_checkable @runtime_checkable class LineBuilderProtocol(Protocol): def __init__(self, measurement: str) -> None: ... - def time_ns(self, value: int) -> None: ... - def tag(self, key: str, value: str) -> None: ... - def string_field(self, key: str, value: str) -> None: ... - def int64_field(self, key: str, value: int) -> None: ... - def float64_field(self, key: str, value: float) -> None: ... - def bool_field(self, key: str, value: bool) -> None: ... + def time_ns(self, timestamp_ns: int) -> Self: ... + def tag(self, key: str, value: str) -> Self: ... + def string_field(self, key: str, value: str) -> Self: ... + def int64_field(self, key: str, value: int) -> Self: ... + def float64_field(self, key: str, value: float) -> Self: ... + def bool_field(self, key: str, value: bool) -> Self: ... def build(self) -> str: ... From 9266fde09debf084c1d02d9828d9b1e07bbdc221 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 5 Mar 2026 18:08:45 +0000 Subject: [PATCH 34/56] feat: add validation for datapoints --- .../influxdb_v3/plugin/influx/client.py | 82 +++++++++++-------- .../influxdb_v3/plugin/influx/mocks.py | 8 +- .../influxdb_v3/plugin/influx/types.py | 4 + .../influxdb_v3/plugin/rollup_hourly.py | 47 +++++++---- .../influxdb_v3/plugin/rollup_monthly.py | 4 +- 5 files changed, 92 insertions(+), 53 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client.py b/dashboards/customized/influxdb_v3/plugin/influx/client.py index a203734..f9e3ef1 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client.py @@ -1,6 +1,6 @@ -from collections.abc import Sequence +import json from datetime import datetime, timezone -from typing import Any, Dict, Iterable +from typing import Any, Dict, List, Tuple from ..dates import parse_iso_date from .line_builder import LineBuilder as LineBuilderImpl @@ -11,29 +11,30 @@ def write_points( client: InfluxDBClient, *, - db_name: str, - table_name: str, - rows: Iterable[Dict[str, Any]], + db: str, + table: str, + rows: List[Dict[str, Any]], time_col: str, - tag_cols: Sequence[str], - field_cols: Sequence[str], + tag_cols: Tuple[str, ...], + field_cols: Tuple[str, ...], task_id: str, ) -> int: """ Write each row as a point into db_name.table_name. - time_col can be RFC3339 string, datetime, or ns int. + time_col can be ISO string, datetime, or ns int. """ written = 0 - for r in rows: - b = _create_line_builder(client, table_name) + for r_orig in rows: + r = r_orig.copy() - t = r.get(time_col) - if isinstance(t, int): - b.time_ns(t) - elif isinstance(t, datetime): - b.time_ns(_ns(t)) - elif isinstance(t, str): + def r_json(): + return json.dumps(r_orig) + + b = _create_line_builder(client, table) + + t = r.pop(time_col, None) + if isinstance(t, str): b.time_ns(_ns(parse_iso_date(f"{time_col!r} column", t))) else: raise ValueError( @@ -41,26 +42,43 @@ def write_points( ) for k in tag_cols: - if (v := r.get(k)) is not None: - b.tag(k, str(v)) + if (v := r.pop(k, None)) is None: + raise ValueError( + f"Expected to find a tag column {k!r}, but it's missing from the given row: {r_json()}" + ) + if isinstance(v, str): + b.tag(k, v) + else: + raise ValueError( + f"Unexpected tag value in the column '{k!r}': {v} of type {type(v)}, but tags are expected to always be strings." + ) for k in field_cols: - if (v := r.get(k)) is not None: - if isinstance(v, bool): - b.bool_field(k, v) - elif isinstance(v, int): - b.int64_field(k, v) - elif isinstance(v, float): - b.float64_field(k, v) - else: - raise ValueError( - f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" - ) - - client.write_to_db(db_name, b) + if (v := r.pop(k, None)) is None: + raise ValueError( + f"Expected to find a field column {k!r}, but it's missing from the given row: {r_json()}" + ) + + if isinstance(v, bool): + b.bool_field(k, v) + elif isinstance(v, int): + b.int64_field(k, v) + elif isinstance(v, float): + b.float64_field(k, v) + else: + raise ValueError( + f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" + ) + + if r: + raise ValueError( + f"There are unhandled fields in the row: {r_json()}" + ) + + client.write_to_db(db, b) written += 1 - client.info(f"[{task_id}] wrote {written} points to {db_name}.{table_name}") + client.info(f"[{task_id}] wrote {written} points to {db}.{table}") return written diff --git a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py index b5809cb..f6d76ba 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py @@ -38,15 +38,13 @@ def query(self, query: str) -> list[Dict[str, Any]]: with urlopen(req) as resp: rows = json.load(resp) - prefix = json.dumps(rows[:3], indent=2) - print( - f"[INFLUX QUERY RESPONSE](rows={len(rows)}):\n{_prettify(prefix)}" - ) + prefix = "\n".join(json.dumps(row) for row in rows[:3]) + print(f"[INFLUX QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}") return rows def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: - print(f"[INFLUX WRITE](db_name={db_name})\n{line.build()}") + print(f"[INFLUX WRITE](db={db_name}) {line.build()}") def info(self, msg: str) -> None: print(f"[PLUGIN INFO] {msg}") diff --git a/dashboards/customized/influxdb_v3/plugin/influx/types.py b/dashboards/customized/influxdb_v3/plugin/influx/types.py index d4ac234..750f90d 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/types.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/types.py @@ -1,6 +1,8 @@ from typing import Any, Dict, Protocol, Self, runtime_checkable +# Find the actual implementation in Python: +# https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L488-L613 @runtime_checkable class LineBuilderProtocol(Protocol): def __init__(self, measurement: str) -> None: ... @@ -13,6 +15,8 @@ def bool_field(self, key: str, value: bool) -> Self: ... def build(self) -> str: ... +# Find the actual implementation in Rust: +# https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L203 @runtime_checkable class InfluxDBClient(Protocol): def query(self, query: str) -> list[Dict[str, Any]]: ... diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index d79796b..946db40 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime +from typing import Any, Dict from .config import Config from .influx.client import InfluxDBClient, write_points @@ -25,6 +26,24 @@ def run_hourly( run_hourly_window(client, config, window, task_id) +def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + # Analytics may produce data points with missing (NULL) project_id + # We are filling the gaps with a default to ease further processing of the data. + for r in rows: + if r.get("project_id") is None: + r["project_id"] = "undefined" + return rows + + +def _normalize_topic(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + # Analytics may produce data points with missing (NULL) topic + # We are filling the gaps with a default to ease further processing of the data. + for r in rows: + if r.get("topic") is None: + r["topic"] = "undefined" + return rows + + def run_hourly_window( client: InfluxDBClient, config: Config, window: Window, task_id: str ) -> None: @@ -55,9 +74,9 @@ def run_hourly_window( stats_rows = client.query(stats_sql) write_points( client, - db_name=config.agg_database, - table_name="default_agg_stats", - rows=stats_rows, + db=config.agg_database, + table="default_agg_stats", + rows=_normalize_project_id(stats_rows), time_col="time", tag_cols=( "deployment", @@ -98,9 +117,9 @@ def run_hourly_window( topic_rows = client.query(topic_sql) write_points( client, - db_name=config.agg_database, - table_name="default_agg_topic_2", - rows=topic_rows, + db=config.agg_database, + table="default_agg_topic_2", + rows=_normalize_topic(topic_rows), time_col="time", tag_cols=("title", "topic", "model"), field_cols=( @@ -136,11 +155,11 @@ def run_hourly_window( token_rows = client.query(token_sql) write_points( client, - db_name=config.agg_database, - table_name="default_agg_topic", + db=config.agg_database, + table="default_agg_topic", rows=token_rows, time_col="time", - tag_cols=("user_type"), + tag_cols=("user_type",), field_cols=( "class_1", "class_2", @@ -172,9 +191,9 @@ def run_hourly_window( kpi_rows = client.query(kpi_sql) write_points( client, - db_name=config.agg_database, - table_name="default_agg_kpi", - rows=kpi_rows, + db=config.agg_database, + table="default_agg_kpi", + rows=_normalize_project_id(kpi_rows), time_col="time", tag_cols=("user_hash", "project_id", "parent_deployment", "title"), field_cols=( @@ -200,8 +219,8 @@ def run_hourly_window( chat_rows = client.query(chat_sql) write_points( client, - db_name=config.agg_database, - table_name="default_agg_chatid", + db=config.agg_database, + table="default_agg_chatid", rows=chat_rows, time_col="time", tag_cols=("chat_id",), diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index 950b7e5..68d6239 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -76,8 +76,8 @@ def run_monthly_window( write_points( client, - db_name=config.agg_database, - table_name="default_agg_month", + db=config.agg_database, + table="default_agg_month", rows=[merged], time_col="time", tag_cols=(), From 0ec271f01ae8320df887d14a2a1f021ad19325e5 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 13:13:24 +0000 Subject: [PATCH 35/56] feat: add InfluxDB decorator --- .../customized/influxdb_v3/plugin/__init__.py | 19 +++++--- .../customized/influxdb_v3/plugin/__main__.py | 6 ++- .../influxdb_v3/plugin/influx/client.py | 12 +++-- .../influxdb_v3/plugin/influx/decorator.py | 45 +++++++++++++++++++ .../influxdb_v3/plugin/influx/line_builder.py | 3 ++ .../influxdb_v3/plugin/influx/mocks.py | 42 +++++++++-------- .../influxdb_v3/plugin/rollup_hourly.py | 13 ++---- .../influxdb_v3/plugin/rollup_monthly.py | 11 ++--- 8 files changed, 104 insertions(+), 47 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/influx/decorator.py diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 489ef66..556bc19 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -51,6 +51,7 @@ from .config import Config, Mode from .influx.client import InfluxDBClient +from .influx.decorator import LoggingDecorator from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly @@ -63,26 +64,30 @@ def process_scheduled_call( """ InfluxDB 3 scheduled plugin entrypoint. """ - task_id = str(uuid.uuid4()) - config = Config.parse(args) call_time = call_time.replace(tzinfo=timezone.utc) + influxdb3_local = LoggingDecorator( + task_id=str(uuid.uuid4()), + database=config.input_database, + client=influxdb3_local, + ) + influxdb3_local.info( - f"[{task_id}] scheduled call mode={config.mode.value} call_time={call_time.isoformat()} args={args}" + f"scheduled call mode={config.mode.value} call_time={call_time.isoformat()} args={args}" ) try: match config.mode: case Mode.MONTHLY: - run_monthly(influxdb3_local, call_time, config, task_id=task_id) + run_monthly(influxdb3_local, call_time, config) case Mode.HOURLY: - run_hourly(influxdb3_local, call_time, config, task_id=task_id) + run_hourly(influxdb3_local, call_time, config) case _: assert_never(config.mode) - influxdb3_local.info(f"[{task_id}] completed OK") + influxdb3_local.info("completed OK") except Exception as e: - influxdb3_local.error(f"[{task_id}] failed: {e}") + influxdb3_local.error(f"failed: {e}") raise diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index c62650b..ab420b9 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -8,7 +8,7 @@ from . import process_scheduled_call from .config import Config -from .influx.mocks import ReadOnlyInfluxDBClient +from .influx.mocks import HTTPInfluxDBClient def _get_env(name: str) -> str: @@ -27,7 +27,9 @@ def main() -> None: url = _get_env("INFLUX_URL") token = _get_env("INFLUX_API_TOKEN") database = Config.parse(args or {}).input_database - client = ReadOnlyInfluxDBClient(url=url, token=token, database=database) + client = HTTPInfluxDBClient( + url=url, token=token, database=database, readonly=True + ) # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. process_scheduled_call( diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client.py b/dashboards/customized/influxdb_v3/plugin/influx/client.py index f9e3ef1..f8788b7 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client.py @@ -3,8 +3,9 @@ from typing import Any, Dict, List, Tuple from ..dates import parse_iso_date +from .decorator import LoggingDecorator from .line_builder import LineBuilder as LineBuilderImpl -from .mocks import ReadOnlyInfluxDBClient +from .mocks import HTTPInfluxDBClient from .types import InfluxDBClient, LineBuilderProtocol @@ -17,7 +18,6 @@ def write_points( time_col: str, tag_cols: Tuple[str, ...], field_cols: Tuple[str, ...], - task_id: str, ) -> int: """ Write each row as a point into db_name.table_name. @@ -78,7 +78,7 @@ def r_json(): client.write_to_db(db, b) written += 1 - client.info(f"[{task_id}] wrote {written} points to {db}.{table}") + client.info(f"wrote {written} points to {db}.{table}") return written @@ -89,7 +89,11 @@ def _ns(dt: datetime) -> int: def _create_line_builder( client: InfluxDBClient, table_name: str ) -> LineBuilderProtocol: - if isinstance(client, ReadOnlyInfluxDBClient): + if ( + isinstance(client, HTTPInfluxDBClient) + or isinstance(client, LoggingDecorator) + and isinstance(client._client, HTTPInfluxDBClient) + ): return LineBuilderImpl(table_name) else: # LineBuilder is available in the runtime: diff --git a/dashboards/customized/influxdb_v3/plugin/influx/decorator.py b/dashboards/customized/influxdb_v3/plugin/influx/decorator.py new file mode 100644 index 0000000..2857296 --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/influx/decorator.py @@ -0,0 +1,45 @@ +import json +from typing import Any, Dict + +from .types import InfluxDBClient, LineBuilderProtocol + + +class LoggingDecorator(InfluxDBClient): + _task_id: str + _database: str + _client: InfluxDBClient + + def __init__(self, *, task_id: str, database: str, client: InfluxDBClient): + self._database = database + self._task_id = task_id + self._client = client + + def query(self, query: str) -> list[Dict[str, Any]]: + print(f"[{self._task_id}][QUERY REQUEST]\n{_prettify(query)}") + + rows = self._client.query(query) + + prefix = "\n".join(json.dumps(row) for row in rows[:3]) + print( + f"[{self._task_id}][QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}" + ) + + return rows + + def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: + print(f"[{self._task_id}][WRITE](db={db_name}) {line.build()}") + self._client.write_to_db(db_name, line) + + def info(self, msg: str) -> None: + self._client.info(f"[{self._task_id}][INFO] {msg}") + + def error(self, msg: str) -> None: + self._client.error(f"[{self._task_id}][ERROR] {msg}") + + +def _prettify(text: str) -> str: + return _line_prefix(" | ", text.strip()) + + +def _line_prefix(prefix: str, text: str) -> str: + return "\n".join(prefix + line for line in text.splitlines()) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py b/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py index cfa8750..06ae503 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/line_builder.py @@ -1,6 +1,9 @@ """ Copied as is from the actual LineBuilder: https://github.com/influxdata/influxdb/blob/37ff7e6cd4598c312df3688026764a322969c1de/influxdb3_py_api/src/system_py.rs#L488-L613 + +The line protocol itself: +https://docs.influxdata.com/influxdb3/core/reference/line-protocol/ """ from collections import OrderedDict diff --git a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py index f6d76ba..807e812 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/mocks.py @@ -5,20 +5,19 @@ from .types import InfluxDBClient, LineBuilderProtocol -class ReadOnlyInfluxDBClient(InfluxDBClient): +class HTTPInfluxDBClient(InfluxDBClient): _influxdb_url: str _influxdb_token: str - _database: str + _readonly: bool - def __init__(self, *, url: str, token: str, database: str): + def __init__(self, *, url: str, token: str, database: str, readonly: bool): self._database = database self._influxdb_url = url self._influxdb_token = token + self._readonly = readonly def query(self, query: str) -> list[Dict[str, Any]]: - print(f"[INFLUX QUERY]\n{_prettify(query)}") - endpoint = f"{self._influxdb_url}/api/v3/query_sql" payload = json.dumps( @@ -38,24 +37,31 @@ def query(self, query: str) -> list[Dict[str, Any]]: with urlopen(req) as resp: rows = json.load(resp) - prefix = "\n".join(json.dumps(row) for row in rows[:3]) - print(f"[INFLUX QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}") - return rows def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: - print(f"[INFLUX WRITE](db={db_name}) {line.build()}") - - def info(self, msg: str) -> None: - print(f"[PLUGIN INFO] {msg}") + if self._readonly: + return - def error(self, msg: str) -> None: - print(f"[PLUGIN ERROR] {msg}") + endpoint = f"{self._influxdb_url}/api/v3/write_lp?db={db_name}&precision=nanosecond&accept_partial=false" + req = Request( + endpoint, + data=line.build().encode(), + method="POST", + headers={ + "Authorization": f"Token {self._influxdb_token}", + "Content-Type": "text/plain", + }, + ) -def _prettify(text: str) -> str: - return _line_prefix(" | ", text.strip()) + with urlopen(req): + # InfluxDB returns 204 on success. + # We are not interested in the response. + pass + def info(self, msg: str) -> None: + print(msg) -def _line_prefix(prefix: str, text: str) -> str: - return "\n".join(prefix + line for line in text.splitlines()) + def error(self, msg: str) -> None: + print(msg) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 946db40..3e3d60a 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -9,7 +9,7 @@ def run_hourly( - client: InfluxDBClient, call_time: datetime, config: Config, *, task_id: str + client: InfluxDBClient, call_time: datetime, config: Config ) -> None: """ Reads: raw_table @@ -21,9 +21,9 @@ def run_hourly( n = len(windows) for idx, window in enumerate(windows, start=1): client.info( - f"[{task_id}] [{idx}/{n}] {config.window_hours}-hours rollup window: {window.display()}" + f"[{idx}/{n}] {config.window_hours}-hours rollup window: {window.display()}" ) - run_hourly_window(client, config, window, task_id) + run_hourly_window(client, config, window) def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: @@ -45,7 +45,7 @@ def _normalize_topic(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: def run_hourly_window( - client: InfluxDBClient, config: Config, window: Window, task_id: str + client: InfluxDBClient, config: Config, window: Window ) -> None: start_s = window.start_s in_window = window.in_window_sql() @@ -94,7 +94,6 @@ def run_hourly_window( "request_count", "unique_user_count", ), - task_id=task_id, ) # 2) default_agg_topic_2 @@ -129,7 +128,6 @@ def run_hourly_window( "prompt_tokens", "completion_tokens", ), - task_id=task_id, ) # 3) default_agg_topic - token class histogram @@ -168,7 +166,6 @@ def run_hourly_window( "class_5", "class_6", ), - task_id=task_id, ) # 4) default_agg_kpi @@ -202,7 +199,6 @@ def run_hourly_window( "completion_tokens", "prompt_tokens", ), - task_id=task_id, ) # 5) default_agg_chatid @@ -225,5 +221,4 @@ def run_hourly_window( time_col="time", tag_cols=("chat_id",), field_cols=("request_count",), - task_id=task_id, ) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index 68d6239..d188262 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -9,7 +9,7 @@ def run_monthly( - client: InfluxDBClient, call_time: datetime, config: Config, *, task_id: str + client: InfluxDBClient, call_time: datetime, config: Config ) -> None: """ Reads: default_agg_stats, default_agg_kpi @@ -19,14 +19,12 @@ def run_monthly( n = len(windows) for idx, window in enumerate(windows, start=1): - client.info( - f"[{task_id}] [{idx}/{n}] monthly rollup window: {window.display()}" - ) - run_monthly_window(client, config, window, task_id) + client.info(f"[{idx}/{n}] monthly rollup window: {window.display()}") + run_monthly_window(client, config, window) def run_monthly_window( - client: InfluxDBClient, config: Config, window: Window, task_id: str + client: InfluxDBClient, config: Config, window: Window ) -> None: start_s = window.start_s in_window = window.in_window_sql() @@ -93,7 +91,6 @@ def run_monthly_window( "total_cost_per_model", "avg_cost_per_model", ), - task_id=task_id, ) From db21bde8539070fce352b385f693fb695696c828 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 14:41:06 +0000 Subject: [PATCH 36/56] feat: add readonly config flag --- dashboards/customized/influxdb_v3/plugin/__main__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index ab420b9..776ca4f 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -22,13 +22,16 @@ def main() -> None: config_file = Path(sys.argv[1]) args = tomllib.loads(Path(config_file).read_text()) else: - args = None + args = {} + + readonly = args.pop("readonly", "false").lower() == "true" url = _get_env("INFLUX_URL") token = _get_env("INFLUX_API_TOKEN") + database = Config.parse(args or {}).input_database client = HTTPInfluxDBClient( - url=url, token=token, database=database, readonly=True + url=url, token=token, database=database, readonly=readonly ) # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. From d8437f33d3ff5801e0909c89083128f1fd3838a4 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 14:48:15 +0000 Subject: [PATCH 37/56] fix: moving code around --- .../influxdb_v3/plugin/rollup_hourly.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 3e3d60a..c57e3df 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -26,24 +26,6 @@ def run_hourly( run_hourly_window(client, config, window) -def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: - # Analytics may produce data points with missing (NULL) project_id - # We are filling the gaps with a default to ease further processing of the data. - for r in rows: - if r.get("project_id") is None: - r["project_id"] = "undefined" - return rows - - -def _normalize_topic(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: - # Analytics may produce data points with missing (NULL) topic - # We are filling the gaps with a default to ease further processing of the data. - for r in rows: - if r.get("topic") is None: - r["topic"] = "undefined" - return rows - - def run_hourly_window( client: InfluxDBClient, config: Config, window: Window ) -> None: @@ -222,3 +204,21 @@ def run_hourly_window( tag_cols=("chat_id",), field_cols=("request_count",), ) + + +def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + # Analytics may produce data points with missing (NULL) project_id field. + # We are filling the gaps with a default to ease further processing of the data. + for r in rows: + if r.get("project_id") is None: + r["project_id"] = "undefined" + return rows + + +def _normalize_topic(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + # Analytics may produce data points with missing (NULL) topic field. + # We are filling the gaps with a default to ease further processing of the data. + for r in rows: + if r.get("topic") is None: + r["topic"] = "undefined" + return rows From 920387d85be9d7421bb374b73ade8996ed256dd9 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 15:31:40 +0000 Subject: [PATCH 38/56] feat: add add_prefix method --- .../customized/influxdb_v3/plugin/__init__.py | 22 +++++++-------- .../customized/influxdb_v3/plugin/__main__.py | 2 +- .../influx/{mocks.py => client_http.py} | 0 .../{decorator.py => client_wrapper.py} | 27 ++++++++++++------- .../plugin/influx/{client.py => write.py} | 21 +++++++-------- .../influxdb_v3/plugin/rollup_hourly.py | 7 ++--- .../influxdb_v3/plugin/rollup_monthly.py | 7 ++--- 7 files changed, 46 insertions(+), 40 deletions(-) rename dashboards/customized/influxdb_v3/plugin/influx/{mocks.py => client_http.py} (100%) rename dashboards/customized/influxdb_v3/plugin/influx/{decorator.py => client_wrapper.py} (50%) rename dashboards/customized/influxdb_v3/plugin/influx/{client.py => write.py} (86%) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 556bc19..6e6a4c0 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -50,8 +50,8 @@ from typing_extensions import assert_never from .config import Config, Mode -from .influx.client import InfluxDBClient -from .influx.decorator import LoggingDecorator +from .influx.client_wrapper import InfluxDBClientWrapper +from .influx.types import InfluxDBClient from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly @@ -67,27 +67,25 @@ def process_scheduled_call( config = Config.parse(args) call_time = call_time.replace(tzinfo=timezone.utc) - influxdb3_local = LoggingDecorator( - task_id=str(uuid.uuid4()), - database=config.input_database, - client=influxdb3_local, - ) + client = InfluxDBClientWrapper( + client=influxdb3_local, database=config.input_database + ).add_prefix(f"[{str(uuid.uuid4())}]") - influxdb3_local.info( + client.info( f"scheduled call mode={config.mode.value} call_time={call_time.isoformat()} args={args}" ) try: match config.mode: case Mode.MONTHLY: - run_monthly(influxdb3_local, call_time, config) + run_monthly(client, call_time, config) case Mode.HOURLY: - run_hourly(influxdb3_local, call_time, config) + run_hourly(client, call_time, config) case _: assert_never(config.mode) - influxdb3_local.info("completed OK") + client.info("completed OK") except Exception as e: - influxdb3_local.error(f"failed: {e}") + client.error(f"failed: {e}") raise diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index 776ca4f..efe7512 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -8,7 +8,7 @@ from . import process_scheduled_call from .config import Config -from .influx.mocks import HTTPInfluxDBClient +from .influx.client_http import HTTPInfluxDBClient def _get_env(name: str) -> str: diff --git a/dashboards/customized/influxdb_v3/plugin/influx/mocks.py b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py similarity index 100% rename from dashboards/customized/influxdb_v3/plugin/influx/mocks.py rename to dashboards/customized/influxdb_v3/plugin/influx/client_http.py diff --git a/dashboards/customized/influxdb_v3/plugin/influx/decorator.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py similarity index 50% rename from dashboards/customized/influxdb_v3/plugin/influx/decorator.py rename to dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 2857296..263b928 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/decorator.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -4,37 +4,46 @@ from .types import InfluxDBClient, LineBuilderProtocol -class LoggingDecorator(InfluxDBClient): - _task_id: str +class InfluxDBClientWrapper(InfluxDBClient): + _log_prefix: str _database: str _client: InfluxDBClient - def __init__(self, *, task_id: str, database: str, client: InfluxDBClient): + def __init__( + self, *, client: InfluxDBClient, database: str, log_prefix: str = "" + ): self._database = database - self._task_id = task_id + self._log_prefix = log_prefix self._client = client + def add_prefix(self, log_prefix: str) -> "InfluxDBClientWrapper": + return InfluxDBClientWrapper( + log_prefix=self._log_prefix + log_prefix, + database=self._database, + client=self._client, + ) + def query(self, query: str) -> list[Dict[str, Any]]: - print(f"[{self._task_id}][QUERY REQUEST]\n{_prettify(query)}") + print(f"{self._log_prefix}[QUERY REQUEST]\n{_prettify(query)}") rows = self._client.query(query) prefix = "\n".join(json.dumps(row) for row in rows[:3]) print( - f"[{self._task_id}][QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}" + f"{self._log_prefix}[QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}" ) return rows def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: - print(f"[{self._task_id}][WRITE](db={db_name}) {line.build()}") + print(f"{self._log_prefix}[WRITE](db={db_name}) {line.build()}") self._client.write_to_db(db_name, line) def info(self, msg: str) -> None: - self._client.info(f"[{self._task_id}][INFO] {msg}") + self._client.info(f"{self._log_prefix}[INFO] {msg}") def error(self, msg: str) -> None: - self._client.error(f"[{self._task_id}][ERROR] {msg}") + self._client.error(f"{self._log_prefix}[ERROR] {msg}") def _prettify(text: str) -> str: diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client.py b/dashboards/customized/influxdb_v3/plugin/influx/write.py similarity index 86% rename from dashboards/customized/influxdb_v3/plugin/influx/client.py rename to dashboards/customized/influxdb_v3/plugin/influx/write.py index f8788b7..95d2dcf 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/write.py @@ -3,14 +3,14 @@ from typing import Any, Dict, List, Tuple from ..dates import parse_iso_date -from .decorator import LoggingDecorator +from .client_http import HTTPInfluxDBClient +from .client_wrapper import InfluxDBClientWrapper from .line_builder import LineBuilder as LineBuilderImpl -from .mocks import HTTPInfluxDBClient -from .types import InfluxDBClient, LineBuilderProtocol +from .types import LineBuilderProtocol def write_points( - client: InfluxDBClient, + client: InfluxDBClientWrapper, *, db: str, table: str, @@ -25,7 +25,8 @@ def write_points( """ written = 0 - for r_orig in rows: + rows_n = len(rows) + for row_idx, r_orig in enumerate(rows, start=1): r = r_orig.copy() def r_json(): @@ -75,7 +76,7 @@ def r_json(): f"There are unhandled fields in the row: {r_json()}" ) - client.write_to_db(db, b) + client.add_prefix(f"[{row_idx}/{rows_n}] ").write_to_db(db, b) written += 1 client.info(f"wrote {written} points to {db}.{table}") @@ -87,13 +88,9 @@ def _ns(dt: datetime) -> int: def _create_line_builder( - client: InfluxDBClient, table_name: str + client: InfluxDBClientWrapper, table_name: str ) -> LineBuilderProtocol: - if ( - isinstance(client, HTTPInfluxDBClient) - or isinstance(client, LoggingDecorator) - and isinstance(client._client, HTTPInfluxDBClient) - ): + if isinstance(client._client, HTTPInfluxDBClient): return LineBuilderImpl(table_name) else: # LineBuilder is available in the runtime: diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index c57e3df..5449649 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -4,12 +4,13 @@ from typing import Any, Dict from .config import Config -from .influx.client import InfluxDBClient, write_points +from .influx.client_wrapper import InfluxDBClientWrapper +from .influx.write import write_points from .window import Window def run_hourly( - client: InfluxDBClient, call_time: datetime, config: Config + client: InfluxDBClientWrapper, call_time: datetime, config: Config ) -> None: """ Reads: raw_table @@ -27,7 +28,7 @@ def run_hourly( def run_hourly_window( - client: InfluxDBClient, config: Config, window: Window + client: InfluxDBClientWrapper, config: Config, window: Window ) -> None: start_s = window.start_s in_window = window.in_window_sql() diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index d188262..f483f00 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -4,12 +4,13 @@ from typing import Any, Dict from .config import Config -from .influx.client import InfluxDBClient, write_points +from .influx.client_wrapper import InfluxDBClientWrapper +from .influx.write import write_points from .window import Window def run_monthly( - client: InfluxDBClient, call_time: datetime, config: Config + client: InfluxDBClientWrapper, call_time: datetime, config: Config ) -> None: """ Reads: default_agg_stats, default_agg_kpi @@ -24,7 +25,7 @@ def run_monthly( def run_monthly_window( - client: InfluxDBClient, config: Config, window: Window + client: InfluxDBClientWrapper, config: Config, window: Window ) -> None: start_s = window.start_s in_window = window.in_window_sql() From 42db861df78cc80b15d3070be6d2c798a30e9047 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 15:40:46 +0000 Subject: [PATCH 39/56] feat: use add_prefix more liberally --- dashboards/customized/influxdb_v3/plugin/influx/write.py | 2 +- .../customized/influxdb_v3/plugin/rollup_hourly.py | 9 ++++++--- .../customized/influxdb_v3/plugin/rollup_monthly.py | 7 +++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/write.py b/dashboards/customized/influxdb_v3/plugin/influx/write.py index 95d2dcf..4d4f766 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/write.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/write.py @@ -76,7 +76,7 @@ def r_json(): f"There are unhandled fields in the row: {r_json()}" ) - client.add_prefix(f"[{row_idx}/{rows_n}] ").write_to_db(db, b) + client.add_prefix(f"[point|{row_idx:>2}/{rows_n}]").write_to_db(db, b) written += 1 client.info(f"wrote {written} points to {db}.{table}") diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 5449649..56ee938 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -21,15 +21,18 @@ def run_hourly( n = len(windows) for idx, window in enumerate(windows, start=1): - client.info( - f"[{idx}/{n}] {config.window_hours}-hours rollup window: {window.display()}" + run_hourly_window( + client.add_prefix(f"[win|{idx:>2}/{n}]"), config, window ) - run_hourly_window(client, config, window) def run_hourly_window( client: InfluxDBClientWrapper, config: Config, window: Window ) -> None: + client.info( + f"{config.window_hours}-hours rollup window: {window.display()}" + ) + start_s = window.start_s in_window = window.in_window_sql() diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index f483f00..c80ae04 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -20,13 +20,16 @@ def run_monthly( n = len(windows) for idx, window in enumerate(windows, start=1): - client.info(f"[{idx}/{n}] monthly rollup window: {window.display()}") - run_monthly_window(client, config, window) + run_monthly_window( + client.add_prefix(f"[win|{idx:>2}/{n}]"), config, window + ) def run_monthly_window( client: InfluxDBClientWrapper, config: Config, window: Window ) -> None: + client.info(f"monthly rollup window: {window.display()}") + start_s = window.start_s in_window = window.in_window_sql() From c1704fc903edc65c23b4a9c36124d0cdf20eb3e6 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 15:43:22 +0000 Subject: [PATCH 40/56] fix: minor fixes --- dashboards/customized/influxdb_v3/plugin/rollup_hourly.py | 4 +--- dashboards/customized/influxdb_v3/plugin/rollup_monthly.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 56ee938..54966f4 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -21,9 +21,7 @@ def run_hourly( n = len(windows) for idx, window in enumerate(windows, start=1): - run_hourly_window( - client.add_prefix(f"[win|{idx:>2}/{n}]"), config, window - ) + run_hourly_window(client.add_prefix(f"[win|{idx}/{n}]"), config, window) def run_hourly_window( diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index c80ae04..d555260 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -21,7 +21,7 @@ def run_monthly( n = len(windows) for idx, window in enumerate(windows, start=1): run_monthly_window( - client.add_prefix(f"[win|{idx:>2}/{n}]"), config, window + client.add_prefix(f"[win|{idx}/{n}]"), config, window ) From a5fb4df21fac154546c7e54eea6048ce6c19886b Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 15:59:56 +0000 Subject: [PATCH 41/56] feat: add verbose config flag --- .../customized/influxdb_v3/plugin/__init__.py | 10 ++++++- .../customized/influxdb_v3/plugin/config.py | 5 ++++ .../influxdb_v3/plugin/influx/client_http.py | 4 +-- .../plugin/influx/client_wrapper.py | 30 ++++++++++++++----- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 6e6a4c0..4c21d75 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -40,6 +40,12 @@ "description": "Optional backfill end (ISO 8601)). If set with start_time, overrides call_time-derived window.", "required": false } + { + "name": "verbose", + "example": "true", + "description": "Enable verbose logging including SQL queries and data points written to InfluxDB. False by default.", + "required": false + } ] } """ @@ -68,7 +74,9 @@ def process_scheduled_call( call_time = call_time.replace(tzinfo=timezone.utc) client = InfluxDBClientWrapper( - client=influxdb3_local, database=config.input_database + client=influxdb3_local, + database=config.input_database, + verbose=config.verbose, ).add_prefix(f"[{str(uuid.uuid4())}]") client.info( diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 734ed02..3001423 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -26,6 +26,8 @@ class Config: start_time: datetime | None end_time: datetime | None + verbose: bool + @classmethod def parse(cls, d: Dict[str, str] | None) -> "Config": d = d or {} @@ -54,6 +56,8 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": f"window_hours must divide 24 evenly (got {window_hours})" ) + verbose = (d.pop("verbose", None) or "false").lower() == "true" + if d: raise ValueError(f"Unexpected config keys: {', '.join(d.keys())}") @@ -64,6 +68,7 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": start_time=start_time, end_time=end_time, window_hours=window_hours, + verbose=verbose, ) @property diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py index 807e812..d8f3086 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py @@ -61,7 +61,7 @@ def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: pass def info(self, msg: str) -> None: - print(msg) + print(msg, flush=True) def error(self, msg: str) -> None: - print(msg) + print(msg, flush=True) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 263b928..761065a 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -7,36 +7,52 @@ class InfluxDBClientWrapper(InfluxDBClient): _log_prefix: str _database: str + _verbose: bool _client: InfluxDBClient def __init__( - self, *, client: InfluxDBClient, database: str, log_prefix: str = "" + self, + *, + client: InfluxDBClient, + database: str, + verbose: bool, + log_prefix: str = "", ): self._database = database self._log_prefix = log_prefix self._client = client + self._verbose = verbose def add_prefix(self, log_prefix: str) -> "InfluxDBClientWrapper": return InfluxDBClientWrapper( log_prefix=self._log_prefix + log_prefix, + verbose=self._verbose, database=self._database, client=self._client, ) def query(self, query: str) -> list[Dict[str, Any]]: - print(f"{self._log_prefix}[QUERY REQUEST]\n{_prettify(query)}") + if self._verbose: + self.info(f"SQL query:\n{_prettify(query)}") rows = self._client.query(query) - prefix = "\n".join(json.dumps(row) for row in rows[:3]) - print( - f"{self._log_prefix}[QUERY RESULT](rows={len(rows)}):\n{_prettify(prefix)}" - ) + if self._verbose: + prefix = "\n".join(json.dumps(row) for row in rows[:3]) + self.info( + f"SQL query returned {len(rows)} rows. First 3 are:\n{_prettify(prefix)}" + ) + else: + self.info(f"SQL query returned {len(rows)} rows.") return rows def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: - print(f"{self._log_prefix}[WRITE](db={db_name}) {line.build()}") + if self._verbose: + self.info(f"Writing to database {db_name}: {line.build()}") + else: + self.info(f"Writing to database {db_name}...") + self._client.write_to_db(db_name, line) def info(self, msg: str) -> None: From 4804a0c80213f16024449931c9164e2a393aeb56 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 16:08:46 +0000 Subject: [PATCH 42/56] feat: moving code around --- .../plugin/influx/client_wrapper.py | 4 +- .../influxdb_v3/plugin/influx/write.py | 106 ++++++++++-------- 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 761065a..e77f136 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -63,8 +63,8 @@ def error(self, msg: str) -> None: def _prettify(text: str) -> str: - return _line_prefix(" | ", text.strip()) + return _add_prefix_to_lines(" | ", text.strip()) -def _line_prefix(prefix: str, text: str) -> str: +def _add_prefix_to_lines(prefix: str, text: str) -> str: return "\n".join(prefix + line for line in text.splitlines()) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/write.py b/dashboards/customized/influxdb_v3/plugin/influx/write.py index 4d4f766..685bfc8 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/write.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/write.py @@ -23,64 +23,78 @@ def write_points( Write each row as a point into db_name.table_name. time_col can be ISO string, datetime, or ns int. """ - written = 0 - rows_n = len(rows) - for row_idx, r_orig in enumerate(rows, start=1): - r = r_orig.copy() + n = len(rows) + for idx, row in enumerate(rows, start=1): + builder = _create_data_point( + _create_line_builder(client, table), + row, + time_col=time_col, + tag_cols=tag_cols, + field_cols=field_cols, + ) - def r_json(): - return json.dumps(r_orig) + prefix = f"[point|{idx:>2}/{n}]" + client.add_prefix(prefix).write_to_db(db, builder) - b = _create_line_builder(client, table) + client.info(f"wrote {n} points to {db}.{table}") + return n - t = r.pop(time_col, None) - if isinstance(t, str): - b.time_ns(_ns(parse_iso_date(f"{time_col!r} column", t))) + +def _create_data_point( + builder: LineBuilderProtocol, + row: Dict[str, Any], + *, + time_col: str, + tag_cols: Tuple[str, ...], + field_cols: Tuple[str, ...], +) -> LineBuilderProtocol: + r = row.copy() + + def r_json(): + return json.dumps(row) + + t = r.pop(time_col, None) + if isinstance(t, str): + builder.time_ns(_ns(parse_iso_date(f"{time_col!r} column", t))) + else: + raise ValueError( + f"Unexpected time value in the column '{time_col!r}': {t} of type {type(t)}" + ) + + for k in tag_cols: + if (v := r.pop(k, None)) is None: + raise ValueError( + f"Expected to find a tag column {k!r}, but it's missing from the given row: {r_json()}" + ) + if isinstance(v, str): + builder.tag(k, v) else: raise ValueError( - f"Unexpected time value in the column '{time_col!r}': {t} of type {type(t)}" + f"Unexpected tag value in the column '{k!r}': {v} of type {type(v)}, but tags are expected to always be strings." ) - for k in tag_cols: - if (v := r.pop(k, None)) is None: - raise ValueError( - f"Expected to find a tag column {k!r}, but it's missing from the given row: {r_json()}" - ) - if isinstance(v, str): - b.tag(k, v) - else: - raise ValueError( - f"Unexpected tag value in the column '{k!r}': {v} of type {type(v)}, but tags are expected to always be strings." - ) - - for k in field_cols: - if (v := r.pop(k, None)) is None: - raise ValueError( - f"Expected to find a field column {k!r}, but it's missing from the given row: {r_json()}" - ) - - if isinstance(v, bool): - b.bool_field(k, v) - elif isinstance(v, int): - b.int64_field(k, v) - elif isinstance(v, float): - b.float64_field(k, v) - else: - raise ValueError( - f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" - ) - - if r: + for k in field_cols: + if (v := r.pop(k, None)) is None: + raise ValueError( + f"Expected to find a field column {k!r}, but it's missing from the given row: {r_json()}" + ) + + if isinstance(v, bool): + builder.bool_field(k, v) + elif isinstance(v, int): + builder.int64_field(k, v) + elif isinstance(v, float): + builder.float64_field(k, v) + else: raise ValueError( - f"There are unhandled fields in the row: {r_json()}" + f"Unexpected field value in the column '{k!r}': {v} of type {type(v)}" ) - client.add_prefix(f"[point|{row_idx:>2}/{rows_n}]").write_to_db(db, b) - written += 1 + if r: + raise ValueError(f"There are unhandled fields in the row: {r_json()}") - client.info(f"wrote {written} points to {db}.{table}") - return written + return builder def _ns(dt: datetime) -> int: From c37e1ed586bc1de2007340b35cae379279327b5b Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 16:36:07 +0000 Subject: [PATCH 43/56] feat: support batching for writes --- .../influxdb_v3/plugin/influx/client_http.py | 13 +++++-- .../plugin/influx/client_wrapper.py | 35 ++++++++++++++++++- .../influxdb_v3/plugin/influx/write.py | 16 ++++----- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py index d8f3086..6c0d86f 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict +from typing import Any, Dict, List from urllib.request import Request, urlopen from .types import InfluxDBClient, LineBuilderProtocol @@ -39,15 +39,22 @@ def query(self, query: str) -> list[Dict[str, Any]]: return rows - def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: + def write_to_db( + self, + db_name: str, + line: LineBuilderProtocol | List[LineBuilderProtocol], + ) -> None: if self._readonly: return endpoint = f"{self._influxdb_url}/api/v3/write_lp?db={db_name}&precision=nanosecond&accept_partial=false" + lines = line if isinstance(line, list) else [line] + data = "\n".join(line.build() for line in lines).encode() + req = Request( endpoint, - data=line.build().encode(), + data=data, method="POST", headers={ "Authorization": f"Token {self._influxdb_token}", diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index e77f136..697705b 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -1,6 +1,7 @@ import json -from typing import Any, Dict +from typing import Any, Dict, List +from .client_http import HTTPInfluxDBClient from .types import InfluxDBClient, LineBuilderProtocol @@ -55,6 +56,38 @@ def write_to_db(self, db_name: str, line: LineBuilderProtocol) -> None: self._client.write_to_db(db_name, line) + def write_to_db_many( + self, db_name: str, lines: List[LineBuilderProtocol] + ) -> None: + if not lines: + return + + table_name = getattr(lines[0], "measurement", "na") + + n = len(lines) + if isinstance(self._client, HTTPInfluxDBClient): + self._client.write_to_db(db_name, lines) + else: + for idx, row in enumerate(lines, start=1): + prefix = f"[row|{idx:>2}/{n}]" + self.add_prefix(prefix).write_to_db(db_name, row) + + self.info(f"wrote {n} rows to {db_name}.{table_name}") + + def write_to_db_batched( + self, + db_name: str, + lines: List[LineBuilderProtocol], + *, + batch_size: int = 50, + ) -> None: + n = (len(lines) // batch_size) + bool(len(lines) % batch_size) + for idx in range(1, n + 1): + batch = lines[:batch_size] + lines = lines[batch_size:] + prefix = f"[batch|{idx:>2}/{n}]" + self.add_prefix(prefix).write_to_db_many(db_name, batch) + def info(self, msg: str) -> None: self._client.info(f"{self._log_prefix}[INFO] {msg}") diff --git a/dashboards/customized/influxdb_v3/plugin/influx/write.py b/dashboards/customized/influxdb_v3/plugin/influx/write.py index 685bfc8..b96f977 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/write.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/write.py @@ -18,27 +18,23 @@ def write_points( time_col: str, tag_cols: Tuple[str, ...], field_cols: Tuple[str, ...], -) -> int: +) -> None: """ Write each row as a point into db_name.table_name. time_col can be ISO string, datetime, or ns int. """ - - n = len(rows) - for idx, row in enumerate(rows, start=1): - builder = _create_data_point( + lines = [ + _create_data_point( _create_line_builder(client, table), row, time_col=time_col, tag_cols=tag_cols, field_cols=field_cols, ) + for row in rows + ] - prefix = f"[point|{idx:>2}/{n}]" - client.add_prefix(prefix).write_to_db(db, builder) - - client.info(f"wrote {n} points to {db}.{table}") - return n + client.write_to_db_batched(db, lines, batch_size=50) def _create_data_point( From 745bc23c5d3c88e4b6f3dcda8aca2b7cb16f59cd Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 16:44:58 +0000 Subject: [PATCH 44/56] fix: fix bug in Config --- dashboards/customized/influxdb_v3/plugin/config.py | 3 ++- dashboards/customized/influxdb_v3/plugin/influx/client_http.py | 1 + .../customized/influxdb_v3/plugin/influx/client_wrapper.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 3001423..c7356cf 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -30,7 +30,7 @@ class Config: @classmethod def parse(cls, d: Dict[str, str] | None) -> "Config": - d = d or {} + d = (d or {}).copy() mode_s = d.pop("mode", None) or "hourly" try: @@ -43,6 +43,7 @@ def parse(cls, d: Dict[str, str] | None) -> "Config": start_arg = d.pop("start_time", None) end_arg = d.pop("end_time", None) + start_time: datetime | None = ( parse_iso_date("start_time", start_arg) if start_arg else None ) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py index 6c0d86f..75ce444 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py @@ -45,6 +45,7 @@ def write_to_db( line: LineBuilderProtocol | List[LineBuilderProtocol], ) -> None: if self._readonly: + print("skipping writing...") return endpoint = f"{self._influxdb_url}/api/v3/write_lp?db={db_name}&precision=nanosecond&accept_partial=false" diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 697705b..473a063 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -79,7 +79,7 @@ def write_to_db_batched( db_name: str, lines: List[LineBuilderProtocol], *, - batch_size: int = 50, + batch_size: int, ) -> None: n = (len(lines) // batch_size) + bool(len(lines) % batch_size) for idx in range(1, n + 1): From d2e514e4968aac35c63a67a13c378fe63f5e06a1 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 17:40:08 +0000 Subject: [PATCH 45/56] feat: add ThreadPoolExecutor --- .../plugin/influx/client_wrapper.py | 9 ++- .../influxdb_v3/plugin/rollup_hourly.py | 66 ++++++++++++++----- .../influxdb_v3/plugin/rollup_monthly.py | 54 ++++++++++----- 3 files changed, 91 insertions(+), 38 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 473a063..0306484 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -62,17 +62,16 @@ def write_to_db_many( if not lines: return - table_name = getattr(lines[0], "measurement", "na") - n = len(lines) if isinstance(self._client, HTTPInfluxDBClient): + # TODO: missing info logs about the lines being written self._client.write_to_db(db_name, lines) else: - for idx, row in enumerate(lines, start=1): + for idx, line in enumerate(lines, start=1): prefix = f"[row|{idx:>2}/{n}]" - self.add_prefix(prefix).write_to_db(db_name, row) + self.add_prefix(prefix).write_to_db(db_name, line) - self.info(f"wrote {n} rows to {db_name}.{table_name}") + self.info(f"wrote {n} rows to {db_name}") def write_to_db_batched( self, diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 54966f4..0723341 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from typing import Any, Dict @@ -24,17 +25,9 @@ def run_hourly( run_hourly_window(client.add_prefix(f"[win|{idx}/{n}]"), config, window) -def run_hourly_window( - client: InfluxDBClientWrapper, config: Config, window: Window -) -> None: - client.info( - f"{config.window_hours}-hours rollup window: {window.display()}" - ) - - start_s = window.start_s - in_window = window.in_window_sql() - - # 1) default_agg_stats +def default_agg_stats( + client: InfluxDBClientWrapper, config: Config, start_s: str, in_window: str +): stats_sql = f""" SELECT '{start_s}' AS time, @@ -80,7 +73,10 @@ def run_hourly_window( ), ) - # 2) default_agg_topic_2 + +def default_agg_topic_2( + client: InfluxDBClientWrapper, config: Config, start_s: str, in_window: str +): topic_sql = f""" SELECT '{start_s}' AS time, @@ -114,7 +110,10 @@ def run_hourly_window( ), ) - # 3) default_agg_topic - token class histogram + +def default_agg_topic( + client: InfluxDBClientWrapper, config: Config, start_s: str, in_window: str +): token_sql = f""" SELECT '{start_s}' AS time, @@ -152,7 +151,10 @@ def run_hourly_window( ), ) - # 4) default_agg_kpi + +def default_agg_kpi( + client: InfluxDBClientWrapper, config: Config, start_s: str, in_window: str +): kpi_sql = f""" SELECT '{start_s}' AS time, @@ -185,7 +187,10 @@ def run_hourly_window( ), ) - # 5) default_agg_chatid + +def default_agg_chatid( + client: InfluxDBClientWrapper, config: Config, start_s: str, in_window: str +): chat_sql = f""" SELECT '{start_s}' AS time, @@ -208,6 +213,37 @@ def run_hourly_window( ) +def run_hourly_window( + client: InfluxDBClientWrapper, config: Config, window: Window +) -> None: + client.info( + f"{config.window_hours}-hours rollup window: {window.display()}" + ) + + start_s = window.start_s + in_window = window.in_window_sql() + + job_defs = [ + ("default_agg_stats", default_agg_stats), + ("default_agg_topic_2", default_agg_topic_2), + ("default_agg_topic", default_agg_topic), + ("default_agg_kpi", default_agg_kpi), + ("default_agg_chatid", default_agg_chatid), + ] + + jobs = [ + lambda client=client, name=name, func=func: func( + client.add_prefix(f"[{name:<19}]"), config, start_s, in_window + ) + for name, func in job_defs + ] + + with ThreadPoolExecutor(max_workers=5) as ex: + futures = [ex.submit(job) for job in jobs] + for fut in as_completed(futures): + fut.result() + + def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: # Analytics may produce data points with missing (NULL) project_id field. # We are filling the gaps with a default to ease further processing of the data. diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index d555260..e482122 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from typing import Any, Dict @@ -37,11 +38,11 @@ def run_monthly_window( api_sql = f""" SELECT '{start_s}' AS time, - SUM(price) AS total_cost_per_api, - AVG(price) AS avg_cost_per_api, - SUM(request_count) AS total_rc_per_api, - AVG(request_count) AS avg_rc_per_api, - COUNT(*) AS active_apis + COALESCE(SUM(price),0) AS total_cost_per_api, + COALESCE(AVG(price),0) AS avg_cost_per_api, + COALESCE(SUM(request_count),0) AS total_rc_per_api, + COALESCE(AVG(request_count),0) AS avg_rc_per_api, + COUNT(*) AS active_apis FROM ({_get_stats_sub_table("project_id", in_window)}) """ @@ -49,8 +50,9 @@ def run_monthly_window( model_sql = f""" SELECT '{start_s}' AS time, - SUM(price) AS total_cost_per_model, - AVG(price) AS avg_cost_per_model + COALESCE(SUM(price),0) AS total_cost_per_model, + COALESCE(AVG(price),0) AS avg_cost_per_model, + COUNT(*) AS active_models FROM ({_get_stats_sub_table("model", in_window)}) """ @@ -58,23 +60,39 @@ def run_monthly_window( user_sql = f""" SELECT '{start_s}' AS time, - SUM(cost) AS total_user_cost, - AVG(cost) AS avg_cost_per_user, + COALESCE(SUM(cost),0) AS total_user_cost, + COALESCE(AVG(cost),0) AS avg_cost_per_user, COUNT(*) AS unique_users FROM ({_get_kpi_sub_table(in_window)}) """ - api_rows = client.query(api_sql) - model_rows = client.query(model_sql) - user_rows = client.query(user_sql) + jobs = [ + lambda: client.add_prefix("[api ]").query(api_sql), + lambda: client.add_prefix("[model]").query(model_sql), + lambda: client.add_prefix("[user ]").query(user_sql), + ] + + results = [] + with ThreadPoolExecutor(max_workers=3) as ex: + futures = [ex.submit(job) for job in jobs] + for fut in as_completed(futures): + results.append(fut.result()) merged: Dict[str, Any] = {"time": start_s} - if api_rows: - merged.update(api_rows[0]) - if model_rows: - merged.update(model_rows[0]) - if user_rows: - merged.update(user_rows[0]) + for result in results: + if result: + merged.update(result[0]) + + if ( + merged.get("unique_users", 0) + + merged.get("active_apis", 0) + + merged.get("active_models", 0) + == 0 + ): + client.info("No data. Skipping.") + return + + merged.pop("active_models", None) write_points( client, From a8d9f908f0ac06c2ab1b8009c747c877d1ed0f6a Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 17:44:42 +0000 Subject: [PATCH 46/56] fix: improve printing --- .../customized/influxdb_v3/plugin/influx/client_wrapper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index 0306484..cbc2275 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -41,10 +41,10 @@ def query(self, query: str) -> list[Dict[str, Any]]: if self._verbose: prefix = "\n".join(json.dumps(row) for row in rows[:3]) self.info( - f"SQL query returned {len(rows)} rows. First 3 are:\n{_prettify(prefix)}" + f"SQL query returned {len(rows):>3} rows. First 3 are:\n{_prettify(prefix)}" ) else: - self.info(f"SQL query returned {len(rows)} rows.") + self.info(f"SQL query returned {len(rows):>3} rows.") return rows @@ -71,7 +71,7 @@ def write_to_db_many( prefix = f"[row|{idx:>2}/{n}]" self.add_prefix(prefix).write_to_db(db_name, line) - self.info(f"wrote {n} rows to {db_name}") + self.info(f"wrote {n:>3} rows to {db_name}") def write_to_db_batched( self, From 5b38cecae99016f51a1b5771bd77cc5ded407220 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Fri, 6 Mar 2026 17:47:12 +0000 Subject: [PATCH 47/56] fix: minor fix --- dashboards/customized/influxdb_v3/plugin/influx/client_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py index 75ce444..401e229 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_http.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_http.py @@ -48,7 +48,7 @@ def write_to_db( print("skipping writing...") return - endpoint = f"{self._influxdb_url}/api/v3/write_lp?db={db_name}&precision=nanosecond&accept_partial=false" + endpoint = f"{self._influxdb_url}/api/v3/write_lp?db={db_name}&precision=nanosecond&accept_partial=false&no_sync=false" lines = line if isinstance(line, list) else [line] data = "\n".join(line.build() for line in lines).encode() From f878989abafcc70e6d44af416356248bf40898f4 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Mon, 9 Mar 2026 10:32:45 +0000 Subject: [PATCH 48/56] chore: update README --- dashboards/customized/influxdb_v3/.gitignore | 2 +- dashboards/customized/influxdb_v3/README.md | 32 ++++++++++++++++--- .../influxdb_v3/config.example.toml | 27 ++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/config.example.toml diff --git a/dashboards/customized/influxdb_v3/.gitignore b/dashboards/customized/influxdb_v3/.gitignore index e33f88a..51325cd 100644 --- a/dashboards/customized/influxdb_v3/.gitignore +++ b/dashboards/customized/influxdb_v3/.gitignore @@ -1,4 +1,4 @@ *.txt *.log -*.toml +config.toml *.sh diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md index 0799153..6ebb376 100644 --- a/dashboards/customized/influxdb_v3/README.md +++ b/dashboards/customized/influxdb_v3/README.md @@ -61,7 +61,7 @@ Runs at the following UTC times every day: 00:02, 06:02, 12:02, 18:02 (UTC) ``` -The 2-minute offset (`02` instead of `00`) ensures that all raw data for the previous window is ingested before aggregation runs. +The 2-minute offset ensures that all raw data for the previous window is ingested before aggregation runs. Trigger spec: @@ -99,15 +99,15 @@ influxdb3 create trigger \ Runs at: ```txt -00:00:02 UTC on the 1st day of each month +00:00:10 UTC on the 1st day of each month ``` -The 2-minute offset (`02` instead of `00`) ensures that all raw data for the previous window is ingested before aggregation runs. +The 10-minute offset ensures that the data for the last 6-hours window of the month was been processed by the [hourly trigger](#6-hour-aggregation-trigger) and was inserted into the aggregate tables. Trigger spec: ```txt -cron:2 0 0 1 * * +cron:10 0 0 1 * * ``` What it does: @@ -123,7 +123,7 @@ Example trigger creation: influxdb3 create trigger \ --database analytics_agg \ --path analytics_rollups \ - --trigger-spec "cron:2 0 0 1 * *" \ + --trigger-spec "cron:10 0 0 1 * *" \ --trigger-arguments mode=monthly,agg_database=analytics_agg \ analytics_monthly_rollups ``` @@ -199,3 +199,25 @@ After triggers are enabled: * Query `analytics_agg.default_agg_stats` to confirm 6-hour data * Query `analytics_agg.default_agg_month` after the first month boundary * Verify Grafana dashboards point to `analytics_agg` tables + +--- + +## Running trigger manually + +The plugin script could be ran manually on an InfluxDB instance configured by the environment variables: + +|Variable|Description| +|---|---| +|INFLUX_URL|URL to the InfluxDB to write the analytics data| +|INFLUX_API_TOKEN|InfluxDB API Token with the write access to the target database| + +Go to script directory and run it with an optional parameter for a TOML configuration file: + +```sh +cd dashboards/customized/influxdb_v3 +export INFLUX_URL=... +export INFLUX_API_TOKEN=... +python -m plugin config.toml +``` + +Find the example configuration at `config.example.toml`. diff --git a/dashboards/customized/influxdb_v3/config.example.toml b/dashboards/customized/influxdb_v3/config.example.toml new file mode 100644 index 0000000..29aad08 --- /dev/null +++ b/dashboards/customized/influxdb_v3/config.example.toml @@ -0,0 +1,27 @@ +# Example configuration for the InfluxDB rollup plugin. +# All options are optional; default values are shown below. + +# Rollup mode: "hourly" or "monthly" +# mode = "hourly" + +# Raw source table (in the trigger database) +# raw_table = "analytics" + +# Database where aggregates will be written +# agg_database = "analytics_agg" + +# Hourly window size in hours (only used when mode="hourly") +# Must divide 24 evenly. +# window_hours = 6 + +# Backfill window (ISO 8601 timestamps) +# If both are set, the plugin processes this interval instead of the scheduler time. +# start_time = "2026-01-01T00:00:00Z" +# end_time = "2026-01-01T06:00:00Z" + +# Enable verbose logging (SQL queries and written data points) +# verbose = false + +# Local CLI option (not used by scheduled plugin) +# Prevents writes when running via __main__.py +# readonly = true From d5a579d0e2c689becab51f519f9c5ddf8c192b3b Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Mon, 9 Mar 2026 10:49:42 +0000 Subject: [PATCH 49/56] chore: update README --- dashboards/customized/influxdb_v3/README.md | 36 +++++---------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md index 6ebb376..7f3951d 100644 --- a/dashboards/customized/influxdb_v3/README.md +++ b/dashboards/customized/influxdb_v3/README.md @@ -29,6 +29,7 @@ Aggregated data is written into the following **tables** (tables are created aut | `default_agg_chatid` | Request counts grouped by `chat_id` | | `default_agg_month` | Monthly roll-ups used by KPI dashboards | +> [!NOTE] > Tables are **schema-on-write** in InfluxDB 3 — no explicit creation step is required. --- @@ -163,45 +164,24 @@ end_time=2026-01-01T06:00:00Z ```sh influxdb3 update trigger analytics_6h_rollups --disable +influxdb3 update trigger analytics_monthly_rollups --disable ``` -### Step 2: Backfill in deterministic 6-hour windows +### Step 2: Backfill in deterministic windows -* Run the plugin repeatedly for each window -* Windows must not overlap -* Use the same plugin and logic as production +Run the plugin first in `hourly` mode, then in the `monthly` mode since the results of the former are used as inputs by the latter. -Typical approaches: +Set `start_time` and `end_time` parameters spanning the desired windows that are needed to be backfilled. -* one-off Kubernetes Job -* admin CLI loop -* temporary high-frequency trigger +Re-running the same window is safe and idempotent, because each aggregate point is written with a deterministic timestamp and the tags uniquely identify the aggregation dimensions. ### Step 3: Re-enable scheduled triggers ```bash influxdb3 update trigger analytics_6h_rollups --enable +influxdb3 update trigger analytics_monthly_rollups --enable ``` -Because: - -* each aggregate point is written with a deterministic timestamp -* tags uniquely identify the aggregation dimensions - -👉 **Re-running the same window is safe and idempotent.** - ---- - -## Validation - -After triggers are enabled: - -* Query `analytics_agg.default_agg_stats` to confirm 6-hour data -* Query `analytics_agg.default_agg_month` after the first month boundary -* Verify Grafana dashboards point to `analytics_agg` tables - ---- - ## Running trigger manually The plugin script could be ran manually on an InfluxDB instance configured by the environment variables: @@ -211,7 +191,7 @@ The plugin script could be ran manually on an InfluxDB instance configured by th |INFLUX_URL|URL to the InfluxDB to write the analytics data| |INFLUX_API_TOKEN|InfluxDB API Token with the write access to the target database| -Go to script directory and run it with an optional parameter for a TOML configuration file: +Go to script directory and run it with an optional configuration file: ```sh cd dashboards/customized/influxdb_v3 From 9755f916f44bd876f37cac340e36f81389958527 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 10 Mar 2026 12:39:44 +0000 Subject: [PATCH 50/56] feat: remove typing_extension package dependency --- dashboards/customized/influxdb_v3/.gitignore | 3 --- dashboards/customized/influxdb_v3/plugin/__init__.py | 3 +-- dashboards/customized/influxdb_v3/plugin/config.py | 4 +--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/dashboards/customized/influxdb_v3/.gitignore b/dashboards/customized/influxdb_v3/.gitignore index 51325cd..5b6c096 100644 --- a/dashboards/customized/influxdb_v3/.gitignore +++ b/dashboards/customized/influxdb_v3/.gitignore @@ -1,4 +1 @@ -*.txt -*.log config.toml -*.sh diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 4c21d75..5a49c9e 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -52,8 +52,7 @@ import uuid from datetime import datetime, timezone - -from typing_extensions import assert_never +from typing import assert_never from .config import Config, Mode from .influx.client_wrapper import InfluxDBClientWrapper diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index c7356cf..5b1a0dc 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -1,9 +1,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum -from typing import Dict, List - -from typing_extensions import assert_never +from typing import Dict, List, assert_never from .dates import parse_iso_date from .window import Window From cfe33cccbb84e5dfb06bb5437955bcf43b3c5eba Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 10 Mar 2026 16:14:43 +0000 Subject: [PATCH 51/56] chore: moving code around --- .../customized/influxdb_v3/plugin/__main__.py | 1 - .../customized/influxdb_v3/plugin/config.py | 6 ++--- .../influxdb_v3/plugin/influx/write.py | 2 +- .../influxdb_v3/plugin/rollup_hourly.py | 2 +- .../influxdb_v3/plugin/rollup_monthly.py | 2 +- .../influxdb_v3/plugin/utils/concurrency.py | 22 +++++++++++++++++++ .../influxdb_v3/plugin/{ => utils}/dates.py | 0 .../influxdb_v3/plugin/{ => utils}/window.py | 0 .../plugin/{ => utils}/window_roller.py | 0 9 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 dashboards/customized/influxdb_v3/plugin/utils/concurrency.py rename dashboards/customized/influxdb_v3/plugin/{ => utils}/dates.py (100%) rename dashboards/customized/influxdb_v3/plugin/{ => utils}/window.py (100%) rename dashboards/customized/influxdb_v3/plugin/{ => utils}/window_roller.py (100%) diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index efe7512..e3e5915 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -34,7 +34,6 @@ def main() -> None: url=url, token=token, database=database, readonly=readonly ) - # Only reads InfluxDB; doesn't write, so it's safe to use a local client for testing. process_scheduled_call( influxdb3_local=client, call_time=datetime.now(timezone.utc), diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 5b1a0dc..18fc8a0 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -3,9 +3,9 @@ from enum import Enum from typing import Dict, List, assert_never -from .dates import parse_iso_date -from .window import Window -from .window_roller import HourlyRoller, MonthlyRoller, roll_windows +from .utils.dates import parse_iso_date +from .utils.window import Window +from .utils.window_roller import HourlyRoller, MonthlyRoller, roll_windows class Mode(Enum): diff --git a/dashboards/customized/influxdb_v3/plugin/influx/write.py b/dashboards/customized/influxdb_v3/plugin/influx/write.py index b96f977..b7bcdf2 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/write.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/write.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Tuple -from ..dates import parse_iso_date +from ..utils.dates import parse_iso_date from .client_http import HTTPInfluxDBClient from .client_wrapper import InfluxDBClientWrapper from .line_builder import LineBuilder as LineBuilderImpl diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 0723341..0a45985 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -7,7 +7,7 @@ from .config import Config from .influx.client_wrapper import InfluxDBClientWrapper from .influx.write import write_points -from .window import Window +from .utils.window import Window def run_hourly( diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index e482122..e88445d 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -7,7 +7,7 @@ from .config import Config from .influx.client_wrapper import InfluxDBClientWrapper from .influx.write import write_points -from .window import Window +from .utils.window import Window def run_monthly( diff --git a/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py b/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py new file mode 100644 index 0000000..9056dac --- /dev/null +++ b/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py @@ -0,0 +1,22 @@ +from concurrent.futures import ThreadPoolExecutor +from typing import Callable, Iterable, Protocol, TypeVar + +_T = TypeVar("_T") + + +class Exec(Protocol): + def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: ... + + +class SequentialExec: + def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: + return [job() for job in jobs] + + +class ThreadPoolExec: + def __init__(self, max_workers: int = 8): + self._max_workers = max_workers + + def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: + with ThreadPoolExecutor(max_workers=self._max_workers) as ex: + return list(ex.map(lambda job: job(), jobs)) diff --git a/dashboards/customized/influxdb_v3/plugin/dates.py b/dashboards/customized/influxdb_v3/plugin/utils/dates.py similarity index 100% rename from dashboards/customized/influxdb_v3/plugin/dates.py rename to dashboards/customized/influxdb_v3/plugin/utils/dates.py diff --git a/dashboards/customized/influxdb_v3/plugin/window.py b/dashboards/customized/influxdb_v3/plugin/utils/window.py similarity index 100% rename from dashboards/customized/influxdb_v3/plugin/window.py rename to dashboards/customized/influxdb_v3/plugin/utils/window.py diff --git a/dashboards/customized/influxdb_v3/plugin/window_roller.py b/dashboards/customized/influxdb_v3/plugin/utils/window_roller.py similarity index 100% rename from dashboards/customized/influxdb_v3/plugin/window_roller.py rename to dashboards/customized/influxdb_v3/plugin/utils/window_roller.py From 1b41552a3ed78c55ac8611cbcbcede8872b7b454 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Tue, 10 Mar 2026 16:24:49 +0000 Subject: [PATCH 52/56] feat: user sequential executor in the plugin --- dashboards/customized/influxdb_v3/plugin/__init__.py | 3 +++ dashboards/customized/influxdb_v3/plugin/__main__.py | 2 ++ .../influxdb_v3/plugin/influx/client_wrapper.py | 4 ++++ .../customized/influxdb_v3/plugin/rollup_hourly.py | 8 +------- .../customized/influxdb_v3/plugin/rollup_monthly.py | 9 +-------- .../customized/influxdb_v3/plugin/utils/concurrency.py | 6 +++--- 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/__init__.py b/dashboards/customized/influxdb_v3/plugin/__init__.py index 5a49c9e..2c08876 100644 --- a/dashboards/customized/influxdb_v3/plugin/__init__.py +++ b/dashboards/customized/influxdb_v3/plugin/__init__.py @@ -59,12 +59,14 @@ from .influx.types import InfluxDBClient from .rollup_hourly import run_hourly from .rollup_monthly import run_monthly +from .utils.concurrency import Exec def process_scheduled_call( influxdb3_local: InfluxDBClient, call_time: datetime, args: dict | None = None, + exec: Exec | None = None, ): """ InfluxDB 3 scheduled plugin entrypoint. @@ -74,6 +76,7 @@ def process_scheduled_call( client = InfluxDBClientWrapper( client=influxdb3_local, + exec=exec, database=config.input_database, verbose=config.verbose, ).add_prefix(f"[{str(uuid.uuid4())}]") diff --git a/dashboards/customized/influxdb_v3/plugin/__main__.py b/dashboards/customized/influxdb_v3/plugin/__main__.py index e3e5915..2fdc65f 100644 --- a/dashboards/customized/influxdb_v3/plugin/__main__.py +++ b/dashboards/customized/influxdb_v3/plugin/__main__.py @@ -9,6 +9,7 @@ from . import process_scheduled_call from .config import Config from .influx.client_http import HTTPInfluxDBClient +from .utils.concurrency import ThreadPoolExec def _get_env(name: str) -> str: @@ -38,6 +39,7 @@ def main() -> None: influxdb3_local=client, call_time=datetime.now(timezone.utc), args=args, + exec=ThreadPoolExec(), ) diff --git a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py index cbc2275..bf86150 100644 --- a/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py +++ b/dashboards/customized/influxdb_v3/plugin/influx/client_wrapper.py @@ -1,6 +1,7 @@ import json from typing import Any, Dict, List +from ..utils.concurrency import Exec, SequentialExec from .client_http import HTTPInfluxDBClient from .types import InfluxDBClient, LineBuilderProtocol @@ -10,6 +11,7 @@ class InfluxDBClientWrapper(InfluxDBClient): _database: str _verbose: bool _client: InfluxDBClient + exec: Exec def __init__( self, @@ -17,12 +19,14 @@ def __init__( client: InfluxDBClient, database: str, verbose: bool, + exec: Exec | None = None, log_prefix: str = "", ): self._database = database self._log_prefix = log_prefix self._client = client self._verbose = verbose + self.exec = exec or SequentialExec() def add_prefix(self, log_prefix: str) -> "InfluxDBClientWrapper": return InfluxDBClientWrapper( diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py index 0a45985..882d9a4 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_hourly.py @@ -1,6 +1,3 @@ -from __future__ import annotations - -from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from typing import Any, Dict @@ -238,10 +235,7 @@ def run_hourly_window( for name, func in job_defs ] - with ThreadPoolExecutor(max_workers=5) as ex: - futures = [ex.submit(job) for job in jobs] - for fut in as_completed(futures): - fut.result() + client.exec.run(jobs) def _normalize_project_id(rows: list[Dict[str, Any]]) -> list[Dict[str, Any]]: diff --git a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py index e88445d..1e133f0 100644 --- a/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py +++ b/dashboards/customized/influxdb_v3/plugin/rollup_monthly.py @@ -1,6 +1,3 @@ -from __future__ import annotations - -from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from typing import Any, Dict @@ -72,11 +69,7 @@ def run_monthly_window( lambda: client.add_prefix("[user ]").query(user_sql), ] - results = [] - with ThreadPoolExecutor(max_workers=3) as ex: - futures = [ex.submit(job) for job in jobs] - for fut in as_completed(futures): - results.append(fut.result()) + results = client.exec.run(jobs) merged: Dict[str, Any] = {"time": start_s} for result in results: diff --git a/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py b/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py index 9056dac..40a83f4 100644 --- a/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py +++ b/dashboards/customized/influxdb_v3/plugin/utils/concurrency.py @@ -5,11 +5,11 @@ class Exec(Protocol): - def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: ... + def run(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: ... class SequentialExec: - def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: + def run(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: return [job() for job in jobs] @@ -17,6 +17,6 @@ class ThreadPoolExec: def __init__(self, max_workers: int = 8): self._max_workers = max_workers - def run_jobs(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: + def run(self, jobs: Iterable[Callable[[], _T]]) -> list[_T]: with ThreadPoolExecutor(max_workers=self._max_workers) as ex: return list(ex.map(lambda job: job(), jobs)) From eb3eb6503e31ceb612e2da30ba6936ce18abe5ad Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 11 Mar 2026 11:51:50 +0000 Subject: [PATCH 53/56] chore: add comment --- dashboards/customized/influxdb_v3/plugin/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/customized/influxdb_v3/plugin/config.py b/dashboards/customized/influxdb_v3/plugin/config.py index 18fc8a0..dde78f5 100644 --- a/dashboards/customized/influxdb_v3/plugin/config.py +++ b/dashboards/customized/influxdb_v3/plugin/config.py @@ -17,9 +17,9 @@ class Mode(Enum): class Config: mode: Mode agg_database: str - raw_table: str - window_hours: int + raw_table: str # used only in mode=hourly + window_hours: int # used only in mode=hourly start_time: datetime | None end_time: datetime | None From 129bbb24323c2025e69d173bd3e169b994c051a2 Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 11 Mar 2026 13:04:45 +0000 Subject: [PATCH 54/56] feat: add migration guide --- .../influx2_to_influx3_migration_guide.md | 63 +++++++++++++++++++ dashboards/customized/influxdb_v3/README.md | 4 +- 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 dashboards/customized/influx2_to_influx3_migration_guide.md diff --git a/dashboards/customized/influx2_to_influx3_migration_guide.md b/dashboards/customized/influx2_to_influx3_migration_guide.md new file mode 100644 index 0000000..e1036b3 --- /dev/null +++ b/dashboards/customized/influx2_to_influx3_migration_guide.md @@ -0,0 +1,63 @@ +# Migrating Data from InfluxDB2 to InfluxDB3 + +## Overview + +This document describes the recommended procedure for migrating analytics data from **InfluxDB2** to **InfluxDB3**. + +Only the **`default` bucket** needs to be migrated. + +All other buckets contain **derived / aggregated data** and should **not be migrated**. Instead, they should be recreated using the **backfilling mode** of the [aggregation plugin](./influxdb_v3/README.md). + + +The migration process consists of: + +1. Exporting the `default` bucket from **InfluxDB2** as **Line Protocol** +2. Splitting the export into chunks +3. Uploading the chunks in parallel to **InfluxDB3** + +The instructions below assume that **InfluxDB2** is running using the Bitnami container image: + +```yaml +image: + registry: bitnamilegacy + repository: influxdb + tag: 2.7.11-debian-12-r20 +``` + +In this image the storage engine resides at: + +``` +/bitnami/influxdb +``` + +--- + +## Migration script + +```sh +set -euo pipefail + +# Get the bucket id of the "default" bucket +BUCKET_ID=$(influx bucket list --org dial --json \ + | jq -r '.[] | select(.name == "default") | .id') + +echo "Bucket id: ${BUCKET_ID}" + +cd /tmp + +# Export and split into chunks 500K datapoints each +influxd inspect export-lp \ + --bucket-id "${BUCKET_ID}" \ + --engine-path /bitnami/influxdb \ + --output-path - \ + | split -l 500000 - chunk_ + +# Upload chunks to InfluxDB 3 in parallel +find . -maxdepth 1 -name 'chunk_*' -print0 \ + | xargs -0 -n1 -P4 -I{} \ + curl --fail -sS \ + -X POST "${INFLUXDB3_URL}/api/v3/write_lp?db=default&precision=ns" \ + -H "Authorization: Bearer ${INFLUXDB3_TOKEN}" \ + -H "Content-Type: text/plain; charset=utf-8" \ + --data-binary @{} +``` diff --git a/dashboards/customized/influxdb_v3/README.md b/dashboards/customized/influxdb_v3/README.md index 7f3951d..7271f00 100644 --- a/dashboards/customized/influxdb_v3/README.md +++ b/dashboards/customized/influxdb_v3/README.md @@ -7,7 +7,7 @@ These aggregates are used by Grafana dashboards for AI DIAL Realtime Analytics. ## Database Layout -We use **two databases**: +We use two databases: | Database | Purpose | | --------------- | ---------------------------------------------------------- | @@ -182,7 +182,7 @@ influxdb3 update trigger analytics_6h_rollups --enable influxdb3 update trigger analytics_monthly_rollups --enable ``` -## Running trigger manually +## Running the plugin manually The plugin script could be ran manually on an InfluxDB instance configured by the environment variables: From db4757aee9840ad9553df8f5c9b8e711caa232cf Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Wed, 11 Mar 2026 17:12:56 +0000 Subject: [PATCH 55/56] chore: fix readme --- .../customized/influx2_to_influx3_migration_guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/customized/influx2_to_influx3_migration_guide.md b/dashboards/customized/influx2_to_influx3_migration_guide.md index e1036b3..19ee439 100644 --- a/dashboards/customized/influx2_to_influx3_migration_guide.md +++ b/dashboards/customized/influx2_to_influx3_migration_guide.md @@ -45,19 +45,19 @@ echo "Bucket id: ${BUCKET_ID}" cd /tmp -# Export and split into chunks 500K datapoints each +# Export and split into chunks 1Mb at most each influxd inspect export-lp \ --bucket-id "${BUCKET_ID}" \ --engine-path /bitnami/influxdb \ --output-path - \ - | split -l 500000 - chunk_ + | split -d -C 1M - chunk_ # Upload chunks to InfluxDB 3 in parallel find . -maxdepth 1 -name 'chunk_*' -print0 \ | xargs -0 -n1 -P4 -I{} \ curl --fail -sS \ - -X POST "${INFLUXDB3_URL}/api/v3/write_lp?db=default&precision=ns" \ - -H "Authorization: Bearer ${INFLUXDB3_TOKEN}" \ + -X POST "${INFLUX_URL}/api/v3/write_lp?db=default&precision=nanosecond&accept_partial=false" \ + -H "Authorization: Bearer ${INFLUX_API_TOKEN}" \ -H "Content-Type: text/plain; charset=utf-8" \ --data-binary @{} ``` From 1fcbe2d484da9ad7d169fd20d62427c8aab4685e Mon Sep 17 00:00:00 2001 From: Anton Dubovik Date: Thu, 12 Mar 2026 13:54:18 +0000 Subject: [PATCH 56/56] chore: improve README --- .../customized/influx2_to_influx3_migration_guide.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dashboards/customized/influx2_to_influx3_migration_guide.md b/dashboards/customized/influx2_to_influx3_migration_guide.md index 19ee439..e1255f6 100644 --- a/dashboards/customized/influx2_to_influx3_migration_guide.md +++ b/dashboards/customized/influx2_to_influx3_migration_guide.md @@ -34,6 +34,16 @@ In this image the storage engine resides at: ## Migration script +Set the environment variables pointing to the InfluxDB 3 instance. + +```init +INFLUX_URL= +INFLUX_API_TOKEN= +``` + +Run the migration script on the InfluxDB 2 instance. +The script exports the `default` bucket into multiple files and sends them to the InfluxDB 3 instance. + ```sh set -euo pipefail