From a475408f18392879b77ad785ea883c01f294ab85 Mon Sep 17 00:00:00 2001 From: KATHIRVEL Date: Thu, 30 Jul 2026 16:25:21 +0530 Subject: [PATCH 1/3] Add Milvus vector database monitoring plugin --- milvus/README.md | 285 +++++++++++++++++++++ milvus/milvus_monitor_v2.cfg | 8 + milvus/milvus_monitor_v2.py | 482 +++++++++++++++++++++++++++++++++++ 3 files changed, 775 insertions(+) create mode 100644 milvus/README.md create mode 100644 milvus/milvus_monitor_v2.cfg create mode 100644 milvus/milvus_monitor_v2.py diff --git a/milvus/README.md b/milvus/README.md new file mode 100644 index 00000000..fcc9cff4 --- /dev/null +++ b/milvus/README.md @@ -0,0 +1,285 @@ +# Milvus Monitoring + +Milvus is an open-source vector database built for high-performance similarity search and AI applications. This plugin collects health, performance, and resource metrics from a Milvus deployment's Prometheus `/metrics` endpoint and reports them to Site24x7 as a custom plugin monitor. + +## Prerequisites + +- Download and install the latest version of the Site24x7 Server Monitoring agent on the server where you plan to run the plugin. +- Python 3 must be available on the host running the plugin. +- The Milvus instance must expose its Prometheus metrics endpoint (enabled by default on most Milvus deployments) and it must be reachable from the plugin host. + +### Metrics Endpoint Access + +By default, Milvus exposes Prometheus-formatted metrics at: + +``` +http://:9091/metrics +``` + +Confirm the endpoint is reachable before installing the plugin: + +```bash +curl http://localhost:9091/metrics +``` + +If Milvus is running remotely, in a container, or behind a different port mapping, update the `HOSTNAME` and `PORT` values at the top of `milvus.py` (see Configuration below). + +## Plugin Installation + +### Linux + +1. Create a directory named `milvus`. + + ```bash + mkdir milvus + cd milvus/ + ``` + +2. Download the plugin file and place it under the `milvus` directory. + + ```bash + wget https://raw.githubusercontent.com/site24x7/plugins/milvus/milvus/milvus.py + ``` + +3. Execute the script to verify it returns valid JSON output. + + ```bash + python3 milvus.py + ``` + +4. Move the `milvus` directory under the Site24x7 Linux Agent plugin directory. + + ```bash + mv milvus /opt/site24x7/monagent/plugins/ + ``` + +### Windows + +1. Create a directory named `milvus`. + +2. Download `milvus.py` and place it under the `milvus` directory. + +3. Since it's a Python plugin, follow the steps in [this link](https://www.site24x7.com/help/admin/monitoring-agent/windows-plugin-monitor.html) to run Python plugins on a Windows server. + +4. Execute the script in `cmd` to verify it returns valid JSON output. + + ```cmd + python milvus.py + ``` + +5. Move the `milvus` folder under the Site24x7 Windows Agent plugin directory. + + ``` + C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins + ``` + +The agent will automatically execute the plugin within five minutes, and you can view the plugin monitor under **Site24x7 > Plugins > Plugin Integrations**. + +## Configuration + +The plugin connects using constants defined at the top of `milvus.py`: + +```python +HOSTNAME = "localhost" +PORT = "9091" +TIMEOUT = 10 +``` + +Update `HOSTNAME` and `PORT` to match your Milvus deployment before moving the plugin into the agent's plugin directory. + +## Diagnostics + +To list every raw metric name exposed by the Milvus `/metrics` endpoint (useful for troubleshooting or extending the plugin), run: + +```bash +python3 milvus.py --list-metrics +``` + +## Supported Metrics + +### Cluster Health + +| Name | Description | +|---|---| +| Milvus_Status | Whether the plugin could successfully reach and parse the Milvus metrics endpoint (1 = up, 0 = down) | +| QueryNodes | Number of QueryNodes registered with the QueryCoord | +| Collections_Loaded | Number of collections currently loaded into memory across QueryNodes | +| DataNodes_Count | Number of DataNodes registered with the DataCoord | +| IndexNode_Count | Number of IndexNodes registered with the DataCoord | +| Proxy_Nodes_Count | Number of active proxy nodes registered with the RootCoord | +| DML_Channels_Count | Number of DML (data manipulation) channels managed by the RootCoord | + +### Component Availability + +| Name | Description | +|---|---| +| RootCoord_Collections | Total number of collections tracked by the RootCoord | +| RootCoord_Partitions | Total number of partitions tracked by the RootCoord | +| Process_Open_FDs | Number of file descriptors currently open by the Milvus process | +| Process_Max_FDs | Maximum number of file descriptors allowed for the Milvus process | + +### Search + +| Name | Description | +|---|---| +| Search_Vectors | Total number of vectors submitted across all search requests | +| Search_Requests | Total number of search-type requests processed | +| Search_Latency.ms | Average end-to-end latency for search requests, in milliseconds | +| Search_Queue_Latency.ms | Average time search requests spend waiting in the QueryNode queue | +| Search_TopK_Avg | Average TopK value requested across search operations | +| Search_WaitResult_Latency.ms | Average time the proxy waits for search results from QueryNodes | +| Search_DecodeResult_Latency.ms | Average time spent decoding search results on the proxy | + +### Query + +| Name | Description | +|---|---| +| Query_Latency.ms | Average end-to-end latency for query (non-search) requests | +| Query_Requests | Total number of query-type requests processed | +| Loaded_Segments | Number of sealed segments currently loaded in QueryNodes | +| Query_Queue_Latency.ms | Average time query requests spend waiting in the proxy queue | +| Query_Reduce_Latency.ms | Average time spent reducing/merging query results on QueryNodes | +| Query_CoreSearch_Latency.ms | Average latency of the internal core search operation | + +### QueryNode Performance + +| Name | Description | +|---|---| +| QN_Entity_Count | Number of entities currently loaded on QueryNodes | +| QN_Entity_Memory.MB | Memory consumed by loaded entities on QueryNodes, in MB | +| QN_Flowgraph_Count | Number of active flowgraphs on QueryNodes | +| QN_DML_Channel_Count | Number of DML channels subscribed to by QueryNodes | +| QN_ReadTask_Concurrency | Current concurrency level for read tasks on QueryNodes | +| QN_ReadTask_Ready_Queue | Number of read tasks ready to be executed | +| QN_ReadTask_Unsolved_Queue | Number of read tasks still waiting to be scheduled | +| QN_LoadSegment_Concurrency | Current concurrency level for segment loading operations | +| QN_LoadSegment_Latency.ms | Average time taken to load a segment into a QueryNode | +| QN_MsgDispatcher_Lag.ms | Lag of the QueryNode message dispatcher behind the latest timestamp | + +### Index Performance + +| Name | Description | +|---|---| +| Index_Build_Latency.ms | Average time taken by IndexNodes to build an index | +| Index_Save_Latency.ms | Average time taken to persist a built index to storage | +| Index_Task_Count | Number of index build tasks currently tracked | +| Index_TaskQueue_Latency.ms | Average time index tasks spend waiting in queue before execution | +| Index_Load_Latency.ms | Average time taken to load an index into memory | +| Index_DataCoord_Tasks | Number of index-related tasks tracked by the DataCoord | + +### Data Ingestion + +| Name | Description | +|---|---| +| Insert_Size.MB | Total size of data received via insert requests, in MB | +| Insert_Request_Count | Total number of insert requests processed by the proxy | +| Delete_Vector_Count | Total number of vectors removed via delete operations | +| Flush_Request_Count | Total number of flush requests processed by DataNodes | +| Flushed_Rows_Count | Total number of rows persisted to storage via flush operations | +| Flushed_Bytes.MB | Total size of data persisted via flush operations, in MB | +| Mutation_Send_Latency.ms | Average latency for sending mutation (insert/delete) requests | + +### DataNode Performance + +| Name | Description | +|---|---| +| DN_Flowgraph_Count | Number of active flowgraphs on DataNodes | +| DN_Consume_Bytes | Total bytes consumed from the message queue by DataNodes | +| DN_Consume_Msg_Count | Total number of messages consumed from the message queue by DataNodes | +| DN_EncodeBuffer_Latency.ms | Average time taken to encode insert buffers before flush | +| DN_Save_Latency.ms | Average time taken by DataNodes to save data to storage | +| DN_AutoFlush_Op_Count | Number of automatic buffer-flush operations triggered | + +### Storage Usage + +| Name | Description | +|---|---| +| Binlog_Size.MB | Total size of stored binlog (write-ahead log) files, in MB | +| Index_Files_Size.MB | Total size of stored index files, in MB | +| Storage_KV_Size.MB | Size of data stored in the underlying key-value storage layer | +| Storage_Op_Count | Total number of operations performed against the storage layer | +| Storage_Request_Latency.ms | Average latency of requests to the storage layer | + +### Memory Usage + +| Name | Description | +|---|---| +| Heap_Usage.MB | Heap memory currently in use by the Milvus process, in MB | +| Heap_Idle.MB | Heap memory currently idle but not yet released to the OS | +| Heap_Sys.MB | Total heap memory obtained from the OS | +| Resident_Memory.MB | Resident memory (RSS) used by the Milvus process | +| Virtual_Memory.MB | Total virtual memory used by the Milvus process | +| MMap_InUse.MB | Memory-mapped space currently in use by Milvus | +| Next_GC_Threshold.MB | Heap size target that will trigger the next garbage collection cycle | + +### Go Runtime + +| Name | Description | +|---|---| +| Threads | Total number of OS threads used by the Milvus process | +| Goroutines | Number of active Go routines running in the Milvus process | +| Go_Threads_Count | Number of OS threads created by the Go runtime | +| GC_Duration_Avg.ms | Average duration of garbage collection cycles | +| GC_Cycle_Count | Total number of garbage collection cycles completed | +| Mallocs_Count | Total number of memory allocations made by the Go runtime | + +### Message Queue + +| Name | Description | +|---|---| +| Consumer_Lag.ms | Lag between DataNode message consumption and the latest published timestamp | +| MQ_Consumer_Count | Number of active consumers subscribed to the message queue | +| MsgStream_Op_Count | Total number of message stream operations performed | +| MsgStream_Request_Latency.ms | Average latency of message stream requests | +| Proxy_TT_Lag.ms | Lag between the proxy's timestamp and the latest allocated timestamp | + +### Proxy Performance + +| Name | Description | +|---|---| +| Proxy_Request_Count | Total number of requests handled by the proxy, across all request types | +| Proxy_Request_Latency.ms | Average latency for requests handled by the proxy | +| Proxy_RateLimit_Count | Number of requests rejected due to rate limiting | +| Proxy_SendBytes.MB | Total bytes sent by the proxy to downstream components | +| Proxy_ApplyPK_Latency.ms | Average latency for primary key allocation on the proxy | +| Proxy_ApplyTimestamp_Latency.ms | Average latency for timestamp allocation on the proxy | +| Proxy_MsgStream_Obj_Num | Number of message stream objects currently held by the proxy | + +### Cache Performance + +| Name | Description | +|---|---| +| Proxy_Cache_Hit_Count | Number of cache hits recorded on the proxy | +| Proxy_Cache_Update_Latency.ms | Average latency for updating the proxy's internal cache | + +### Vector Index Algorithms + +| Name | Description | +|---|---| +| HNSW_BitsetRatio_Avg | Average ratio of filtered (bitset) entities during HNSW search | +| HNSW_SearchHops_Avg | Average number of graph hops traversed during HNSW search | +| IVF_Search_Count | Total number of IVF index search operations performed | +| DiskANN_SearchHops_Avg | Average number of hops traversed during DiskANN search | +| DiskANN_RangeSearchIters_Avg | Average number of iterations performed during DiskANN range search | +| DiskANN_BitsetRatio_Avg | Average ratio of filtered (bitset) entities during DiskANN search | + +### Reliability + +| Name | Description | +|---|---| +| Write_Blocks | Number of times writes were force-denied by the RootCoord (e.g. due to quota limits) | +| RootCoord_DDL_Request_Count | Total number of DDL (data definition) requests processed by the RootCoord | +| RootCoord_DDL_Latency.ms | Average latency for DDL requests on the RootCoord | +| RootCoord_SyncTimetick_Latency.ms | Average latency for timestamp synchronization across the cluster | +| QN_Disk_Cache_Evict_Count | Number of entries evicted from the QueryNode disk cache | +| QN_Disk_Cache_Load_Count | Number of entries loaded into the QueryNode disk cache | +| QN_Segment_Access_Wait_Count | Number of times a query had to wait for a segment to be loaded from disk cache | + +### Storage & Data Overview + +| Name | Description | +|---|---| +| Vectors | Total number of vector rows stored across the cluster | +| Segments | Total number of segments currently tracked by QueryNodes | +| Metrics_Total | Total number of individual metric samples parsed from the `/metrics` endpoint (diagnostic counter) | +| CPU_Time.seconds | Cumulative CPU time consumed by the Milvus process, in seconds | \ No newline at end of file diff --git a/milvus/milvus_monitor_v2.cfg b/milvus/milvus_monitor_v2.cfg new file mode 100644 index 00000000..9bfac06b --- /dev/null +++ b/milvus/milvus_monitor_v2.cfg @@ -0,0 +1,8 @@ +[global_configurations] +use_agent_python=1 + +[milvus] +encrypted.host=8gob+RFoPLO1X7KpvHCZ3A== +encrypted.port=WDnQHa9uEA4sayqW2xUVwg== +encrypted.timeout=dwGnbKGOZO7+WjA7/y66Lg== + diff --git a/milvus/milvus_monitor_v2.py b/milvus/milvus_monitor_v2.py new file mode 100644 index 00000000..5975c31e --- /dev/null +++ b/milvus/milvus_monitor_v2.py @@ -0,0 +1,482 @@ +import argparse +import configparser +import json +import os +import re +import ssl +import sys +import tempfile +import time +import urllib.error +import urllib.request + +# Global Constants & Defaults +PLUGIN_VERSION = 10 # Bumped version to force Site24x7 template refresh +HEARTBEAT = "true" +REQUEST_TIMEOUT = 5 + +LINE_RE = re.compile( + r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(\{[^}]*\})?\s+(-?[0-9.eE+\-]+|NaN|\+Inf|-Inf)\s*$' +) +LABEL_RE = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') + +UNITS = { + "Response Time": "ms", + "CPU Percent": "%", + "Resident Memory": "MB", + "Virtual Memory": "MB", + "Heap Usage": "MB", + "Heap Idle": "MB", + "Heap System": "MB", + "Memory Mapped Usage": "MB", + "Next GC Threshold": "MB", + "Entity Memory Usage": "MB", + "System Threads": "count", + "Active Goroutines": "count", + "Total Metrics Count": "count", + "Search Latency": "ms", + "Query Latency": "ms", + "Query Queue Latency": "ms", + "Query Reduce Latency": "ms", + "Core Search Latency": "ms", + "Search Queue Latency": "ms", + "Wait Result Latency": "ms", + "Decode Result Latency": "ms", + "Segment Load Latency": "ms", + "Dispatcher Time Lag": "ms", + "Index Build Latency": "ms", + "Index Save Latency": "ms", + "Index Queue Latency": "ms", + "Index Load Latency": "ms", + "Mutation Latency": "ms", + "Encode Buffer Latency": "ms", + "Data Save Latency": "ms", + "Storage Request Latency": "ms", + "GC Duration Average": "ms", + "Message Stream Latency": "ms", + "Proxy Time Lag": "ms", + "Proxy Request Latency": "ms", + "Primary Key Latency": "ms", + "Timestamp Latency": "ms", + "Cache Update Latency": "ms", + "DDL Request Latency": "ms", + "Sync Timetick Latency": "ms", + "Consumer Time Lag": "ms", + "Flushed Data Size": "MB", + "Binlog Size": "MB", + "Index Size": "MB", + "Key Value Storage Size": "MB", + "Proxy Sent Data": "MB", + "Insert Data Size": "MB", + "Query Nodes": "nodes", + "Data Nodes": "nodes", + "Index Nodes": "nodes", + "Proxy Nodes": "nodes", + "Collections Loaded": "count", + "Total Collections": "count", + "Total Partitions": "count", + "Segments": "count", + "Loaded Segments": "count", + "Vectors": "count", + "Search Vectors": "count", + "Search Requests": "reqs", + "Query Requests": "reqs", + "Insert Requests": "reqs", + "Search Top K Avg": "count", + "Active Index Tasks": "tasks", + "Deleted Vectors": "count", + "Flush Requests": "reqs", + "Flushed Rows": "rows", + "Consumed Messages": "msgs", + "Consumed Bytes": "bytes", + "Storage Operations": "ops", + "Write Denials": "count", + "Open File Descriptors": "fds", + "Max File Descriptors": "fds", +} + +TABS = { + "CPU & Memory": { + "order": 1, + "tablist": [ + "CPU Percent", + "Heap Usage", + "Heap Idle", + "Heap System", + "Resident Memory", + "Virtual Memory", + "System Threads", + "Active Goroutines", + ], + }, + "Query Performance": { + "order": 2, + "tablist": [ + "Query Latency", + "Query Requests", + "Query Queue Latency", + "Query Reduce Latency", + "Search Latency", + "Search Requests", + "Search Vectors", + "Search Top K Avg", + ], + }, + "Index Performance": { + "order": 3, + "tablist": [ + "Index Build Latency", + "Index Load Latency", + "Index Save Latency", + "Index Size", + "Active Index Tasks", + "Index Queue Latency", + ], + }, + "Data Ingestion": { + "order": 4, + "tablist": [ + "Insert Requests", + "Insert Data Size", + "Deleted Vectors", + "Flush Requests", + "Flushed Rows", + "Consumed Messages", + "Consumed Bytes", + ], + }, + "Cluster Health": { + "order": 5, + "tablist": [ + "Query Nodes", + "Data Nodes", + "Index Nodes", + "Proxy Nodes", + "Collections Loaded", + "Segments", + "Vectors", + ], + }, + "Storage": { + "order": 6, + "tablist": [ + "Binlog Size", + "Key Value Storage Size", + "Storage Operations", + "Storage Request Latency", + "Memory Mapped Usage", + ], + }, +} + +METRIC_SPECS = [ + {"key": "Data Nodes", "source": "milvus_datacoord_datanode_num", "kind": "gauge"}, + {"key": "Index Nodes", "source": "milvus_datacoord_index_node_num", "kind": "gauge"}, + {"key": "Proxy Nodes", "source": "milvus_rootcoord_proxy_num", "kind": "gauge"}, + {"key": "Total Collections", "source": "milvus_rootcoord_collection_num", "kind": "gauge"}, + {"key": "Total Partitions", "source": "milvus_rootcoord_partition_num", "kind": "gauge"}, + {"key": "Open File Descriptors", "source": "process_open_fds", "kind": "gauge"}, + {"key": "Max File Descriptors", "source": "process_max_fds", "kind": "gauge"}, + {"key": "Query Queue Latency", "source": "milvus_proxy_req_in_queue_latency", "kind": "hist_ms"}, + {"key": "Query Reduce Latency", "source": "milvus_querynode_sq_reduce_latency", "kind": "hist_ms"}, + {"key": "Core Search Latency", "source": "milvus_internal_core_search_latency", "kind": "hist_us_ms"}, + {"key": "Search Queue Latency", "source": "milvus_querynode_sq_queue_latency", "kind": "hist_ms"}, + {"key": "Search Top K Avg", "source": "milvus_search_topk", "kind": "hist_raw"}, + {"key": "Wait Result Latency", "source": "milvus_proxy_sq_wait_result_latency", "kind": "hist_ms"}, + {"key": "Decode Result Latency", "source": "milvus_proxy_sq_decode_result_latency", "kind": "hist_ms"}, + {"key": "Entity Memory Usage", "source": "milvus_querynode_entity_size", "kind": "gauge_mb"}, + {"key": "Segment Load Latency", "source": "milvus_querynode_load_segment_latency", "kind": "hist_ms"}, + {"key": "Dispatcher Time Lag", "source": "milvus_querynode_msg_dispatcher_tt_lag_ms", "kind": "gauge"}, + {"key": "Index Build Latency", "source": "milvus_indexnode_build_index_latency", "kind": "hist_ms"}, + {"key": "Index Save Latency", "source": "milvus_indexnode_save_index_latency", "kind": "hist_ms"}, + {"key": "Active Index Tasks", "source": "milvus_indexnode_index_task_count", "kind": "gauge"}, + {"key": "Index Queue Latency", "source": "milvus_indexnode_index_task_latency_in_queue", "kind": "hist_ms"}, + {"key": "Index Load Latency", "source": "milvus_load_latency", "kind": "hist_ms"}, + {"key": "Insert Requests", "source": "milvus_proxy_req_count", "kind": "gauge", "labels": {"rpc_type": "Insert"}}, + {"key": "Deleted Vectors", "source": "milvus_proxy_delete_vectors_count", "kind": "gauge"}, + {"key": "Flush Requests", "source": "milvus_datanode_flush_req_count", "kind": "gauge"}, + {"key": "Flushed Rows", "source": "milvus_datanode_flushed_data_rows_count", "kind": "gauge"}, + {"key": "Flushed Data Size", "source": "milvus_datanode_flushed_data_size_count", "kind": "gauge_mb"}, + {"key": "Mutation Latency", "source": "milvus_proxy_mutation_send_latency", "kind": "hist_ms"}, + {"key": "Consumed Bytes", "source": "milvus_datanode_consume_bytes_count", "kind": "gauge"}, + {"key": "Consumed Messages", "source": "milvus_datanode_consume_msg_count", "kind": "gauge"}, + {"key": "Encode Buffer Latency", "source": "milvus_datanode_encode_buffer_latency", "kind": "hist_ms"}, + {"key": "Data Save Latency", "source": "milvus_datanode_save_latency", "kind": "hist_ms"}, + {"key": "Binlog Size", "source": "milvus_datacoord_stored_binlog_size", "kind": "gauge_mb"}, + {"key": "Index Size", "source": "milvus_datacoord_stored_index_files_size", "kind": "gauge_mb"}, + {"key": "Key Value Storage Size", "source": "milvus_storage_kv_size", "kind": "gauge_mb"}, + {"key": "Storage Operations", "source": "milvus_storage_op_count", "kind": "gauge"}, + {"key": "Storage Request Latency", "source": "milvus_storage_request_latency", "kind": "gauge"}, + {"key": "Heap Idle", "source": "go_memstats_heap_idle_bytes", "kind": "gauge_mb"}, + {"key": "Heap System", "source": "go_memstats_heap_sys_bytes", "kind": "gauge_mb"}, + {"key": "Resident Memory", "source": "process_resident_memory_bytes", "kind": "gauge_mb"}, + {"key": "Virtual Memory", "source": "process_virtual_memory_bytes", "kind": "gauge_mb"}, + {"key": "Memory Mapped Usage", "source": "milvus_internal_mmap_in_used_space_bytes", "kind": "gauge_mb"}, + {"key": "Next GC Threshold", "source": "go_memstats_next_gc_bytes", "kind": "gauge_mb"}, + {"key": "GC Duration Average", "source": "go_gc_duration_seconds", "kind": "hist_sec_ms"}, +] + +def get_error_payload(error_message): + return { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "status": 0, + "msg": error_message, + "units": UNITS, + "tabs": TABS, + } + +def timed_get(url, ssl_verify=True): + try: + start = time.time() + req = urllib.request.Request(url, headers={'User-Agent': 'Site24x7-Plugin'}) + ctx = ssl.create_default_context() + if not ssl_verify: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT, context=ctx) as response: + elapsed_ms = round((time.time() - start) * 1000, 2) + content = response.read().decode('utf-8', errors='replace') + return response.status == 200, elapsed_ms, content + except Exception: + return False, 0, None + +def parse_metrics(text): + series = {} + if not text: + return series + + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + + match = LINE_RE.match(line) + if not match: + continue + + name, labels_text, value_text = match.groups() + try: + value = float(value_text) + if value != value: # Check for NaN + continue + except Exception: + continue + + labels = {} + if labels_text: + for k, v in LABEL_RE.findall(labels_text): + labels[k] = v + + series.setdefault(name, []).append({"labels": labels, "value": value}) + + return series + +def get_sum(series, metric, labels=None): + total = 0.0 + for item in series.get(metric, []): + if labels: + if not all(item["labels"].get(k) == v for k, v in labels.items()): + continue + total += item["value"] + return total + +def histogram_average(series, metric): + total = get_sum(series, metric + "_sum") + count = get_sum(series, metric + "_count") + if count > 0: + return round((total / count) * 0.001, 2) + return 0 + +def histogram_stat(series, metric_base, labels=None, scale=1.0, ndigits=2): + total = get_sum(series, metric_base + "_sum", labels) + count = get_sum(series, metric_base + "_count", labels) + if count > 0: + return round((total / count) * scale, ndigits) + return 0 + +def bytes_to_mb(value): + if not value: + return 0 + return round(value / (1024 * 1024), 2) + +def compute_new_metrics(series): + new_values = {} + for spec in METRIC_SPECS: + key = spec["key"] + source = spec["source"] + kind = spec["kind"] + labels = spec.get("labels") + + try: + if kind == "gauge": + value = round(get_sum(series, source, labels)) + elif kind == "gauge_mb": + value = bytes_to_mb(get_sum(series, source, labels)) + elif kind == "hist_ms": + value = histogram_stat(series, source, labels, scale=1.0) + elif kind == "hist_us_ms": + value = histogram_stat(series, source, labels, scale=0.001) + elif kind == "hist_sec_ms": + value = histogram_stat(series, source, labels, scale=1000.0) + elif kind == "hist_raw": + value = histogram_stat(series, source, labels, scale=1.0) + else: + value = 0 + except Exception: + value = 0 + + new_values[key] = value + return new_values + +CPU_STATE_FILE = os.path.join(tempfile.gettempdir(), "milvus_cpu_state.json") + +def get_cpu_utilization_percent(current_cpu_seconds, cores=1): + now = time.time() + previous_time, previous_cpu = None, None + + try: + with open(CPU_STATE_FILE, "r") as f: + state = json.load(f) + previous_time = state.get("timestamp") + previous_cpu = state.get("cpu_seconds") + except Exception: + pass + + try: + with open(CPU_STATE_FILE, "w") as f: + json.dump({"timestamp": now, "cpu_seconds": current_cpu_seconds}, f) + except Exception: + pass + + if previous_time is None or previous_cpu is None: + return 0 + + delta_time = now - previous_time + delta_cpu = current_cpu_seconds - previous_cpu + + if delta_cpu < 0 or delta_time <= 0: + return 0 + + cores = cores if cores and cores > 0 else 1 + return round(((delta_cpu / delta_time) * 100) / cores, 2) + +def metricCollector(param): + host = param.get("host", "localhost") + port = param.get("port", "9091") + ssl_enabled = str(param.get("ssl", "false")).lower() == "true" + ssl_verify_val = str(param.get("ssl_verify", "true")).lower() == "true" + + protocol = "https" if ssl_enabled else "http" + metrics_url = f"{protocol}://{host}:{port}/metrics" + + status_ok, response_time_ms, raw_text = timed_get(metrics_url, ssl_verify=ssl_verify_val) + + if not status_ok or not raw_text: + return get_error_payload(f"Failed to reach Milvus Prometheus Endpoint at {metrics_url}") + + series = parse_metrics(raw_text) + + cpu_seconds = get_sum(series, "process_cpu_seconds_total") + cpu_percent = get_cpu_utilization_percent(cpu_seconds, cores=os.cpu_count() or 1) + threads = get_sum(series, "go_threads") or get_sum(series, "milvus_thread_num") + + payload = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "status": 1, + "Response Time": response_time_ms, + "Total Metrics Count": sum(len(v) for v in series.values()), + "Query Nodes": round(get_sum(series, "milvus_querycoord_querynode_num")), + "Collections Loaded": round(get_sum(series, "milvus_querynode_collection_num")), + "Search Vectors": round(get_sum(series, "milvus_proxy_search_vectors_count")), + "Search Requests": round( + get_sum(series, "milvus_proxy_collection_sq_latency_count", {"query_type": "search"}) + ), + "Search Latency": histogram_average(series, "milvus_proxy_collection_sq_latency"), + "Vectors": round(get_sum(series, "milvus_datacoord_stored_rows_num")), + "Segments": round(get_sum(series, "milvus_querynode_segment_num")), + "Insert Data Size": round(get_sum(series, "milvus_proxy_receive_bytes_count") / (1024 * 1024), 2), + "Query Latency": histogram_average(series, "milvus_proxy_collection_sq_latency"), + "Query Requests": round( + get_sum(series, "milvus_proxy_collection_sq_latency_count", {"query_type": "query"}) + ), + "Loaded Segments": round( + get_sum(series, "milvus_querynode_segment_num", {"segment_state": "Sealed"}) + ), + "CPU Percent": cpu_percent, + "System Threads": round(threads), + "Active Goroutines": round(get_sum(series, "go_goroutines")), + "Heap Usage": round(get_sum(series, "go_memstats_heap_inuse_bytes") / (1024 * 1024), 2), + "Write Denials": round(get_sum(series, "milvus_rootcoord_force_deny_writing_counter")), + "Consumer Time Lag": round(get_sum(series, "milvus_datanode_consume_tt_lag_ms")), + "units": UNITS, + "tabs": TABS, + } + + payload.update(compute_new_metrics(series)) + return payload + +def load_cfg_parameters(cfg_path=None): + """ + Reads parameters from a .cfg file if found. + Checks the explicitly passed file path, or falls back to looking for a file + with the same base name as this script in the same directory. + """ + config_params = {} + + if not cfg_path: + script_dir = os.path.dirname(os.path.abspath(__file__)) + script_name = os.path.splitext(os.path.basename(__file__))[0] + cfg_path = os.path.join(script_dir, f"{script_name}.cfg") + + if os.path.exists(cfg_path): + config = configparser.ConfigParser() + try: + config.read(cfg_path) + # Scan all common Site24x7 sections for configurations + for section in ["global_configurations", "display", "plugin"]: + if config.has_section(section): + for key, value in config.items(section): + # Clean key names to avoid encrypted prefixes or unwanted spaces + clean_key = key.split(".")[-1] + config_params[clean_key] = value + except Exception: + pass # Fallback gracefully to default parameters if read fails + + return config_params + +def run(param): + try: + return metricCollector(param) + except Exception as e: + return get_error_payload(f"Plugin Execution Error: {str(e)}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", help="Host Name", nargs="?", default=None) + parser.add_argument("--port", help="Metrics Port", nargs="?", default=None) + parser.add_argument("--ssl", help="Use SSL/HTTPS", default=None) + parser.add_argument("--ssl_verify", help="SSL Verification", default=None) + parser.add_argument("--name", help="Plugin Name", nargs="?", default=None) + parser.add_argument("--cfg_file", help="Custom path to .cfg file", default=None) + + args, _ = parser.parse_known_args() + + # 1. Read configurations from the .cfg file first + final_params = load_cfg_parameters(args.cfg_file) + + # 2. Merge values explicitly passed as CLI arguments (CLI takes priority) + cli_args = {k: v for k, v in vars(args).items() if v is not None and k != "cfg_file"} + final_params.update(cli_args) + + # 3. Apply absolute defaults for any remaining missing key parameters + final_params.setdefault("host", "localhost") + final_params.setdefault("port", "9091") + final_params.setdefault("ssl", "false") + final_params.setdefault("ssl_verify", "true") + final_params.setdefault("name", "milvus_monitor_v2") + + result = run(final_params) + print(json.dumps(result)) \ No newline at end of file From df5aacd1e5cd444f0c9a79f665966c2d315feeb8 Mon Sep 17 00:00:00 2001 From: KATHIRVEL Date: Thu, 30 Jul 2026 17:06:21 +0530 Subject: [PATCH 2/3] Update milvus_monitor_v2.cfg --- milvus/milvus_monitor_v2.cfg | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/milvus/milvus_monitor_v2.cfg b/milvus/milvus_monitor_v2.cfg index 9bfac06b..0dfec598 100644 --- a/milvus/milvus_monitor_v2.cfg +++ b/milvus/milvus_monitor_v2.cfg @@ -2,7 +2,6 @@ use_agent_python=1 [milvus] -encrypted.host=8gob+RFoPLO1X7KpvHCZ3A== -encrypted.port=WDnQHa9uEA4sayqW2xUVwg== -encrypted.timeout=dwGnbKGOZO7+WjA7/y66Lg== - +host = "localhost" +port = "9091" +timeout = "10" \ No newline at end of file From 2f7e9fe0a04c18c25160207134abd1582db50406 Mon Sep 17 00:00:00 2001 From: KATHIRVEL Date: Mon, 3 Aug 2026 17:27:34 +0530 Subject: [PATCH 3/3] Rename Milvus plugin to milvus_monitordb and update documentation --- milvus/README.md | 285 --------------- milvus/milvus_monitor_v2.cfg | 7 - milvus/milvus_monitor_v2.py | 482 -------------------------- milvus_monitordb/README.md | 202 +++++++++++ milvus_monitordb/milvus_monitordb.cfg | 4 + milvus_monitordb/milvus_monitordb.py | 320 +++++++++++++++++ 6 files changed, 526 insertions(+), 774 deletions(-) delete mode 100644 milvus/README.md delete mode 100644 milvus/milvus_monitor_v2.cfg delete mode 100644 milvus/milvus_monitor_v2.py create mode 100644 milvus_monitordb/README.md create mode 100644 milvus_monitordb/milvus_monitordb.cfg create mode 100644 milvus_monitordb/milvus_monitordb.py diff --git a/milvus/README.md b/milvus/README.md deleted file mode 100644 index fcc9cff4..00000000 --- a/milvus/README.md +++ /dev/null @@ -1,285 +0,0 @@ -# Milvus Monitoring - -Milvus is an open-source vector database built for high-performance similarity search and AI applications. This plugin collects health, performance, and resource metrics from a Milvus deployment's Prometheus `/metrics` endpoint and reports them to Site24x7 as a custom plugin monitor. - -## Prerequisites - -- Download and install the latest version of the Site24x7 Server Monitoring agent on the server where you plan to run the plugin. -- Python 3 must be available on the host running the plugin. -- The Milvus instance must expose its Prometheus metrics endpoint (enabled by default on most Milvus deployments) and it must be reachable from the plugin host. - -### Metrics Endpoint Access - -By default, Milvus exposes Prometheus-formatted metrics at: - -``` -http://:9091/metrics -``` - -Confirm the endpoint is reachable before installing the plugin: - -```bash -curl http://localhost:9091/metrics -``` - -If Milvus is running remotely, in a container, or behind a different port mapping, update the `HOSTNAME` and `PORT` values at the top of `milvus.py` (see Configuration below). - -## Plugin Installation - -### Linux - -1. Create a directory named `milvus`. - - ```bash - mkdir milvus - cd milvus/ - ``` - -2. Download the plugin file and place it under the `milvus` directory. - - ```bash - wget https://raw.githubusercontent.com/site24x7/plugins/milvus/milvus/milvus.py - ``` - -3. Execute the script to verify it returns valid JSON output. - - ```bash - python3 milvus.py - ``` - -4. Move the `milvus` directory under the Site24x7 Linux Agent plugin directory. - - ```bash - mv milvus /opt/site24x7/monagent/plugins/ - ``` - -### Windows - -1. Create a directory named `milvus`. - -2. Download `milvus.py` and place it under the `milvus` directory. - -3. Since it's a Python plugin, follow the steps in [this link](https://www.site24x7.com/help/admin/monitoring-agent/windows-plugin-monitor.html) to run Python plugins on a Windows server. - -4. Execute the script in `cmd` to verify it returns valid JSON output. - - ```cmd - python milvus.py - ``` - -5. Move the `milvus` folder under the Site24x7 Windows Agent plugin directory. - - ``` - C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins - ``` - -The agent will automatically execute the plugin within five minutes, and you can view the plugin monitor under **Site24x7 > Plugins > Plugin Integrations**. - -## Configuration - -The plugin connects using constants defined at the top of `milvus.py`: - -```python -HOSTNAME = "localhost" -PORT = "9091" -TIMEOUT = 10 -``` - -Update `HOSTNAME` and `PORT` to match your Milvus deployment before moving the plugin into the agent's plugin directory. - -## Diagnostics - -To list every raw metric name exposed by the Milvus `/metrics` endpoint (useful for troubleshooting or extending the plugin), run: - -```bash -python3 milvus.py --list-metrics -``` - -## Supported Metrics - -### Cluster Health - -| Name | Description | -|---|---| -| Milvus_Status | Whether the plugin could successfully reach and parse the Milvus metrics endpoint (1 = up, 0 = down) | -| QueryNodes | Number of QueryNodes registered with the QueryCoord | -| Collections_Loaded | Number of collections currently loaded into memory across QueryNodes | -| DataNodes_Count | Number of DataNodes registered with the DataCoord | -| IndexNode_Count | Number of IndexNodes registered with the DataCoord | -| Proxy_Nodes_Count | Number of active proxy nodes registered with the RootCoord | -| DML_Channels_Count | Number of DML (data manipulation) channels managed by the RootCoord | - -### Component Availability - -| Name | Description | -|---|---| -| RootCoord_Collections | Total number of collections tracked by the RootCoord | -| RootCoord_Partitions | Total number of partitions tracked by the RootCoord | -| Process_Open_FDs | Number of file descriptors currently open by the Milvus process | -| Process_Max_FDs | Maximum number of file descriptors allowed for the Milvus process | - -### Search - -| Name | Description | -|---|---| -| Search_Vectors | Total number of vectors submitted across all search requests | -| Search_Requests | Total number of search-type requests processed | -| Search_Latency.ms | Average end-to-end latency for search requests, in milliseconds | -| Search_Queue_Latency.ms | Average time search requests spend waiting in the QueryNode queue | -| Search_TopK_Avg | Average TopK value requested across search operations | -| Search_WaitResult_Latency.ms | Average time the proxy waits for search results from QueryNodes | -| Search_DecodeResult_Latency.ms | Average time spent decoding search results on the proxy | - -### Query - -| Name | Description | -|---|---| -| Query_Latency.ms | Average end-to-end latency for query (non-search) requests | -| Query_Requests | Total number of query-type requests processed | -| Loaded_Segments | Number of sealed segments currently loaded in QueryNodes | -| Query_Queue_Latency.ms | Average time query requests spend waiting in the proxy queue | -| Query_Reduce_Latency.ms | Average time spent reducing/merging query results on QueryNodes | -| Query_CoreSearch_Latency.ms | Average latency of the internal core search operation | - -### QueryNode Performance - -| Name | Description | -|---|---| -| QN_Entity_Count | Number of entities currently loaded on QueryNodes | -| QN_Entity_Memory.MB | Memory consumed by loaded entities on QueryNodes, in MB | -| QN_Flowgraph_Count | Number of active flowgraphs on QueryNodes | -| QN_DML_Channel_Count | Number of DML channels subscribed to by QueryNodes | -| QN_ReadTask_Concurrency | Current concurrency level for read tasks on QueryNodes | -| QN_ReadTask_Ready_Queue | Number of read tasks ready to be executed | -| QN_ReadTask_Unsolved_Queue | Number of read tasks still waiting to be scheduled | -| QN_LoadSegment_Concurrency | Current concurrency level for segment loading operations | -| QN_LoadSegment_Latency.ms | Average time taken to load a segment into a QueryNode | -| QN_MsgDispatcher_Lag.ms | Lag of the QueryNode message dispatcher behind the latest timestamp | - -### Index Performance - -| Name | Description | -|---|---| -| Index_Build_Latency.ms | Average time taken by IndexNodes to build an index | -| Index_Save_Latency.ms | Average time taken to persist a built index to storage | -| Index_Task_Count | Number of index build tasks currently tracked | -| Index_TaskQueue_Latency.ms | Average time index tasks spend waiting in queue before execution | -| Index_Load_Latency.ms | Average time taken to load an index into memory | -| Index_DataCoord_Tasks | Number of index-related tasks tracked by the DataCoord | - -### Data Ingestion - -| Name | Description | -|---|---| -| Insert_Size.MB | Total size of data received via insert requests, in MB | -| Insert_Request_Count | Total number of insert requests processed by the proxy | -| Delete_Vector_Count | Total number of vectors removed via delete operations | -| Flush_Request_Count | Total number of flush requests processed by DataNodes | -| Flushed_Rows_Count | Total number of rows persisted to storage via flush operations | -| Flushed_Bytes.MB | Total size of data persisted via flush operations, in MB | -| Mutation_Send_Latency.ms | Average latency for sending mutation (insert/delete) requests | - -### DataNode Performance - -| Name | Description | -|---|---| -| DN_Flowgraph_Count | Number of active flowgraphs on DataNodes | -| DN_Consume_Bytes | Total bytes consumed from the message queue by DataNodes | -| DN_Consume_Msg_Count | Total number of messages consumed from the message queue by DataNodes | -| DN_EncodeBuffer_Latency.ms | Average time taken to encode insert buffers before flush | -| DN_Save_Latency.ms | Average time taken by DataNodes to save data to storage | -| DN_AutoFlush_Op_Count | Number of automatic buffer-flush operations triggered | - -### Storage Usage - -| Name | Description | -|---|---| -| Binlog_Size.MB | Total size of stored binlog (write-ahead log) files, in MB | -| Index_Files_Size.MB | Total size of stored index files, in MB | -| Storage_KV_Size.MB | Size of data stored in the underlying key-value storage layer | -| Storage_Op_Count | Total number of operations performed against the storage layer | -| Storage_Request_Latency.ms | Average latency of requests to the storage layer | - -### Memory Usage - -| Name | Description | -|---|---| -| Heap_Usage.MB | Heap memory currently in use by the Milvus process, in MB | -| Heap_Idle.MB | Heap memory currently idle but not yet released to the OS | -| Heap_Sys.MB | Total heap memory obtained from the OS | -| Resident_Memory.MB | Resident memory (RSS) used by the Milvus process | -| Virtual_Memory.MB | Total virtual memory used by the Milvus process | -| MMap_InUse.MB | Memory-mapped space currently in use by Milvus | -| Next_GC_Threshold.MB | Heap size target that will trigger the next garbage collection cycle | - -### Go Runtime - -| Name | Description | -|---|---| -| Threads | Total number of OS threads used by the Milvus process | -| Goroutines | Number of active Go routines running in the Milvus process | -| Go_Threads_Count | Number of OS threads created by the Go runtime | -| GC_Duration_Avg.ms | Average duration of garbage collection cycles | -| GC_Cycle_Count | Total number of garbage collection cycles completed | -| Mallocs_Count | Total number of memory allocations made by the Go runtime | - -### Message Queue - -| Name | Description | -|---|---| -| Consumer_Lag.ms | Lag between DataNode message consumption and the latest published timestamp | -| MQ_Consumer_Count | Number of active consumers subscribed to the message queue | -| MsgStream_Op_Count | Total number of message stream operations performed | -| MsgStream_Request_Latency.ms | Average latency of message stream requests | -| Proxy_TT_Lag.ms | Lag between the proxy's timestamp and the latest allocated timestamp | - -### Proxy Performance - -| Name | Description | -|---|---| -| Proxy_Request_Count | Total number of requests handled by the proxy, across all request types | -| Proxy_Request_Latency.ms | Average latency for requests handled by the proxy | -| Proxy_RateLimit_Count | Number of requests rejected due to rate limiting | -| Proxy_SendBytes.MB | Total bytes sent by the proxy to downstream components | -| Proxy_ApplyPK_Latency.ms | Average latency for primary key allocation on the proxy | -| Proxy_ApplyTimestamp_Latency.ms | Average latency for timestamp allocation on the proxy | -| Proxy_MsgStream_Obj_Num | Number of message stream objects currently held by the proxy | - -### Cache Performance - -| Name | Description | -|---|---| -| Proxy_Cache_Hit_Count | Number of cache hits recorded on the proxy | -| Proxy_Cache_Update_Latency.ms | Average latency for updating the proxy's internal cache | - -### Vector Index Algorithms - -| Name | Description | -|---|---| -| HNSW_BitsetRatio_Avg | Average ratio of filtered (bitset) entities during HNSW search | -| HNSW_SearchHops_Avg | Average number of graph hops traversed during HNSW search | -| IVF_Search_Count | Total number of IVF index search operations performed | -| DiskANN_SearchHops_Avg | Average number of hops traversed during DiskANN search | -| DiskANN_RangeSearchIters_Avg | Average number of iterations performed during DiskANN range search | -| DiskANN_BitsetRatio_Avg | Average ratio of filtered (bitset) entities during DiskANN search | - -### Reliability - -| Name | Description | -|---|---| -| Write_Blocks | Number of times writes were force-denied by the RootCoord (e.g. due to quota limits) | -| RootCoord_DDL_Request_Count | Total number of DDL (data definition) requests processed by the RootCoord | -| RootCoord_DDL_Latency.ms | Average latency for DDL requests on the RootCoord | -| RootCoord_SyncTimetick_Latency.ms | Average latency for timestamp synchronization across the cluster | -| QN_Disk_Cache_Evict_Count | Number of entries evicted from the QueryNode disk cache | -| QN_Disk_Cache_Load_Count | Number of entries loaded into the QueryNode disk cache | -| QN_Segment_Access_Wait_Count | Number of times a query had to wait for a segment to be loaded from disk cache | - -### Storage & Data Overview - -| Name | Description | -|---|---| -| Vectors | Total number of vector rows stored across the cluster | -| Segments | Total number of segments currently tracked by QueryNodes | -| Metrics_Total | Total number of individual metric samples parsed from the `/metrics` endpoint (diagnostic counter) | -| CPU_Time.seconds | Cumulative CPU time consumed by the Milvus process, in seconds | \ No newline at end of file diff --git a/milvus/milvus_monitor_v2.cfg b/milvus/milvus_monitor_v2.cfg deleted file mode 100644 index 0dfec598..00000000 --- a/milvus/milvus_monitor_v2.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[global_configurations] -use_agent_python=1 - -[milvus] -host = "localhost" -port = "9091" -timeout = "10" \ No newline at end of file diff --git a/milvus/milvus_monitor_v2.py b/milvus/milvus_monitor_v2.py deleted file mode 100644 index 5975c31e..00000000 --- a/milvus/milvus_monitor_v2.py +++ /dev/null @@ -1,482 +0,0 @@ -import argparse -import configparser -import json -import os -import re -import ssl -import sys -import tempfile -import time -import urllib.error -import urllib.request - -# Global Constants & Defaults -PLUGIN_VERSION = 10 # Bumped version to force Site24x7 template refresh -HEARTBEAT = "true" -REQUEST_TIMEOUT = 5 - -LINE_RE = re.compile( - r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(\{[^}]*\})?\s+(-?[0-9.eE+\-]+|NaN|\+Inf|-Inf)\s*$' -) -LABEL_RE = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') - -UNITS = { - "Response Time": "ms", - "CPU Percent": "%", - "Resident Memory": "MB", - "Virtual Memory": "MB", - "Heap Usage": "MB", - "Heap Idle": "MB", - "Heap System": "MB", - "Memory Mapped Usage": "MB", - "Next GC Threshold": "MB", - "Entity Memory Usage": "MB", - "System Threads": "count", - "Active Goroutines": "count", - "Total Metrics Count": "count", - "Search Latency": "ms", - "Query Latency": "ms", - "Query Queue Latency": "ms", - "Query Reduce Latency": "ms", - "Core Search Latency": "ms", - "Search Queue Latency": "ms", - "Wait Result Latency": "ms", - "Decode Result Latency": "ms", - "Segment Load Latency": "ms", - "Dispatcher Time Lag": "ms", - "Index Build Latency": "ms", - "Index Save Latency": "ms", - "Index Queue Latency": "ms", - "Index Load Latency": "ms", - "Mutation Latency": "ms", - "Encode Buffer Latency": "ms", - "Data Save Latency": "ms", - "Storage Request Latency": "ms", - "GC Duration Average": "ms", - "Message Stream Latency": "ms", - "Proxy Time Lag": "ms", - "Proxy Request Latency": "ms", - "Primary Key Latency": "ms", - "Timestamp Latency": "ms", - "Cache Update Latency": "ms", - "DDL Request Latency": "ms", - "Sync Timetick Latency": "ms", - "Consumer Time Lag": "ms", - "Flushed Data Size": "MB", - "Binlog Size": "MB", - "Index Size": "MB", - "Key Value Storage Size": "MB", - "Proxy Sent Data": "MB", - "Insert Data Size": "MB", - "Query Nodes": "nodes", - "Data Nodes": "nodes", - "Index Nodes": "nodes", - "Proxy Nodes": "nodes", - "Collections Loaded": "count", - "Total Collections": "count", - "Total Partitions": "count", - "Segments": "count", - "Loaded Segments": "count", - "Vectors": "count", - "Search Vectors": "count", - "Search Requests": "reqs", - "Query Requests": "reqs", - "Insert Requests": "reqs", - "Search Top K Avg": "count", - "Active Index Tasks": "tasks", - "Deleted Vectors": "count", - "Flush Requests": "reqs", - "Flushed Rows": "rows", - "Consumed Messages": "msgs", - "Consumed Bytes": "bytes", - "Storage Operations": "ops", - "Write Denials": "count", - "Open File Descriptors": "fds", - "Max File Descriptors": "fds", -} - -TABS = { - "CPU & Memory": { - "order": 1, - "tablist": [ - "CPU Percent", - "Heap Usage", - "Heap Idle", - "Heap System", - "Resident Memory", - "Virtual Memory", - "System Threads", - "Active Goroutines", - ], - }, - "Query Performance": { - "order": 2, - "tablist": [ - "Query Latency", - "Query Requests", - "Query Queue Latency", - "Query Reduce Latency", - "Search Latency", - "Search Requests", - "Search Vectors", - "Search Top K Avg", - ], - }, - "Index Performance": { - "order": 3, - "tablist": [ - "Index Build Latency", - "Index Load Latency", - "Index Save Latency", - "Index Size", - "Active Index Tasks", - "Index Queue Latency", - ], - }, - "Data Ingestion": { - "order": 4, - "tablist": [ - "Insert Requests", - "Insert Data Size", - "Deleted Vectors", - "Flush Requests", - "Flushed Rows", - "Consumed Messages", - "Consumed Bytes", - ], - }, - "Cluster Health": { - "order": 5, - "tablist": [ - "Query Nodes", - "Data Nodes", - "Index Nodes", - "Proxy Nodes", - "Collections Loaded", - "Segments", - "Vectors", - ], - }, - "Storage": { - "order": 6, - "tablist": [ - "Binlog Size", - "Key Value Storage Size", - "Storage Operations", - "Storage Request Latency", - "Memory Mapped Usage", - ], - }, -} - -METRIC_SPECS = [ - {"key": "Data Nodes", "source": "milvus_datacoord_datanode_num", "kind": "gauge"}, - {"key": "Index Nodes", "source": "milvus_datacoord_index_node_num", "kind": "gauge"}, - {"key": "Proxy Nodes", "source": "milvus_rootcoord_proxy_num", "kind": "gauge"}, - {"key": "Total Collections", "source": "milvus_rootcoord_collection_num", "kind": "gauge"}, - {"key": "Total Partitions", "source": "milvus_rootcoord_partition_num", "kind": "gauge"}, - {"key": "Open File Descriptors", "source": "process_open_fds", "kind": "gauge"}, - {"key": "Max File Descriptors", "source": "process_max_fds", "kind": "gauge"}, - {"key": "Query Queue Latency", "source": "milvus_proxy_req_in_queue_latency", "kind": "hist_ms"}, - {"key": "Query Reduce Latency", "source": "milvus_querynode_sq_reduce_latency", "kind": "hist_ms"}, - {"key": "Core Search Latency", "source": "milvus_internal_core_search_latency", "kind": "hist_us_ms"}, - {"key": "Search Queue Latency", "source": "milvus_querynode_sq_queue_latency", "kind": "hist_ms"}, - {"key": "Search Top K Avg", "source": "milvus_search_topk", "kind": "hist_raw"}, - {"key": "Wait Result Latency", "source": "milvus_proxy_sq_wait_result_latency", "kind": "hist_ms"}, - {"key": "Decode Result Latency", "source": "milvus_proxy_sq_decode_result_latency", "kind": "hist_ms"}, - {"key": "Entity Memory Usage", "source": "milvus_querynode_entity_size", "kind": "gauge_mb"}, - {"key": "Segment Load Latency", "source": "milvus_querynode_load_segment_latency", "kind": "hist_ms"}, - {"key": "Dispatcher Time Lag", "source": "milvus_querynode_msg_dispatcher_tt_lag_ms", "kind": "gauge"}, - {"key": "Index Build Latency", "source": "milvus_indexnode_build_index_latency", "kind": "hist_ms"}, - {"key": "Index Save Latency", "source": "milvus_indexnode_save_index_latency", "kind": "hist_ms"}, - {"key": "Active Index Tasks", "source": "milvus_indexnode_index_task_count", "kind": "gauge"}, - {"key": "Index Queue Latency", "source": "milvus_indexnode_index_task_latency_in_queue", "kind": "hist_ms"}, - {"key": "Index Load Latency", "source": "milvus_load_latency", "kind": "hist_ms"}, - {"key": "Insert Requests", "source": "milvus_proxy_req_count", "kind": "gauge", "labels": {"rpc_type": "Insert"}}, - {"key": "Deleted Vectors", "source": "milvus_proxy_delete_vectors_count", "kind": "gauge"}, - {"key": "Flush Requests", "source": "milvus_datanode_flush_req_count", "kind": "gauge"}, - {"key": "Flushed Rows", "source": "milvus_datanode_flushed_data_rows_count", "kind": "gauge"}, - {"key": "Flushed Data Size", "source": "milvus_datanode_flushed_data_size_count", "kind": "gauge_mb"}, - {"key": "Mutation Latency", "source": "milvus_proxy_mutation_send_latency", "kind": "hist_ms"}, - {"key": "Consumed Bytes", "source": "milvus_datanode_consume_bytes_count", "kind": "gauge"}, - {"key": "Consumed Messages", "source": "milvus_datanode_consume_msg_count", "kind": "gauge"}, - {"key": "Encode Buffer Latency", "source": "milvus_datanode_encode_buffer_latency", "kind": "hist_ms"}, - {"key": "Data Save Latency", "source": "milvus_datanode_save_latency", "kind": "hist_ms"}, - {"key": "Binlog Size", "source": "milvus_datacoord_stored_binlog_size", "kind": "gauge_mb"}, - {"key": "Index Size", "source": "milvus_datacoord_stored_index_files_size", "kind": "gauge_mb"}, - {"key": "Key Value Storage Size", "source": "milvus_storage_kv_size", "kind": "gauge_mb"}, - {"key": "Storage Operations", "source": "milvus_storage_op_count", "kind": "gauge"}, - {"key": "Storage Request Latency", "source": "milvus_storage_request_latency", "kind": "gauge"}, - {"key": "Heap Idle", "source": "go_memstats_heap_idle_bytes", "kind": "gauge_mb"}, - {"key": "Heap System", "source": "go_memstats_heap_sys_bytes", "kind": "gauge_mb"}, - {"key": "Resident Memory", "source": "process_resident_memory_bytes", "kind": "gauge_mb"}, - {"key": "Virtual Memory", "source": "process_virtual_memory_bytes", "kind": "gauge_mb"}, - {"key": "Memory Mapped Usage", "source": "milvus_internal_mmap_in_used_space_bytes", "kind": "gauge_mb"}, - {"key": "Next GC Threshold", "source": "go_memstats_next_gc_bytes", "kind": "gauge_mb"}, - {"key": "GC Duration Average", "source": "go_gc_duration_seconds", "kind": "hist_sec_ms"}, -] - -def get_error_payload(error_message): - return { - "plugin_version": PLUGIN_VERSION, - "heartbeat_required": HEARTBEAT, - "status": 0, - "msg": error_message, - "units": UNITS, - "tabs": TABS, - } - -def timed_get(url, ssl_verify=True): - try: - start = time.time() - req = urllib.request.Request(url, headers={'User-Agent': 'Site24x7-Plugin'}) - ctx = ssl.create_default_context() - if not ssl_verify: - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - - with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT, context=ctx) as response: - elapsed_ms = round((time.time() - start) * 1000, 2) - content = response.read().decode('utf-8', errors='replace') - return response.status == 200, elapsed_ms, content - except Exception: - return False, 0, None - -def parse_metrics(text): - series = {} - if not text: - return series - - for line in text.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - - match = LINE_RE.match(line) - if not match: - continue - - name, labels_text, value_text = match.groups() - try: - value = float(value_text) - if value != value: # Check for NaN - continue - except Exception: - continue - - labels = {} - if labels_text: - for k, v in LABEL_RE.findall(labels_text): - labels[k] = v - - series.setdefault(name, []).append({"labels": labels, "value": value}) - - return series - -def get_sum(series, metric, labels=None): - total = 0.0 - for item in series.get(metric, []): - if labels: - if not all(item["labels"].get(k) == v for k, v in labels.items()): - continue - total += item["value"] - return total - -def histogram_average(series, metric): - total = get_sum(series, metric + "_sum") - count = get_sum(series, metric + "_count") - if count > 0: - return round((total / count) * 0.001, 2) - return 0 - -def histogram_stat(series, metric_base, labels=None, scale=1.0, ndigits=2): - total = get_sum(series, metric_base + "_sum", labels) - count = get_sum(series, metric_base + "_count", labels) - if count > 0: - return round((total / count) * scale, ndigits) - return 0 - -def bytes_to_mb(value): - if not value: - return 0 - return round(value / (1024 * 1024), 2) - -def compute_new_metrics(series): - new_values = {} - for spec in METRIC_SPECS: - key = spec["key"] - source = spec["source"] - kind = spec["kind"] - labels = spec.get("labels") - - try: - if kind == "gauge": - value = round(get_sum(series, source, labels)) - elif kind == "gauge_mb": - value = bytes_to_mb(get_sum(series, source, labels)) - elif kind == "hist_ms": - value = histogram_stat(series, source, labels, scale=1.0) - elif kind == "hist_us_ms": - value = histogram_stat(series, source, labels, scale=0.001) - elif kind == "hist_sec_ms": - value = histogram_stat(series, source, labels, scale=1000.0) - elif kind == "hist_raw": - value = histogram_stat(series, source, labels, scale=1.0) - else: - value = 0 - except Exception: - value = 0 - - new_values[key] = value - return new_values - -CPU_STATE_FILE = os.path.join(tempfile.gettempdir(), "milvus_cpu_state.json") - -def get_cpu_utilization_percent(current_cpu_seconds, cores=1): - now = time.time() - previous_time, previous_cpu = None, None - - try: - with open(CPU_STATE_FILE, "r") as f: - state = json.load(f) - previous_time = state.get("timestamp") - previous_cpu = state.get("cpu_seconds") - except Exception: - pass - - try: - with open(CPU_STATE_FILE, "w") as f: - json.dump({"timestamp": now, "cpu_seconds": current_cpu_seconds}, f) - except Exception: - pass - - if previous_time is None or previous_cpu is None: - return 0 - - delta_time = now - previous_time - delta_cpu = current_cpu_seconds - previous_cpu - - if delta_cpu < 0 or delta_time <= 0: - return 0 - - cores = cores if cores and cores > 0 else 1 - return round(((delta_cpu / delta_time) * 100) / cores, 2) - -def metricCollector(param): - host = param.get("host", "localhost") - port = param.get("port", "9091") - ssl_enabled = str(param.get("ssl", "false")).lower() == "true" - ssl_verify_val = str(param.get("ssl_verify", "true")).lower() == "true" - - protocol = "https" if ssl_enabled else "http" - metrics_url = f"{protocol}://{host}:{port}/metrics" - - status_ok, response_time_ms, raw_text = timed_get(metrics_url, ssl_verify=ssl_verify_val) - - if not status_ok or not raw_text: - return get_error_payload(f"Failed to reach Milvus Prometheus Endpoint at {metrics_url}") - - series = parse_metrics(raw_text) - - cpu_seconds = get_sum(series, "process_cpu_seconds_total") - cpu_percent = get_cpu_utilization_percent(cpu_seconds, cores=os.cpu_count() or 1) - threads = get_sum(series, "go_threads") or get_sum(series, "milvus_thread_num") - - payload = { - "plugin_version": PLUGIN_VERSION, - "heartbeat_required": HEARTBEAT, - "status": 1, - "Response Time": response_time_ms, - "Total Metrics Count": sum(len(v) for v in series.values()), - "Query Nodes": round(get_sum(series, "milvus_querycoord_querynode_num")), - "Collections Loaded": round(get_sum(series, "milvus_querynode_collection_num")), - "Search Vectors": round(get_sum(series, "milvus_proxy_search_vectors_count")), - "Search Requests": round( - get_sum(series, "milvus_proxy_collection_sq_latency_count", {"query_type": "search"}) - ), - "Search Latency": histogram_average(series, "milvus_proxy_collection_sq_latency"), - "Vectors": round(get_sum(series, "milvus_datacoord_stored_rows_num")), - "Segments": round(get_sum(series, "milvus_querynode_segment_num")), - "Insert Data Size": round(get_sum(series, "milvus_proxy_receive_bytes_count") / (1024 * 1024), 2), - "Query Latency": histogram_average(series, "milvus_proxy_collection_sq_latency"), - "Query Requests": round( - get_sum(series, "milvus_proxy_collection_sq_latency_count", {"query_type": "query"}) - ), - "Loaded Segments": round( - get_sum(series, "milvus_querynode_segment_num", {"segment_state": "Sealed"}) - ), - "CPU Percent": cpu_percent, - "System Threads": round(threads), - "Active Goroutines": round(get_sum(series, "go_goroutines")), - "Heap Usage": round(get_sum(series, "go_memstats_heap_inuse_bytes") / (1024 * 1024), 2), - "Write Denials": round(get_sum(series, "milvus_rootcoord_force_deny_writing_counter")), - "Consumer Time Lag": round(get_sum(series, "milvus_datanode_consume_tt_lag_ms")), - "units": UNITS, - "tabs": TABS, - } - - payload.update(compute_new_metrics(series)) - return payload - -def load_cfg_parameters(cfg_path=None): - """ - Reads parameters from a .cfg file if found. - Checks the explicitly passed file path, or falls back to looking for a file - with the same base name as this script in the same directory. - """ - config_params = {} - - if not cfg_path: - script_dir = os.path.dirname(os.path.abspath(__file__)) - script_name = os.path.splitext(os.path.basename(__file__))[0] - cfg_path = os.path.join(script_dir, f"{script_name}.cfg") - - if os.path.exists(cfg_path): - config = configparser.ConfigParser() - try: - config.read(cfg_path) - # Scan all common Site24x7 sections for configurations - for section in ["global_configurations", "display", "plugin"]: - if config.has_section(section): - for key, value in config.items(section): - # Clean key names to avoid encrypted prefixes or unwanted spaces - clean_key = key.split(".")[-1] - config_params[clean_key] = value - except Exception: - pass # Fallback gracefully to default parameters if read fails - - return config_params - -def run(param): - try: - return metricCollector(param) - except Exception as e: - return get_error_payload(f"Plugin Execution Error: {str(e)}") - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--host", help="Host Name", nargs="?", default=None) - parser.add_argument("--port", help="Metrics Port", nargs="?", default=None) - parser.add_argument("--ssl", help="Use SSL/HTTPS", default=None) - parser.add_argument("--ssl_verify", help="SSL Verification", default=None) - parser.add_argument("--name", help="Plugin Name", nargs="?", default=None) - parser.add_argument("--cfg_file", help="Custom path to .cfg file", default=None) - - args, _ = parser.parse_known_args() - - # 1. Read configurations from the .cfg file first - final_params = load_cfg_parameters(args.cfg_file) - - # 2. Merge values explicitly passed as CLI arguments (CLI takes priority) - cli_args = {k: v for k, v in vars(args).items() if v is not None and k != "cfg_file"} - final_params.update(cli_args) - - # 3. Apply absolute defaults for any remaining missing key parameters - final_params.setdefault("host", "localhost") - final_params.setdefault("port", "9091") - final_params.setdefault("ssl", "false") - final_params.setdefault("ssl_verify", "true") - final_params.setdefault("name", "milvus_monitor_v2") - - result = run(final_params) - print(json.dumps(result)) \ No newline at end of file diff --git a/milvus_monitordb/README.md b/milvus_monitordb/README.md new file mode 100644 index 00000000..ce822bb1 --- /dev/null +++ b/milvus_monitordb/README.md @@ -0,0 +1,202 @@ +# Milvus Monitoring + +Milvus is an open-source vector database built to power AI applications, vector search, and Retrieval-Augmented Generation (RAG). It exposes Prometheus-style metrics that this plugin uses to report on cluster topology, collection segments, storage usage, search latency, and request throughput. + +## Prerequisites + +- Download and install the latest version of the [Site24x7 Server Monitoring agent](https://www.site24x7.com/help/admin/adding-a-monitor/linux-server-monitoring.html#add-linux-server-monitor) on the server where you plan to run the plugin. +- Milvus must be running and exposing its metrics endpoint (default port `9091`). + +--- + +# Plugin Installation + +## Linux + +### Step 1 + +Create a directory named `milvus_monitordb`. + +```bash +mkdir milvus_monitordb +cd milvus_monitordb/ +``` + +### Step 2 + +Place the following files under the `milvus_monitordb` directory: + +- `milvus_monitordb.py` +- `milvus_monitordb.cfg` + +### Step 3 + +Execute the below command with appropriate arguments to check for valid JSON output: + +```bash +python3 milvus_monitordb.py host='127.0.0.1' metrics_port='9091' +``` + +### Step 4 + +Provide your Milvus configurations in the `milvus_monitordb.cfg` file: + +```ini +[milvus_monitordb] +host = "127.0.0.1" +metrics_port = "9091" +``` + +### Step 5 + +Move the directory `milvus_monitordb` under the Site24x7 Linux Agent plugin directory: + +```bash +mv milvus_monitordb /opt/site24x7/monagent/plugins/ +``` + +--- + +## Windows + +### Step 1 + +Create a directory named `milvus_monitordb`. + +### Step 2 + +Place the files: + +- `milvus_monitordb.py` +- `milvus_monitordb.cfg` + +under the `milvus_monitordb` directory. + +### Step 3 + +Create a PowerShell wrapper file `milvus_monitordb.ps1` in the same directory: + +```powershell +& "python.exe" "C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins\milvus_monitordb\milvus_monitordb.py" $args +``` + +### Step 4 + +Execute the below command with appropriate arguments in PowerShell to check for valid JSON output: + +```powershell +powershell.exe -ExecutionPolicy Bypass -File "C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins\milvus_monitordb\milvus_monitordb.ps1" host="127.0.0.1" metrics_port="9091" +``` + +### Step 5 + +Provide your Milvus configurations in the `milvus_monitordb.cfg` file: + +```ini +[milvus_monitordb] +host = "127.0.0.1" +metrics_port = "9091" +``` + +### Step 6 + +Move the folder `milvus_monitordb` under the Site24x7 Windows Agent plugin directory: + +```text +C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins +``` + +The agent will automatically execute the plugin within five minutes, and users can see the plugin monitor under **Site24x7 > Plugins > Plugin Integrations**. + +--- + +# Supported Metrics + +## Overview + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Response Time | Time taken to execute the metric collection request. | Higher values indicate latency in communicating with the Milvus metrics endpoint. | +| CPU Percent | Process CPU utilization of the Milvus instance. | Sustained high CPU usage can lead to search latency spikes and ingestion bottlenecks. | +| Memory Usage | Total resident memory consumed by the Milvus process. | High memory utilization can lead to host swapping or OOM (Out-Of-Memory) kills. | +| Active Goroutines | Number of active Go routines in the runtime. | Unusually high counts can indicate thread leakage or high concurrency backlogs. | +| OS Threads | Number of operating system threads allocated by Go runtime. | Reflects system context switching and thread resource allocation. | +| Open File Descriptors | Number of file handles opened by Milvus. | Values approaching system limits (ulimit) can block file I/O and network sockets. | + +--- + +## Nodes and Topology + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Total Nodes | Total number of nodes participating in the Milvus cluster. | Tracks cluster scale and node membership availability. | +| Proxy Nodes | Count of active Proxy coordinator nodes handling client requests. | Low proxy counts relative to traffic can bottleneck incoming API connections. | +| Query Nodes | Count of active QueryNodes executing vector search tasks. | Lower QueryNode availability reduces search capacity and throughput. | +| Data Nodes | Count of active DataNodes handling vector data ingestion and flush. | Unavailability degrades data persistence and binlog generation capabilities. | +| Index Nodes | Count of active IndexNodes processing vector index building. | Fewer index nodes slow down vector index construction times. | +| Streaming Nodes | Count of active StreamingNodes processing WAL/streaming requests. | Affects stream ingestion latency and real-time message stream routing. | +| gRPC Active Conns | Current active gRPC connection count across cluster components. | High counts reflect heavy internal inter-node communication. | +| Proxy Active Conns | Active client connections connected directly to Proxy nodes. | High active connections indicate heavy incoming SDK client traffic. | + +--- + +## Collections and Segments + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Data Collections | Total number of collections registered in DataCoord. | Indicates the logical schema scale managed by the cluster. | +| Query Collections | Number of collections loaded into QueryCoord for searching. | Unloaded collections cannot serve search/query traffic. | +| Query Replicas | Total number of loaded collection replicas across QueryNodes. | More replicas improve search concurrency and fault tolerance. | +| Data Segments | Total count of segments managed by DataCoord. | Higher segment counts increase metadata coordination overhead. | +| Loaded Segments | Total segments currently loaded in memory for query processing. | High counts consume QueryNode RAM; must fit within available memory. | +| Growing Segments | Segments currently receiving new vector insertions in real time. | Large numbers of growing segments increase search latency before indexing. | +| Sealed Segments | Segments closed for insertion and queued/ready for indexing. | Indicates segments awaiting background index building. | +| Flushed Segments | Segments completely persisted to object storage/disk. | Assures durability and completion of data sync processes. | + +--- + +## Storage and Memory + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Total Indexed Rows | Total number of vector entity rows persisted in DataCoord storage. | Primary indicator of overall vector database size over time. | +| Loaded Entities QN | Count of vector entities currently loaded into QueryNodes. | Tracks active search-ready entities loaded in memory. | +| Binlog Size | Total size of raw unindexed insert binlogs on storage. | High binlog volume without indexing increases memory overhead. | +| Index Files Size | Total disk space consumed by built vector index structures. | Directly influences disk capacity planning for index storage. | +| Storage KV Size | Storage footprint consumed by key-value storage engine backend. | Reflects underlying KV metadata and system state disk footprint. | +| Meta KV Size | Memory/disk space used for metadata KV state store. | Excessive meta size can slow down cluster coordination tasks. | +| Raw Data Size | Total uncompressed raw vector data size across DataNodes. | Measures raw ingestion data scale before compaction and indexing. | +| QN CGO Memory | Memory allocated by QueryNode C++ core via CGO for vector search engines. | Primary driver of QueryNode RAM utilization (Knowhere engine usage). | + +--- + +## Latency + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Search Query Latency | End-to-end vector search latency measured at the Proxy level. | Directly impacts end-user application query responsiveness. | +| Core Search Latency | Time spent inside the execution C++ vector search engine (Knowhere). | Isolates vector index search speed from network/framework overhead. | +| QN Search Latency | Time taken by QueryNode to process and return vector search results. | Higher values point to QueryNode compute or memory bottlenecks. | +| Proxy Req Latency | Average latency for general proxy client request processing. | Measures overall frontend API gateway responsiveness. | +| DataNode Flush Lat | Time taken for DataNode to flush segment memory buffers to disk. | Slow flushes can delay data durability and segment sealing. | +| Index Build Latency | Duration taken by IndexNodes to generate vector index structures. | Longer durations delay search availability on newly inserted vectors. | +| gRPC Request Latency | Average latency of inter-component gRPC communications. | Indicates internal network latency between proxy, coordinators, and worker nodes. | + +--- + +## Throughput and Queues + +| Metric Name | Description | Impact on Milvus | +|------------|-------------|------------------| +| Proxy Request Count | Total number of requests processed by Proxy nodes. | Reflects overall system traffic volume. | +| Ingestion Volume | Total volume of vector data ingested through Proxy nodes. | Measures raw bandwidth and data ingestion throughput. | +| Delete Vectors Count | Cumulative count of deleted vector entities processed. | High deletion rates trigger compaction tasks and tombstone overhead. | +| Flush Request Count | Total segment flush requests issued across DataNodes. | Tracks segment lifecycle transition frequency. | +| Flushed Rows Count | Total number of rows successfully written during flush operations. | Measures persistence throughput. | +| Searched Vector Count | Total number of target search query vectors processed. | High values indicate heavy batch search workloads. | +| Query Request Count | Cumulative scalar/entity query requests executed. | Measures non-vector scalar query workload. | +| Insert Request Count | Cumulative vector insert API requests received by Proxy. | Reflects bulk ingestion request activity. | +| Search QPS Rate | Current vector search Queries Per Second rate. | Core measure of search throughput. | +| Insert QPS Rate | Current vector insert Queries Per Second rate. | Core measure of ingestion throughput. | +| Proxy Queue Length | Number of search/query tasks waiting in Proxy execution queues. | Non-zero values indicate proxy worker pool saturation. | +| QN Queue Length | Number of vector execution tasks queued inside QueryNodes. | Queue backlogs directly increase search latency. | \ No newline at end of file diff --git a/milvus_monitordb/milvus_monitordb.cfg b/milvus_monitordb/milvus_monitordb.cfg new file mode 100644 index 00000000..3d2119da --- /dev/null +++ b/milvus_monitordb/milvus_monitordb.cfg @@ -0,0 +1,4 @@ +[milvus_monitordb] +python_path=C:\Program Files\Python313\python.exe +host=127.0.0.1 +metrics_port=9091 \ No newline at end of file diff --git a/milvus_monitordb/milvus_monitordb.py b/milvus_monitordb/milvus_monitordb.py new file mode 100644 index 00000000..65b2cf53 --- /dev/null +++ b/milvus_monitordb/milvus_monitordb.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +import json +import time +import sys +import socket +import urllib.request + +# Strict timeout (2s) to prevent Site24x7 hard process termination +HTTP_TIMEOUT = 2.0 +socket.setdefaulttimeout(HTTP_TIMEOUT) + +# Version 37 forces Site24x7 to re-register metrics and tabs +PLUGIN_VERSION = 37 +HEARTBEAT = "true" + +# Define Dashboard Tabs Configuration (Exactly 49 total metrics including Response Time) +TABS = { + "Overview": { + "order": 1, + "tablist": [ + "Response Time", + "CPU Percent", + "Memory Usage", + "Active Goroutines", + "OS Threads", + "Open File Descriptors" + ] + }, + "Nodes and Topology": { + "order": 2, + "tablist": [ + "Total Nodes", + "Proxy Nodes", + "Query Nodes", + "Data Nodes", + "Index Nodes", + "Streaming Nodes", + "gRPC Active Conns", + "Proxy Active Conns" + ] + }, + "Collections and Segments": { + "order": 3, + "tablist": [ + "Data Collections", + "Query Collections", + "Query Replicas", + "Data Segments", + "Loaded Segments", + "Growing Segments", + "Sealed Segments", + "Flushed Segments" + ] + }, + "Storage and Memory": { + "order": 4, + "tablist": [ + "Total Indexed Rows", + "Loaded Entities QN", + "Binlog Size", + "Index Files Size", + "Storage KV Size", + "Meta KV Size", + "Raw Data Size", + "QN CGO Memory" + ] + }, + "Latency": { + "order": 5, + "tablist": [ + "Search Query Latency", + "Core Search Latency", + "QN Search Latency", + "Proxy Req Latency", + "DataNode Flush Lat", + "Index Build Latency", + "gRPC Request Latency" + ] + }, + "Throughput and Queues": { + "order": 6, + "tablist": [ + "Proxy Request Count", + "Ingestion Volume", + "Delete Vectors Count", + "Flush Request Count", + "Flushed Rows Count", + "Searched Vector Count", + "Query Request Count", + "Insert Request Count", + "Search QPS Rate", + "Insert QPS Rate", + "Proxy Queue Length", + "QN Queue Length" + ] + } +} + +UNITS = { + "Response Time": "ms", + "CPU Percent": "%", + "Memory Usage": "MB", + "Active Goroutines": "goroutines", + "OS Threads": "threads", + "Open File Descriptors": "files", + "Total Nodes": "nodes", + "Proxy Nodes": "nodes", + "Query Nodes": "nodes", + "Data Nodes": "nodes", + "Index Nodes": "nodes", + "Streaming Nodes": "nodes", + "gRPC Active Conns": "connections", + "Proxy Active Conns": "connections", + "Data Collections": "collections", + "Query Collections": "collections", + "Query Replicas": "replicas", + "Data Segments": "segments", + "Loaded Segments": "segments", + "Growing Segments": "segments", + "Sealed Segments": "segments", + "Flushed Segments": "segments", + "Total Indexed Rows": "entities", + "Loaded Entities QN": "entities", + "Binlog Size": "MB", + "Index Files Size": "MB", + "Storage KV Size": "MB", + "Meta KV Size": "MB", + "Raw Data Size": "MB", + "QN CGO Memory": "MB", + "Search Query Latency": "ms", + "Core Search Latency": "ms", + "QN Search Latency": "ms", + "Proxy Req Latency": "ms", + "DataNode Flush Lat": "ms", + "Index Build Latency": "ms", + "gRPC Request Latency": "ms", + "Proxy Request Count": "requests", + "Ingestion Volume": "MB", + "Delete Vectors Count": "vectors", + "Flush Request Count": "requests", + "Flushed Rows Count": "rows", + "Searched Vector Count": "vectors", + "Query Request Count": "requests", + "Insert Request Count": "requests", + "Search QPS Rate": "qps", + "Insert QPS Rate": "qps", + "Proxy Queue Length": "items", + "QN Queue Length": "items" +} + +METRICS_MAP = { + "process_cpu_seconds_total": ("CPU Percent", "DIRECT"), + "process_resident_memory_bytes": ("Memory Usage", "MB"), + "go_goroutines": ("Active Goroutines", "DIRECT"), + "go_threads": ("OS Threads", "DIRECT"), + "process_open_fds": ("Open File Descriptors", "DIRECT"), + "milvus_num_node": ("Total Nodes", "DIRECT"), + "milvus_rootcoord_proxy_num": ("Proxy Nodes", "DIRECT"), + "milvus_querycoord_querynode_num": ("Query Nodes", "DIRECT"), + "milvus_datacoord_datanode_num": ("Data Nodes", "DIRECT"), + "milvus_datacoord_index_node_num": ("Index Nodes", "DIRECT"), + "milvus_streamingnode_num": ("Streaming Nodes", "DIRECT"), + "milvus_grpc_active_connections": ("gRPC Active Conns", "DIRECT"), + "milvus_proxy_active_connections": ("Proxy Active Conns", "DIRECT"), + "milvus_datacoord_collection_num": ("Data Collections", "DIRECT"), + "milvus_querycoord_collection_num": ("Query Collections", "DIRECT"), + "milvus_querycoord_replica_num": ("Query Replicas", "DIRECT"), + "milvus_datacoord_segment_num": ("Data Segments", "DIRECT"), + "milvus_querycoord_segment_num": ("Loaded Segments", "DIRECT"), + "milvus_datacoord_growing_segment_num": ("Growing Segments", "DIRECT"), + "milvus_datacoord_sealed_segment_num": ("Sealed Segments", "DIRECT"), + "milvus_datacoord_flushed_segment_num": ("Flushed Segments", "DIRECT"), + "milvus_datacoord_stored_rows_num": ("Total Indexed Rows", "DIRECT"), + "milvus_querynode_entity_num": ("Loaded Entities QN", "DIRECT"), + "milvus_datacoord_stored_binlog_size": ("Binlog Size", "MB"), + "milvus_datacoord_stored_index_files_size": ("Index Files Size", "MB"), + "milvus_storage_kv_size": ("Storage KV Size", "MB"), + "milvus_meta_kv_size": ("Meta KV Size", "MB"), + "milvus_datanode_raw_data_size": ("Raw Data Size", "MB"), + "milvus_querynode_cgo_memory_bytes": ("QN CGO Memory", "MB"), + "milvus_proxy_collection_sq_latency": ("Search Query Latency", "AVG"), + "milvus_internal_core_search_latency": ("Core Search Latency", "AVG_US"), + "milvus_querynode_search_latency": ("QN Search Latency", "AVG"), + "milvus_proxy_req_latency": ("Proxy Req Latency", "AVG"), + "milvus_datanode_flush_latency": ("DataNode Flush Lat", "AVG"), + "milvus_indexnode_build_latency": ("Index Build Latency", "AVG"), + "milvus_proxy_grpc_request_latency": ("gRPC Request Latency", "AVG"), + "milvus_proxy_req_count": ("Proxy Request Count", "DIRECT"), + "milvus_proxy_receive_bytes_count": ("Ingestion Volume", "MB"), + "milvus_proxy_delete_vectors_count": ("Delete Vectors Count", "DIRECT"), + "milvus_datanode_flush_req_count": ("Flush Request Count", "DIRECT"), + "milvus_datanode_flushed_data_rows_count": ("Flushed Rows Count", "DIRECT"), + "milvus_proxy_search_vectors_count": ("Searched Vector Count", "DIRECT"), + "milvus_proxy_query_req_count": ("Query Request Count", "DIRECT"), + "milvus_proxy_insert_req_count": ("Insert Request Count", "DIRECT"), + "milvus_proxy_search_qps": ("Search QPS Rate", "DIRECT"), + "milvus_proxy_insert_qps": ("Insert QPS Rate", "DIRECT"), + "milvus_proxy_sq_queue_length": ("Proxy Queue Length", "DIRECT"), + "milvus_querynode_sq_queue_length": ("QN Queue Length", "DIRECT") +} + +def get_error_payload(msg): + payload = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "status": 0, + "msg": str(msg)[:100], + "units": UNITS, + "tabs": TABS + } + for k in UNITS.keys(): + payload[k] = 0 + return payload + +def metricCollector(host, port): + start = time.time() + res = {display_name: 0.0 for display_name in UNITS.keys()} + url = f"http://{host}:{port}/metrics" + + try: + req = urllib.request.Request(url, headers={"User-Agent": "Site24x7-Plugin"}) + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: + raw_text = resp.read().decode("utf-8") + except Exception as e: + err = get_error_payload(f"Connect failed: {str(e)}") + err["Response Time"] = round((time.time() - start) * 1000, 2) + return err + + gauges, sums, counts = {}, {}, {} + for line in raw_text.split("\n"): + if not line or line.startswith("#"): + continue + + parts = line.rsplit(" ", 1) + if len(parts) != 2: + continue + + k = parts[0].split("{")[0] + if not (k in METRICS_MAP or k.endswith("_sum") or k.endswith("_count")): + continue + + try: + val = float(parts[1]) + if val != val or val in (float('inf'), float('-inf')): + val = 0.0 + except ValueError: + continue + + if k in METRICS_MAP: + gauges[k] = gauges.get(k, 0.0) + val + elif k.endswith("_sum"): + base = k[:-4] + if base in METRICS_MAP: + sums[base] = sums.get(base, 0.0) + val + elif k.endswith("_count"): + base = k[:-6] + if base in METRICS_MAP: + counts[base] = counts.get(base, 0.0) + val + + for prom_key, val in gauges.items(): + if prom_key in METRICS_MAP: + display_name, calc_type = METRICS_MAP[prom_key] + if calc_type == "MB": + res[display_name] = round(val / (1024 * 1024), 2) + elif calc_type == "DIRECT": + res[display_name] = round(val, 2) + + for prom_key, (display_name, calc_type) in METRICS_MAP.items(): + if calc_type in ("AVG", "AVG_US"): + cnt = counts.get(prom_key, 0.0) + avg = (sums.get(prom_key, 0.0) / cnt) if cnt > 0 else 0.0 + res[display_name] = round(avg / 1000.0 if calc_type == "AVG_US" else avg, 2) + + res["Response Time"] = round((time.time() - start) * 1000, 2) + + out = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "status": 1, + "msg": "OK", + "units": UNITS, + "tabs": TABS + } + out.update(res) + return out + +def clean_param(val, default): + if not val or not isinstance(val, str): + return default + val = val.strip("\"'") + if len(val) > 20 and ("==" in val or "+" in val or "/" in val): + return default + return val + +def parse_cli_args(): + raw_args = {} + for arg in sys.argv[1:]: + if "=" in arg: + k, v = arg.split("=", 1) + k = k.lstrip("-").replace("encrypted.", "") + raw_args[k] = v + + host = clean_param(raw_args.get("host"), "127.0.0.1") + port = clean_param(raw_args.get("metrics_port"), "9091") + + try: + int(port) + except ValueError: + port = "9091" + + return host, port + +if __name__ == "__main__": + try: + host, port = parse_cli_args() + print(json.dumps(metricCollector(host, port))) + sys.stdout.flush() + except Exception as fatal_e: + print(json.dumps(get_error_payload(f"Fatal Exec Error: {str(fatal_e)}"))) + sys.stdout.flush() \ No newline at end of file