From e11d5dfc629a8caacb81542cf2a95668b785c6f4 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran Date: Fri, 29 Aug 2025 17:49:05 +0530 Subject: [PATCH 01/17] MySQL Plugin Version 1 --- mysql/mysql.py | 729 ++++++++++++++++++------------------------------- 1 file changed, 267 insertions(+), 462 deletions(-) diff --git a/mysql/mysql.py b/mysql/mysql.py index 9af34284..55d7fd2e 100644 --- a/mysql/mysql.py +++ b/mysql/mysql.py @@ -1,493 +1,298 @@ #!/usr/bin/python3 -""" -Site24x7 MySql table stats Plugin -""" -import traceback -import re import json -import os +import pymysql +import traceback +from decimal import Decimal -VERSION_QUERY = 'SELECT VERSION()' +try: + import psutil + HAS_PSUTIL = True +except ImportError: + HAS_PSUTIL = False -#if any impacting changes to this plugin kindly increment the plugin version here. PLUGIN_VERSION = "1" +HEARTBEAT = True -#Setting this to true will alert you when there is a communication problem while posting plugin data to server -HEARTBEAT="true" - -#Config Section: -MYSQL_HOST = "localhost" - -MYSQL_PORT="3306" - -MYSQL_USERNAME="user" -MYSQL_PASSWORD="" +MYSQL_DEFAULTS = { + "host": "localhost", + "port": 3306, + "username": "site24x7", + "password": "StrongPassword123!" +} -METRICS_JSON={ - "Uptime":"uptime", - "Open_tables":"open_tables", - "Slow_queries":"slow_queries", - - #Threads - "Threads_connected":"connected", - "threads_running":"running", - "Threads_cached":"cached", - "Threads_created":"created", - - # Handler - "Handler_rollback":"handler_rollback", - "Handler_delete":"handler_delete", - "Handler_read_first":"read_first", - "Handler_read_key":"read_key", - "Handler_read_rnd_next":"read_rnd_next", - "Handler_read_rnd":"read_rnd", - "Handler_update":"handler_update", - "Handler_write":"handler_write", - - #Buffer pool - "Innodb_buffer_pool_pages_total":"buffer_pool_pages_total", - "Innodb_buffer_pool_pages_free":"buffer_pool_pages_free", - "Innodb_buffer_pool_pages_dirty":"buffer_pool_pages_dirty", - "Innodb_buffer_pool_pages_data":"buffer_pool_pages_data", - "Innodb_buffer_pool_wait_free":"buffer_pool_wait_free", - "Innodb_log_waits":"log_waits", - "Innodb_row_lock_time_avg":"row_lock_time_avg", - "Innodb_row_lock_waits":"row_lock_waits", - "Innodb_buffer_pool_pages_flushed":"buffer_pool_pages_flushed", - "Innodb_buffer_pool_read_ahead_evicted":"buffer_pool_read_ahead_evicted", - "Innodb_buffer_pool_read_ahead":"buffer_pool_read_ahead", - "Innodb_buffer_pool_read_ahead_rnd":"buffer_pool_read_ahead_rnd", - "Innodb_buffer_pool_read_requests":"buffer_pool_read_requests", - "Innodb_buffer_pool_reads":"buffer_pool_reads", - "Innodb_buffer_pool_write_requests":"buffer_pool_write_requests", - "Innodb_data_fsyncs":"data_fsyncs", - "Innodb_data_pending_fsyncs":"data_pending_fsyncs", - "Innodb_data_pending_reads":"data_pending_reads", - "Innodb_data_pending_writes":"data_pending_writes", - "Innodb_data_reads":"data_reads", - "Innodb_data_writes":"data_writes", - "Innodb_log_write_requests":"log_write_requests", - "Innodb_log_writes":"log_writes", - "Innodb_os_log_fsyncs":"os_log_fsyncs", - "Innodb_os_log_pending_fsyncs":"os_log_pending_fsyncs", - "Innodb_os_log_pending_writes":"os_log_pending_writes", - "Innodb_os_log_written":"os_log_written", - "Innodb_pages_created":"pages_created", - "Innodb_pages_read":"pages_read", - "Innodb_pages_written":"pages_written", - "Innodb_rows_deleted":"rows_deleted", - "Innodb_rows_inserted":"rows_inserted", - "Innodb_rows_read":"rows_read", - "Innodb_rows_updated":"rows_updated", - # Query cache items - # The query cache is deprecated as of MySQL 5.7.20, and is removed in MySQL 8.0. Deprecation - "Qcache_hits":"hits", - "Qcache_free_memory":"free_memory", - "Qcache_not_cached":"not_cached", - "Qcache_queries_in_cache":"in_cache", - "Qcache_free_blocks":"free_blocks", - "Qcache_inserts":"inserts", - "Qcache_lowmem_prunes":"lowmem_prunes", - "Qcache_total_blocks":"total_blocks", - - # Aborted connections and clients - "Aborted_clients":"aborted_clients", - "Aborted_connects":"aborted_connects", - # Bytes sent and received - "Bytes_received":"received", - "Bytes_sent":"sent", - - #Connection - - "Connection_errors_max_connections":"connection_errors_max_connections", - "max_connections":"max_connections", - "Max_used_connections":"max_used_connections", - - # Created temporary tables in memory and on disk - "Created_tmp_tables":"tmp_tables", - "Created_tmp_disk_tables":"disk_tables", - "Created_tmp_files":"tmp_files", - #Select - "Select_full_join":"full_join", - "Select_full_range_join":"full_range_join", - "Select_range":"select_range", - "Select_range_check":"range_check", - "Select_scan":"select_scan", - "Max_execution_time_exceeded":"max_execution_time_exceeded", - - # open files - "Open_files":"open_files", - #"open_files_limit":"open_files_limit", - "Table_locks_waited":"table_locks_waited", - - #Table cache - "Table_open_cache_hits":"open_cache_hits", - "Table_open_cache_misses":"open_cache_misses", - "Table_open_cache_overflows":"open_cache_overflows", - - #Com - "Com_commit":"commit", - "Com_delete":"com_delete", - "Com_delete_multi":"delete_multi", - "Com_insert":"com_insert", - "Com_insert_select":"insert_select", - "Com_replace_select":"replace_select", - "Com_rollback":"com_rollback", - "Com_select":"select", - "Com_update":"com_update", - "Com_update_multi":"update_multi", - #Prepared statement - "Prepared_stmt_count":"prepared_stmt_count", - #Queries - "Queries":"application_queries", - #Questions - "Questions":"client_queries", - #Sort - "Sort_merge_passes":"merge_passes", - "Sort_range":"range", - "Sort_rows":"rows", - "Sort_scan":"scan", - #MyISAM Key Cache - "Key_blocks_not_flushed":"blocks_not_flushed", - "Key_read_requests":"read_requests", - "Key_reads":"key_reads", - "Key_write_requests":"write_requests", - "Key_writes":"key_writes" +METRICS_UNITS = { + "Connection_usage": "%", + "Open_files_usage": "%", + "Database": { + "Db_size_MB": "MB", + "Index_MB": "MB" + }, + "Session": { + "Lock_Latency": "ms", + "CPU": "s", + "Memory": "KB" } - -REPLICATION_JSON = { - "Slave_IO_State": "slave_IO_state", - "Replica_IO_State": "slave_IO_state", - "Master_Host": "master_host", - "Source_Host": "master_host", - "Master_User": "master_user", - "Source_User": "master_user", - "Connect_Retry": "connect_retry", - "Master_Server_Id": "master_server_id", - "Source_Server_Id": "master_server_id", - "Master_Retry_Count": "master_retry_count", - "Source_Retry_Count": "master_retry_count", - "Skip_Counter": "skip_counter", - "Relay_Log_Space": "relay_log_space", - "Seconds_Behind_Master": "seconds_behind_master", # For MySQL 5.7 - "Seconds_Behind_Source": "seconds_behind_master", # For MySQL 8.0+ - "Last_IO_Errno": "last_IO_errno", - "Last_SQL_Errno": "last_sql_errno", - "Slave_IO_Running": "slave_IO_running", - "Replica_IO_Running": "slave_IO_running", - "Slave_SQL_Running": "slave_sql_running", - "Replica_SQL_Running": "slave_sql_running" } - -#Mention the units of your metrics in this python dictionary. If any new metrics are added make an entry here for its unit. -METRICS_UNITS={'uptime':'seconds', - 'row_length':'bytes', - 'data_length': 'bytes', - 'max_data_length': 'bytes', - 'index_length': 'bytes', - 'row_count': 'units', - 'connection_usage':'%', - 'open_files_usage':'%', - 'row_lock_time_avg':'ms', - 'received':'bytes', - 'sent':'bytes', - 'relay_log_space':'bytes', - 'os_log_written':'bytes', - 'free_memory':'bytes', - 'seconds_behind_master':'seconds' - } - +METRICS_MAPPING = { + "Database": ["name", "Db_size_MB", "Index_MB", "Number_of_Tables", "status"], + "Session": ["name", "User", "db", "Memory", "CPU", "Lock_Latency"], + "Threads": [ + "Threads_running", "Threads_connected", "Threads_cached", + "Threads_created", "Aborted_clients", "Aborted_connects", + "Max_used_connections", "Connections" + ], + "Handler": [ + "Com_select", "Com_insert", "Com_update", "Com_delete", + "Com_replace", "Com_load", "Handler_read_first", "Handler_read_key", + "Handler_read_rnd", "Handler_read_rnd_next", "Handler_write", + "Handler_update", "Handler_delete", "Handler_commit", "Handler_rollback" + ], + "Query and Storage": [ + "Queries", "Questions", "Slow_queries", "Opened_tables", "Opened_files", + "Binlog_cache_use", "Binlog_cache_disk_use", "Bytes_received", "Bytes_sent", + "Com_commit", "Com_rollback", "Table_locks_waited", "Table_locks_immediate", + "Created_tmp_files", "Created_tmp_tables", "Created_tmp_disk_tables", + "Commands_per_second", "Avg_query_time", "Max_used_connections", "Connection_usage", + "Open_files_usage", "Innodb_buffer_pool_pages_data", "Innodb_buffer_pool_pages_dirty", + "Innodb_buffer_pool_pages_free", "Innodb_rows_deleted", "Innodb_rows_inserted", + "Innodb_rows_updated", "Slave_running", "Slave_sql_running", "Slave_io_running", + "Seconds_behind_master", "Relay_log_space", "Master_host", "Master_user", + "Master_retry_count", "Skip_counter", "Open_files_limit", "Open_files_used", + "Key_reads", "Key_writes", "Key_blocks_used", "Key_blocks_unused", + "Key_blocks_not_flushed", "Innodb_buffer_pool_pages_total", "Innodb_pages_read", + "Innodb_pages_written", "Innodb_log_writes", "Innodb_log_waits", + "Innodb_buffer_pool_reads", "Innodb_buffer_pool_write_requests", "Innodb_data_reads", + "Innodb_data_writes", "Innodb_pages_created", "Innodb_row_lock_time_avg", + "Innodb_row_lock_time_max", "Innodb_row_lock_waits" + ] +} -class MySQL(object): - - def __init__(self,args): +class MySQLMonitor: + def __init__(self, args): + self.args = args + self.host = getattr(args, "host", MYSQL_DEFAULTS["host"]) + self.port = int(getattr(args, "port", MYSQL_DEFAULTS["port"])) + self.username = getattr(args, "username", MYSQL_DEFAULTS["username"]) + self.password = getattr(args, "password", MYSQL_DEFAULTS["password"]) + self.maindata = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "units": METRICS_UNITS, + "tabs": { + "Database and Session": { + "order": 1, + "tablist": ["Database", "Session"] + }, + "Threads": {"order": 2, "tablist": METRICS_MAPPING["Threads"]}, + "Handler": {"order": 3, "tablist": METRICS_MAPPING["Handler"]}, + "Query and Storage": {"order": 4, "tablist": METRICS_MAPPING["Query and Storage"]} + } + } self.connection = None - self.host = args.host - self.port = args.port - self.username = args.username - self.password = args.password - - self.logsenabled=args.logs_enabled - self.logtypename=args.log_type_name - self.logfilepath=args.log_file_path - + self.cursor = None - #execute a mysql query and returns a dictionary - def executeQuery(self, query): + def safe_int(self, val): try: - cursor = self.connection.cursor() - cursor.execute(query) - metric = {} - field_names = [i[0] for i in cursor.description] - for entry in cursor: - for i in range(len(entry)): - metric[field_names[i]] = entry[i] - return metric + return int(val) + except Exception: + return 0 + + def safe_float(self, val): + try: + return float(val) + except Exception: + return 0.0 + + def to_str(self, val): + try: + f = float(val) + return f"{f:.2f}" + except Exception: + return str(val) + + def connect(self): + try: + self.connection = pymysql.connect( + host=self.host, + port=self.port, + user=self.username, + password=self.password, + cursorclass=pymysql.cursors.DictCursor, + autocommit=True, + connect_timeout=10 + ) + self.cursor = self.connection.cursor() + return True except Exception as e: - metric["error"] = str(e) - return metric - def executeQuery_replica(self, query): + self.maindata["availability"] = 0 + self.maindata["msg"] = f"Connection error: {repr(e)}" + return False + + def close(self): try: - cursor = self.connection.cursor() + if self.cursor: + self.cursor.close() + if self.connection: + self.connection.close() + except Exception: + pass + + def collect_database(self): + dbs = [] + try: + self.cursor.execute(""" + SELECT table_schema AS name, + ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS Db_size_MB, + ROUND(SUM(index_length) / 1024 / 1024, 2) AS Index_MB, + COUNT(table_name) AS Number_of_Tables, + 1 AS status + FROM information_schema.tables + WHERE table_schema NOT IN ('mysql', 'performance_schema', 'sys', 'information_schema') + GROUP BY table_schema + LIMIT 25 + """) + for row in self.cursor.fetchall(): + dbs.append({k: self.to_str(v) if k != "name" else v for k, v in row.items()}) except Exception as e: - metric["error"] = str(e) - return metric - def executeQuery_mysql(self, con, query): + self.maindata["availability"] = 0 + self.maindata["msg"] = f"Database collection error: {e}" + return dbs + + def collect_session(self): + sess = [] try: - cursor = con.cursor() - cursor.execute(query) - metric = {} - for entry in cursor: + self.cursor.execute(""" + SELECT THREAD_ID, PROCESSLIST_ID + FROM performance_schema.threads + WHERE TYPE = 'FOREGROUND' + """) + thread_pid_map = {r["THREAD_ID"]: r["PROCESSLIST_ID"] for r in self.cursor.fetchall()} + self.cursor.execute("SHOW FULL PROCESSLIST") + sessions = self.cursor.fetchall() + proc = None + if HAS_PSUTIL: try: - metric[entry[0]] = float(entry[1]) - except ValueError as e: - metric[entry[0]] = entry[1] + procs = [p for p in psutil.process_iter(['pid', 'name']) if 'mysqld' in (p.info.get('name', '') or '')] + pid = procs[0].pid if procs else None + proc = psutil.Process(pid) if pid else None + except Exception: + proc = None + pid_idx_map = {} + count = 0 + for s in sessions: + if count >= 25: + break + pid = s.get("Id", 0) + pid_str = f"PID_{pid}" + rec = { + "name": pid_str, # renamed from Id to name + "User": str(s.get("User") or ""), + "db": str(s.get("db") or ""), + # "Host": str(s.get("Host") or ""), # commented intentionally + # "State": str(s.get("State") or ""), # commented intentionally + "Memory": "0.00", + "CPU": "0.00", + "Lock_Latency": self.to_str(s.get("Time", 0)) + } + sess.append(rec) + pid_idx_map[pid] = count + count += 1 + if proc: + for thread in proc.threads(): + tid = thread.id + if tid in thread_pid_map: + sid = thread_pid_map[tid] + if sid in pid_idx_map: + idx = pid_idx_map[sid] + sess[idx]["CPU"] = self.to_str(round(thread.user_time + thread.system_time, 2)) + sess[idx]["Memory"] = self.to_str(int(proc.memory_info().rss / 1024)) + except Exception as e: + self.maindata["availability"] = 0 + self.maindata["msg"] = f"Session collection error: {e}" + return sess - return metric - except pymysql.OperationalError as message: - pass - - def getDbConnection(self): + def collect_performance(self): + perf = {} try: - import pymysql - db = pymysql.connect(host=self.host,user=self.username,passwd=self.password,port=int(self.port)) - self.connection = db + self.cursor.execute(""" + SELECT digest_text, count_star, sum_timer_wait, avg_timer_wait + FROM performance_schema.events_statements_summary_by_digest + ORDER BY count_star DESC + LIMIT 10 + """) + rows = self.cursor.fetchall() + for i, row in enumerate(rows, 1): + perf[f"PS_{i}_count"] = self.safe_int(row.get("count_star", 0)) + perf[f"PS_{i}_sum_timer"] = self.safe_float(row.get("sum_timer_wait", 0.0)) + perf[f"PS_{i}_avg_timer"] = self.safe_float(row.get("avg_timer_wait", 0.0)) except Exception as e: - global con_error - con_error=str(e) - #traceback.print_exc() - return False - return True + perf["perf_error"] = f"Performance collection error: {e}" + return perf - def checkPreRequisites(self,data): - bool_result = True + def collect_metrics(self): + if not self.connect(): + return self.maindata try: - import pymysql - except Exception: - data['status']=0 - data['msg']='pymysql module not installed' - bool_result=False - pymysql_returnVal=os.system('pip install pymysql >/dev/null 2>&1') - if pymysql_returnVal==0: - bool_result=True - data.pop('status') - data.pop('msg') - return bool_result,data + self.cursor.execute("SHOW GLOBAL STATUS") + status = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} + self.cursor.execute("SHOW GLOBAL VARIABLES") + variables = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} + for tab_name in ["Threads", "Handler", "Query and Storage"]: + for metric in METRICS_MAPPING[tab_name]: + val = status.get(metric) or variables.get(metric) + if val is not None: + self.maindata[metric] = self.to_str(val) + max_used = status.get("Max_used_connections") + self.maindata["Max_used_connections"] = self.to_str(int(max_used)) if max_used else "0.00" + max_conn = int(variables.get("max_connections", "0")) + threads = int(status.get("Threads_running", "0")) + open_files = int(status.get("Open_files", "0")) + open_files_limit = int(variables.get("open_files_limit", "0")) + self.maindata["Connection_usage"] = self.to_str(round(threads / max_conn * 100, 2)) if max_conn else "0.00" + self.maindata["Open_files_usage"] = self.to_str(round(open_files / open_files_limit * 100, 2)) if open_files_limit else "0.00" - def metricCollector(self): - data = {} + self.maindata["Database"] = self.collect_database() + self.maindata["Session"] = self.collect_session() - #bool_result,data = self.checkPreRequisites(data) - bool_result = True - - if bool_result==False: - return data - else: - try: - import pymysql - except Exception: - data['status']=0 - data['msg']='pymysql module not installed\n Solution : Use the following command to install pymysql\n pip install pymysql \n(or)\n pip3 install pymysql' - return data + self.maindata["s247config"] = { + "childdiscovery": ["Session"] + } - if not self.getDbConnection(): - data['status']=0 - data['msg']='Connection Error: '+con_error - return data - - try: - con = self.connection - # get MySQL version - try: - cursor = con.cursor() - cursor.execute(VERSION_QUERY) - result = cursor.fetchone() - data['mysql_version'] = result[0] - version=result[0].split(".") - if int(version[0]) >=8: - slave_query="SHOW REPLICA STATUS" - #master_query="SHOW BINARY LOG STATUS" - else: - slave_query='SHOW SLAVE STATUS' - master_query='SHOW MASTER STATUS' - - except pymysql.OperationalError as message: - data["msg"] = repr(e) - data["status"]=0 - return data - - cursor.execute(slave_query) - myresult_slave_key=cursor.description - myresult_slave=cursor.fetchall() - try: - cursor.execute(master_query) - except pymysql.ProgrammingError as e: - if int(version[0]) >=8: - cursor.execute('SHOW BINARY LOG STATUS') - else: - data["msg"] = repr(e) - data["status"]=0 - - myresult_master=cursor.fetchall() - if myresult_master and myresult_slave: - data['mysql_node_type']='Master & slave' - for i in range(len(myresult_slave[0])): - if REPLICATION_JSON.get(myresult_slave_key[i][0]): - - data[REPLICATION_JSON[myresult_slave_key[i][0]]]=myresult_slave[0][i] - #data['mysql_node_type']='Slave' - elif myresult_master: - data['mysql_node_type']='Master' - elif myresult_slave : - for i in range(len(myresult_slave[0])): - if REPLICATION_JSON.get(myresult_slave_key[i][0]): - - data[REPLICATION_JSON[myresult_slave_key[i][0]]]=myresult_slave[0][i] - data['mysql_node_type']='Slave' - else: - data['mysql_node_type']='Standalone' + perf = self.collect_performance() + self.maindata.update(perf) + + self.close() + return self.maindata + except Exception as e: + self.maindata["availability"] = 0 + self.maindata["msg"] = f"Metric collection error: {e}\n{traceback.format_exc()}" + self.close() + return self.maindata + + +def normalize(obj): + if isinstance(obj, dict): + return {k: normalize(v) for k, v in obj.items()} + if isinstance(obj, list): + return [normalize(i) for i in obj] + if isinstance(obj, Decimal): + return float(obj) + return obj - - json_file={} - #MySQL Replication - - file_name="mysql_info_"+self.host+".json" - if os.path.exists(file_name): - if os.stat(file_name).st_size == 0: - json_file['MySQLNodeType']=data['mysql_node_type'] - with open(file_name, 'w') as f: - json.dump(json_file, f) - f = open(file_name) - json_val = json.load(f) - json_data = json_val["MySQLNodeType"] - if(json_data != data['mysql_node_type'] and (json_data in ['Slave', 'Master', 'Standalone'])): - json_file['MySQLNodeType']=data['mysql_node_type'] - with open(file_name, 'w') as f: - json.dump(json_file, f) - data["msg"] = "Failover happened -"+json_data+" was Switched to "+data['mysql_node_type'] - data["status"] = 0 - return data - else: - json_file['MySQLNodeType']=data['mysql_node_type'] - with open(file_name, 'w') as f: - json.dump(json_file, f) - else: - json_file['MySQLNodeType']=data['mysql_node_type'] - with open(file_name, 'w') as f: - json.dump(json_file, f) - #global_table = self.executeQuery('select * from information_schema.tables where table_schema="' + self.database + '" and table_name="'+self.table+'"') - data["row_length"] = 0 - data["data_length"] = 0 - data["index_length"] = 0 - data["max_data_length"] = 0 - data["rows_count"] = 0 - global_metrics = self.executeQuery_mysql(con,'SHOW GLOBAL STATUS') - global_variables = self.executeQuery_mysql(con,'SHOW VARIABLES') - """global_db = self.executeQuery_mysql(con, 'SELECT table_schema "DB Name",ROUND(SUM(data_length + index_length) / 1024 / 1024, 1) "DB Size in MB" FROM information_schema.tables GROUP BY table_schema;') - - for k,v in global_db.items(): - db_list = {} - db_list["name"]=k - db_list["size"]=v - db.append(db_list)""" - #data[k]=v - #METRICS_UNITS[k] = "MB" - for attribute_keys in METRICS_JSON: - if attribute_keys in global_metrics: - data[METRICS_JSON[attribute_keys]]=global_metrics[attribute_keys] - elif attribute_keys in global_variables: - data[METRICS_JSON[attribute_keys]]=global_variables[attribute_keys] - else: - data[METRICS_JSON[attribute_keys]]=0 - if 'threads_running' in data and 'max_connections' in global_variables: - data['connection_usage'] = ((data['threads_running'] /global_variables['max_connections'])*100) - else: - data['connection_usage'] = 0 - if 'open_files' in data and 'open_files_limit' in global_variables: - data['open_files_usage'] = ((data['open_files'] /global_variables['open_files_limit'])*100) - else: - data['open_files_usage'] = 0 - #no of reads & writes - if 'Com_insert' in global_metrics and 'Com_replace' in global_metrics and 'Com_update' in global_metrics and 'Com_delete' in global_metrics: - writes = (global_metrics['Com_insert'] +global_metrics['Com_replace'] +global_metrics['Com_update'] +global_metrics['Com_delete']) - data['writes'] = writes - else: - data['writes'] = 0 - # reads - if 'Com_select' in global_metrics and 'qcache_hits' in data: - reads = global_metrics['Com_select'] + data['qcache_hits'] - data['reads'] = reads - else: - data['reads'] = 0 - reads = 0 - try: - data['rw_ratio'] = reads/writes - except ZeroDivisionError: - data['rw_ratio'] = 0 - except Exception as e: - data["msg"] = repr(e) - data["status"]=0 - - # transactions - if 'Com_commit' in global_metrics and 'Com_rollback' in global_metrics: - transactions = (global_metrics['Com_commit'] +global_metrics['Com_rollback']) - data['transactions'] = transactions - else: - data['transactions'] = 0 - # slave_running - if 'Slave_running' in global_metrics: - result = global_metrics['Slave_running'] - if result == 'OFF': - result = 0 - else: - result = 1 - data['slave_running'] = result - else: - if myresult_slave: - data['slave_running'] = 1 - else: - data['slave_running'] = 0 - - cursor.execute('SHOW VARIABLES LIKE "wsrep_cluster_name"') - cluster=cursor.fetchall() - if cluster: - data['tags']="MYSQL_CLUSTER:"+cluster[0][1]+",MYSQL_NODE:"+self.host+"" - except Exception as e: - data["error"] = repr(e) - cursor.close() - con.close() - return data - applog={} - if(self.logsenabled in ['True', 'true', '1']): - applog["logs_enabled"]=True - applog["log_type_name"]=self.logtypename - applog["log_file_path"]=self.logfilepath - else: - applog["logs_enabled"]=False - data['applog'] = applog - #data['tags']="Node Type:"+data['mysql_node_type']+"" - data['units']=METRICS_UNITS - data['plugin_version'] = PLUGIN_VERSION - data['heartbeat_required']=HEARTBEAT - cursor.close() - con.close() - return data if __name__ == "__main__": - import argparse - parser=argparse.ArgumentParser() - parser.add_argument('--host',help="Host Name",nargs='?', default= MYSQL_HOST) - parser.add_argument('--port',help="Port",nargs='?', default= MYSQL_PORT) - parser.add_argument('--username',help="username", default= MYSQL_USERNAME) - parser.add_argument('--password',help="Password", default= MYSQL_PASSWORD) - - parser.add_argument('--logs_enabled', help='enable log collection for this plugin application',default="False") - parser.add_argument('--log_type_name', help='Display name of the log type', nargs='?', default=None) - parser.add_argument('--log_file_path', help='list of comma separated log file paths', nargs='?', default=None) - - args=parser.parse_args() - mysql_plugins = MySQL(args) - result = mysql_plugins.metricCollector() - print(json.dumps(result)) + + parser = argparse.ArgumentParser() + parser.add_argument("--host", default=MYSQL_DEFAULTS["host"]) + parser.add_argument("--port", default=MYSQL_DEFAULTS["port"]) + parser.add_argument("--username", default=MYSQL_DEFAULTS["username"]) + parser.add_argument("--password", default=MYSQL_DEFAULTS["password"]) + + args = parser.parse_args() + monitor = MySQLMonitor(args) + result = monitor.collect_metrics() + result = normalize(result) + print(json.dumps(result, indent=2)) + From 3b6325b18700d324bd2672ca71e4978df812fa1f Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Tue, 2 Sep 2025 14:39:10 +0530 Subject: [PATCH 02/17] Final MySQL plugin code --- mysql/mysql.py | 430 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 343 insertions(+), 87 deletions(-) diff --git a/mysql/mysql.py b/mysql/mysql.py index 55d7fd2e..778f4582 100644 --- a/mysql/mysql.py +++ b/mysql/mysql.py @@ -3,71 +3,266 @@ import pymysql import traceback from decimal import Decimal - -try: - import psutil - HAS_PSUTIL = True -except ImportError: - HAS_PSUTIL = False - +# import psutil +# HAS_PSUTIL = True +HAS_PSUTIL = False PLUGIN_VERSION = "1" HEARTBEAT = True - MYSQL_DEFAULTS = { "host": "localhost", "port": 3306, "username": "site24x7", "password": "StrongPassword123!" } - METRICS_UNITS = { + "Uptime": "seconds", "Connection_usage": "%", "Open_files_usage": "%", + "Open_tables": "units", + "threads_running": "units", + "Innodb_buffer_pool_wait_free": "units", + "Innodb_buffer_pool_pages_flushed": "units", + "Innodb_buffer_pool_read_ahead_evicted": "units", + "Innodb_buffer_pool_read_ahead": "units", + "Innodb_buffer_pool_read_ahead_rnd": "units", + "Innodb_buffer_pool_read_requests": "units", + "Innodb_data_fsyncs": "units", + "Innodb_data_pending_fsyncs": "units", + "Innodb_data_pending_reads": "units", + "Innodb_data_pending_writes": "units", + "Innodb_log_write_requests": "units", + "Innodb_os_log_fsyncs": "units", + "Innodb_os_log_pending_fsyncs": "units", + "Innodb_os_log_pending_writes": "units", + "Innodb_os_log_written": "units", + "Innodb_rows_read": "units", + "Qcache_hits": "units", + "Qcache_free_memory": "bytes", + "Qcache_not_cached": "units", + "Qcache_queries_in_cache": "units", + "Qcache_free_blocks": "units", + "Qcache_inserts": "units", + "Qcache_lowmem_prunes": "units", + "Qcache_total_blocks": "units", + "Connection_errors_max_connections": "units", + # Added metrics for displayed database table + "Fetch_Latency_ms": "ms", + "Insert_Latency_ms": "ms", + "Throughput_qps": "queries/s", + "Queries_executed": "units", + "Errors": "units", + # Added ManageEngine metrics with direct SQL access (examples) + "Version": "text", + "Type_Instance": "text", + "Connections_attempted": "count", + "Table_open_cache_hit_ratio": "%", + "Buffer_pool_utilization": "%", + "Threads_connected": "count", + "Threads_running": "count", + "Threads_cached": "count", + "Threads_created": "count", + "Update_command": "count", + "Insert_command": "count", + "Insert_select_command": "count", + "Delete_command": "count", + "Create_db_command": "count", + "Drop_db_command": "count", + "Drop_table_command": "count", + "Alter_table_command": "count", + "Binlog_command": "count", + "Load_command": "count", + "Replace_command": "count", + "Replace_select_command": "count", + "Select_command": "count", + "Shutdown_command": "count", + "Group_replication_start_command": "count", + "Group_replication_stop_command": "count", + "Slave_start_command": "count", + "Slave_stop_command": "count", + "Change_master_command": "count", + "Revoke_command": "count", + "Revoke_all_command": "count", + "Rollback_command": "count", + "Rollback_to_savepoint_command": "count", + "Savepoint_command": "count", + "Handler_external_lock": "count", + "Handler_savepoint": "count", + "Handler_savepoint_rollback": "count", + "Binlog_stmt_cache_use": "count", + "Binlog_stmt_cache_disk_use": "count", + "Binlog_size": "bytes", + "Binlog_file_count": "count", + "Relay_log_size": "bytes", + "Relay_log_file_count": "count", + "InnoDB_active_transactions": "count", + "InnoDB_locked_transactions": "count", + "InnoDB_current_transactions": "count", + "InnoDB_history_list_length": "count", + "InnoDB_lock_structs": "count", + "InnoDB_tables_in_use": "count", + "InnoDB_locked_tables": "count", + # Existing nested units "Database": { "Db_size_MB": "MB", - "Index_MB": "MB" + "Index_MB": "MB", + "Fetch_Latency_ms": "ms", + "Insert_Latency_ms": "ms", + "Throughput_qps": "queries/s", + "Queries_executed": "units", + "Errors": "units" }, "Session": { "Lock_Latency": "ms", - "CPU": "s", - "Memory": "KB" + # "CPU": "s", # Commented out as requested + # "Memory": "KB" # Commented out as requested } } - METRICS_MAPPING = { - "Database": ["name", "Db_size_MB", "Index_MB", "Number_of_Tables", "status"], - "Session": ["name", "User", "db", "Memory", "CPU", "Lock_Latency"], + "Database": [ + "name", "Db_size_MB", "Index_MB", "Number_of_Tables", "status", "Open_tables", + "Fetch_Latency_ms", "Insert_Latency_ms", "Throughput_qps", "Queries_executed", "Errors" + ], + "Session": [ + "name", "User", "db", + # "Memory", # Commented out as requested + # "CPU", # Commented out as requested + "Lock_Latency" + ], "Threads": [ - "Threads_running", "Threads_connected", "Threads_cached", - "Threads_created", "Aborted_clients", "Aborted_connects", - "Max_used_connections", "Connections" + "Threads_running", + "Threads_connected", + "Threads_cached", + "Threads_created", + "Aborted_clients", + "Aborted_connects", + "Max_used_connections", + "Connections", + "threads_running" # from old name compatibility ], "Handler": [ - "Com_select", "Com_insert", "Com_update", "Com_delete", - "Com_replace", "Com_load", "Handler_read_first", "Handler_read_key", - "Handler_read_rnd", "Handler_read_rnd_next", "Handler_write", - "Handler_update", "Handler_delete", "Handler_commit", "Handler_rollback" + "Com_select", + "Com_insert", + "Com_update", + "Com_delete", + "Com_replace", + "Com_load", + "Handler_read_first", + "Handler_read_key", + "Handler_read_rnd", + "Handler_read_rnd_next", + "Handler_write", + "Handler_update", + "Handler_delete", + "Handler_commit", + "Handler_rollback", + "Com_delete_multi", + "Com_insert_select", + "Com_replace_select", + "Com_update_multi" ], "Query and Storage": [ - "Queries", "Questions", "Slow_queries", "Opened_tables", "Opened_files", - "Binlog_cache_use", "Binlog_cache_disk_use", "Bytes_received", "Bytes_sent", - "Com_commit", "Com_rollback", "Table_locks_waited", "Table_locks_immediate", - "Created_tmp_files", "Created_tmp_tables", "Created_tmp_disk_tables", - "Commands_per_second", "Avg_query_time", "Max_used_connections", "Connection_usage", - "Open_files_usage", "Innodb_buffer_pool_pages_data", "Innodb_buffer_pool_pages_dirty", - "Innodb_buffer_pool_pages_free", "Innodb_rows_deleted", "Innodb_rows_inserted", - "Innodb_rows_updated", "Slave_running", "Slave_sql_running", "Slave_io_running", - "Seconds_behind_master", "Relay_log_space", "Master_host", "Master_user", - "Master_retry_count", "Skip_counter", "Open_files_limit", "Open_files_used", - "Key_reads", "Key_writes", "Key_blocks_used", "Key_blocks_unused", - "Key_blocks_not_flushed", "Innodb_buffer_pool_pages_total", "Innodb_pages_read", - "Innodb_pages_written", "Innodb_log_writes", "Innodb_log_waits", - "Innodb_buffer_pool_reads", "Innodb_buffer_pool_write_requests", "Innodb_data_reads", - "Innodb_data_writes", "Innodb_pages_created", "Innodb_row_lock_time_avg", - "Innodb_row_lock_time_max", "Innodb_row_lock_waits" + "Queries", + "Questions", + "Slow_queries", + "Opened_tables", + "Opened_files", + "Binlog_cache_use", + "Binlog_cache_disk_use", + "Bytes_received", + "Bytes_sent", + "Com_commit", + "Com_rollback", + "Table_locks_waited", + "Table_locks_immediate", + "Created_tmp_files", + "Created_tmp_tables", + "Created_tmp_disk_tables", + "Commands_per_second", + "Avg_query_time", + "Max_used_connections", + "Connection_usage", + "Open_files_usage", + "Innodb_buffer_pool_pages_data", + "Innodb_buffer_pool_pages_dirty", + "Innodb_buffer_pool_pages_free", + "Innodb_rows_deleted", + "Innodb_rows_inserted", + "Innodb_rows_updated", + "Slave_running", + "Slave_sql_running", + "Slave_io_running", + "Seconds_behind_master", + "Relay_log_space", + "Master_host", + "Master_user", + "Master_retry_count", + "Skip_counter", + "Open_files_limit", + "Open_files_used", + "Key_reads", + "Key_writes", + "Key_blocks_used", + "Key_blocks_unused", + "Key_blocks_not_flushed", + "Innodb_buffer_pool_pages_total", + "Innodb_pages_read", + "Innodb_pages_written", + "Innodb_log_writes", + "Innodb_log_waits", + "Innodb_buffer_pool_reads", + "Innodb_buffer_pool_write_requests", + "Innodb_data_reads", + "Innodb_data_writes", + "Innodb_pages_created", + "Innodb_row_lock_time_avg", + "Innodb_row_lock_time_max", + "Innodb_row_lock_waits", + # Added missing old metrics + "Innodb_buffer_pool_wait_free", + "Innodb_buffer_pool_pages_flushed", + "Innodb_buffer_pool_read_ahead_evicted", + "Innodb_buffer_pool_read_ahead", + "Innodb_buffer_pool_read_ahead_rnd", + "Innodb_buffer_pool_read_requests", + "Innodb_data_fsyncs", + "Innodb_data_pending_fsyncs", + "Innodb_data_pending_reads", + "Innodb_data_pending_writes", + "Innodb_log_write_requests", + "Innodb_os_log_fsyncs", + "Innodb_os_log_pending_fsyncs", + "Innodb_os_log_pending_writes", + "Innodb_os_log_written", + "Innodb_rows_read", + "Qcache_hits", + "Qcache_free_memory", + "Qcache_not_cached", + "Qcache_queries_in_cache", + "Qcache_free_blocks", + "Qcache_inserts", + "Qcache_lowmem_prunes", + "Qcache_total_blocks", + "Connection_errors_max_connections", + "max_connections", + "Select_full_join", + "Select_full_range_join", + "Select_range", + "Select_range_check", + "Select_scan", + "Max_execution_time_exceeded", + "Open_files", + "Table_open_cache_hits", + "Table_open_cache_misses", + "Table_open_cache_overflows", + "Prepared_stmt_count", + "Sort_merge_passes", + "Sort_range", + "Sort_rows", + "Sort_scan", + "Key_read_requests", + "Key_write_requests" ] } - class MySQLMonitor: def __init__(self, args): self.args = args @@ -84,33 +279,38 @@ def __init__(self, args): "order": 1, "tablist": ["Database", "Session"] }, - "Threads": {"order": 2, "tablist": METRICS_MAPPING["Threads"]}, - "Handler": {"order": 3, "tablist": METRICS_MAPPING["Handler"]}, - "Query and Storage": {"order": 4, "tablist": METRICS_MAPPING["Query and Storage"]} + "Threads": { + "order": 2, + "tablist": METRICS_MAPPING["Threads"] + }, + "Handler": { + "order": 3, + "tablist": METRICS_MAPPING["Handler"] + }, + "Query and Storage": { + "order": 4, + "tablist": METRICS_MAPPING["Query and Storage"] + } } } self.connection = None self.cursor = None - def safe_int(self, val): try: return int(val) except Exception: return 0 - def safe_float(self, val): try: return float(val) except Exception: return 0.0 - def to_str(self, val): try: f = float(val) return f"{f:.2f}" except Exception: return str(val) - def connect(self): try: self.connection = pymysql.connect( @@ -128,7 +328,6 @@ def connect(self): self.maindata["availability"] = 0 self.maindata["msg"] = f"Connection error: {repr(e)}" return False - def close(self): try: if self.cursor: @@ -137,10 +336,10 @@ def close(self): self.connection.close() except Exception: pass - def collect_database(self): dbs = [] try: + # Collect base db info self.cursor.execute(""" SELECT table_schema AS name, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS Db_size_MB, @@ -152,13 +351,98 @@ def collect_database(self): GROUP BY table_schema LIMIT 25 """) - for row in self.cursor.fetchall(): - dbs.append({k: self.to_str(v) if k != "name" else v for k, v in row.items()}) + db_size_info = {row["name"]: row for row in self.cursor.fetchall()} + # Fetch Latency (average SELECT latency in ms) + self.cursor.execute(""" + SELECT ROUND(IFNULL(AVG_TIMER_WAIT/1000000000, 0)*1000, 2) AS avg_fetch_latency_ms + FROM performance_schema.events_statements_summary_by_digest + WHERE digest_text LIKE 'select%%' AND SCHEMA_NAME IS NOT NULL + LIMIT 1 + """) + fetch_latency = 0.0 + r = self.cursor.fetchone() + if r and "avg_fetch_latency_ms" in r: + fetch_latency = float(r["avg_fetch_latency_ms"]) + # Insert Latency (average INSERT latency in ms) + self.cursor.execute(""" + SELECT ROUND(IFNULL(AVG_TIMER_WAIT/1000000000, 0)*1000, 2) AS avg_insert_latency_ms + FROM performance_schema.events_statements_summary_by_digest + WHERE digest_text LIKE 'insert%%' AND SCHEMA_NAME IS NOT NULL + LIMIT 1 + """) + insert_latency = 0.0 + r = self.cursor.fetchone() + if r and "avg_insert_latency_ms" in r: + insert_latency = float(r["avg_insert_latency_ms"]) + # Throughput (queries per second) calculated over uptime + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Questions'") + questions_row = self.cursor.fetchone() + questions = int(questions_row['Value']) if questions_row else 0 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Uptime'") + uptime_row = self.cursor.fetchone() + uptime = int(uptime_row['Value']) if uptime_row else 1 # avoid zero division + throughput_qps = round(questions / uptime, 2) if uptime > 0 else 0.0 + # Queries executed - total queries count + queries_executed = questions + # Errors - sum of errors from SHOW GLOBAL STATUS variables related + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Errors'") + errors_row = self.cursor.fetchone() + errors = int(errors_row['Value']) if errors_row else 0 + # Additional ManageEngine metrics examples fetched by direct SQL queries + self.cursor.execute("SELECT VERSION() AS version") + version_row = self.cursor.fetchone() + version = version_row.get("version", "") if version_row else "" + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Connections'") + connections_attempted_row = self.cursor.fetchone() + connections_attempted = int(connections_attempted_row['Value']) if connections_attempted_row else 0 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Threads_connected'") + threads_connected_row = self.cursor.fetchone() + threads_connected = int(threads_connected_row['Value']) if threads_connected_row else 0 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Table_open_cache_hits'") + table_open_cache_hits_row = self.cursor.fetchone() + table_open_cache_hits = int(table_open_cache_hits_row['Value']) if table_open_cache_hits_row else 0 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Table_open_cache_misses'") + table_open_cache_misses_row = self.cursor.fetchone() + table_open_cache_misses = int(table_open_cache_misses_row['Value']) if table_open_cache_misses_row else 0 + table_open_cache_hit_ratio = 0.0 + total = table_open_cache_hits + table_open_cache_misses + if total > 0: + table_open_cache_hit_ratio = round((table_open_cache_hits / total) * 100, 2) + # Buffer Pool Utilization: sum dirty+clean pages / total pages * 100 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Innodb_buffer_pool_pages_data'") + pages_data_row = self.cursor.fetchone() + pages_data = int(pages_data_row['Value']) if pages_data_row else 0 + self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Innodb_buffer_pool_pages_total'") + pages_total_row = self.cursor.fetchone() + pages_total = int(pages_total_row['Value']) if pages_total_row else 0 + buffer_pool_utilization = 0.0 + if pages_total > 0: + buffer_pool_utilization = round((pages_data / pages_total) * 100, 2) + # Add fetched metrics to each DB record as demo; generally these are global but added for visibility + for dbname, dbdata in db_size_info.items(): + rec = { + "name": dbname, + "Db_size_MB": self.to_str(dbdata.get("Db_size_MB", 0)), + "Index_MB": self.to_str(dbdata.get("Index_MB", 0)), + "Number_of_Tables": self.to_str(dbdata.get("Number_of_Tables", 0)), + "status": 1, + "Open_tables": "0", + "Fetch_Latency_ms": self.to_str(fetch_latency), + "Insert_Latency_ms": self.to_str(insert_latency), + "Throughput_qps": self.to_str(throughput_qps), + "Queries_executed": str(queries_executed), + "Errors": str(errors), + "Version": version, + "Connections_attempted": str(connections_attempted), + "Threads_connected": str(threads_connected), + "Table_open_cache_hit_ratio": str(table_open_cache_hit_ratio), + "Buffer_pool_utilization": str(buffer_pool_utilization) + } + dbs.append(rec) except Exception as e: self.maindata["availability"] = 0 self.maindata["msg"] = f"Database collection error: {e}" return dbs - def collect_session(self): sess = [] try: @@ -170,14 +454,6 @@ def collect_session(self): thread_pid_map = {r["THREAD_ID"]: r["PROCESSLIST_ID"] for r in self.cursor.fetchall()} self.cursor.execute("SHOW FULL PROCESSLIST") sessions = self.cursor.fetchall() - proc = None - if HAS_PSUTIL: - try: - procs = [p for p in psutil.process_iter(['pid', 'name']) if 'mysqld' in (p.info.get('name', '') or '')] - pid = procs[0].pid if procs else None - proc = psutil.Process(pid) if pid else None - except Exception: - proc = None pid_idx_map = {} count = 0 for s in sessions: @@ -186,32 +462,20 @@ def collect_session(self): pid = s.get("Id", 0) pid_str = f"PID_{pid}" rec = { - "name": pid_str, # renamed from Id to name + "name": pid_str, "User": str(s.get("User") or ""), "db": str(s.get("db") or ""), - # "Host": str(s.get("Host") or ""), # commented intentionally - # "State": str(s.get("State") or ""), # commented intentionally - "Memory": "0.00", - "CPU": "0.00", + # "Memory": "0.00", # commented as requested + # "CPU": "0.00", # commented as requested "Lock_Latency": self.to_str(s.get("Time", 0)) } sess.append(rec) pid_idx_map[pid] = count count += 1 - if proc: - for thread in proc.threads(): - tid = thread.id - if tid in thread_pid_map: - sid = thread_pid_map[tid] - if sid in pid_idx_map: - idx = pid_idx_map[sid] - sess[idx]["CPU"] = self.to_str(round(thread.user_time + thread.system_time, 2)) - sess[idx]["Memory"] = self.to_str(int(proc.memory_info().rss / 1024)) except Exception as e: self.maindata["availability"] = 0 self.maindata["msg"] = f"Session collection error: {e}" return sess - def collect_performance(self): perf = {} try: @@ -229,7 +493,6 @@ def collect_performance(self): except Exception as e: perf["perf_error"] = f"Performance collection error: {e}" return perf - def collect_metrics(self): if not self.connect(): return self.maindata @@ -238,6 +501,10 @@ def collect_metrics(self): status = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} self.cursor.execute("SHOW GLOBAL VARIABLES") variables = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} + # Add Uptime explicitly + self.maindata["Uptime"] = self.to_str(status.get("Uptime", 0)) + # Add threads_running old metric name as well + self.maindata["threads_running"] = self.to_str(status.get("Threads_running", 0)) for tab_name in ["Threads", "Handler", "Query and Storage"]: for metric in METRICS_MAPPING[tab_name]: val = status.get(metric) or variables.get(metric) @@ -251,17 +518,13 @@ def collect_metrics(self): open_files_limit = int(variables.get("open_files_limit", "0")) self.maindata["Connection_usage"] = self.to_str(round(threads / max_conn * 100, 2)) if max_conn else "0.00" self.maindata["Open_files_usage"] = self.to_str(round(open_files / open_files_limit * 100, 2)) if open_files_limit else "0.00" - self.maindata["Database"] = self.collect_database() self.maindata["Session"] = self.collect_session() - self.maindata["s247config"] = { - "childdiscovery": ["Session"] + "childdiscovery": ["Session", "Database"] } - perf = self.collect_performance() self.maindata.update(perf) - self.close() return self.maindata except Exception as e: @@ -269,8 +532,6 @@ def collect_metrics(self): self.maindata["msg"] = f"Metric collection error: {e}\n{traceback.format_exc()}" self.close() return self.maindata - - def normalize(obj): if isinstance(obj, dict): return {k: normalize(v) for k, v in obj.items()} @@ -279,20 +540,15 @@ def normalize(obj): if isinstance(obj, Decimal): return float(obj) return obj - - if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser() parser.add_argument("--host", default=MYSQL_DEFAULTS["host"]) parser.add_argument("--port", default=MYSQL_DEFAULTS["port"]) parser.add_argument("--username", default=MYSQL_DEFAULTS["username"]) parser.add_argument("--password", default=MYSQL_DEFAULTS["password"]) - args = parser.parse_args() monitor = MySQLMonitor(args) result = monitor.collect_metrics() result = normalize(result) print(json.dumps(result, indent=2)) - From b07d4d3080f65077e73af5932f418e857a140146 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Fri, 5 Sep 2025 16:32:58 +0530 Subject: [PATCH 03/17] Updated MySQL Plugin code --- mysql/mysql.py | 428 ++++++++++++++++++++++++------------------------- 1 file changed, 210 insertions(+), 218 deletions(-) diff --git a/mysql/mysql.py b/mysql/mysql.py index 778f4582..9f53e582 100644 --- a/mysql/mysql.py +++ b/mysql/mysql.py @@ -3,118 +3,40 @@ import pymysql import traceback from decimal import Decimal -# import psutil -# HAS_PSUTIL = True HAS_PSUTIL = False PLUGIN_VERSION = "1" HEARTBEAT = True MYSQL_DEFAULTS = { "host": "localhost", "port": 3306, - "username": "site24x7", - "password": "StrongPassword123!" + "username": "user", + "password": "" } METRICS_UNITS = { "Uptime": "seconds", "Connection_usage": "%", "Open_files_usage": "%", - "Open_tables": "units", - "threads_running": "units", - "Innodb_buffer_pool_wait_free": "units", - "Innodb_buffer_pool_pages_flushed": "units", - "Innodb_buffer_pool_read_ahead_evicted": "units", - "Innodb_buffer_pool_read_ahead": "units", - "Innodb_buffer_pool_read_ahead_rnd": "units", - "Innodb_buffer_pool_read_requests": "units", - "Innodb_data_fsyncs": "units", - "Innodb_data_pending_fsyncs": "units", - "Innodb_data_pending_reads": "units", - "Innodb_data_pending_writes": "units", - "Innodb_log_write_requests": "units", - "Innodb_os_log_fsyncs": "units", - "Innodb_os_log_pending_fsyncs": "units", - "Innodb_os_log_pending_writes": "units", - "Innodb_os_log_written": "units", - "Innodb_rows_read": "units", - "Qcache_hits": "units", - "Qcache_free_memory": "bytes", - "Qcache_not_cached": "units", - "Qcache_queries_in_cache": "units", - "Qcache_free_blocks": "units", - "Qcache_inserts": "units", - "Qcache_lowmem_prunes": "units", - "Qcache_total_blocks": "units", - "Connection_errors_max_connections": "units", - # Added metrics for displayed database table "Fetch_Latency_ms": "ms", "Insert_Latency_ms": "ms", "Throughput_qps": "queries/s", - "Queries_executed": "units", - "Errors": "units", - # Added ManageEngine metrics with direct SQL access (examples) "Version": "text", "Type_Instance": "text", - "Connections_attempted": "count", "Table_open_cache_hit_ratio": "%", "Buffer_pool_utilization": "%", - "Threads_connected": "count", - "Threads_running": "count", - "Threads_cached": "count", - "Threads_created": "count", - "Update_command": "count", - "Insert_command": "count", - "Insert_select_command": "count", - "Delete_command": "count", - "Create_db_command": "count", - "Drop_db_command": "count", - "Drop_table_command": "count", - "Alter_table_command": "count", - "Binlog_command": "count", - "Load_command": "count", - "Replace_command": "count", - "Replace_select_command": "count", - "Select_command": "count", - "Shutdown_command": "count", - "Group_replication_start_command": "count", - "Group_replication_stop_command": "count", - "Slave_start_command": "count", - "Slave_stop_command": "count", - "Change_master_command": "count", - "Revoke_command": "count", - "Revoke_all_command": "count", - "Rollback_command": "count", - "Rollback_to_savepoint_command": "count", - "Savepoint_command": "count", - "Handler_external_lock": "count", - "Handler_savepoint": "count", - "Handler_savepoint_rollback": "count", - "Binlog_stmt_cache_use": "count", - "Binlog_stmt_cache_disk_use": "count", - "Binlog_size": "bytes", - "Binlog_file_count": "count", - "Relay_log_size": "bytes", - "Relay_log_file_count": "count", - "InnoDB_active_transactions": "count", - "InnoDB_locked_transactions": "count", - "InnoDB_current_transactions": "count", - "InnoDB_history_list_length": "count", - "InnoDB_lock_structs": "count", - "InnoDB_tables_in_use": "count", - "InnoDB_locked_tables": "count", - # Existing nested units + + "Slave_running": "boolean", + "Slave_sql_running": "boolean", + "Slave_io_running": "boolean", + "Seconds_behind_master": "seconds", + "Relay_log_space": "bytes", + "Master_host": "text", + "Master_user": "text", "Database": { "Db_size_MB": "MB", "Index_MB": "MB", "Fetch_Latency_ms": "ms", "Insert_Latency_ms": "ms", - "Throughput_qps": "queries/s", - "Queries_executed": "units", - "Errors": "units" - }, - "Session": { - "Lock_Latency": "ms", - # "CPU": "s", # Commented out as requested - # "Memory": "KB" # Commented out as requested + "Throughput_qps": "queries/s" } } METRICS_MAPPING = { @@ -122,12 +44,6 @@ "name", "Db_size_MB", "Index_MB", "Number_of_Tables", "status", "Open_tables", "Fetch_Latency_ms", "Insert_Latency_ms", "Throughput_qps", "Queries_executed", "Errors" ], - "Session": [ - "name", "User", "db", - # "Memory", # Commented out as requested - # "CPU", # Commented out as requested - "Lock_Latency" - ], "Threads": [ "Threads_running", "Threads_connected", @@ -136,8 +52,7 @@ "Aborted_clients", "Aborted_connects", "Max_used_connections", - "Connections", - "threads_running" # from old name compatibility + "Connections" ], "Handler": [ "Com_select", @@ -217,7 +132,6 @@ "Innodb_row_lock_time_avg", "Innodb_row_lock_time_max", "Innodb_row_lock_waits", - # Added missing old metrics "Innodb_buffer_pool_wait_free", "Innodb_buffer_pool_pages_flushed", "Innodb_buffer_pool_read_ahead_evicted", @@ -273,44 +187,18 @@ def __init__(self, args): self.maindata = { "plugin_version": PLUGIN_VERSION, "heartbeat_required": HEARTBEAT, - "units": METRICS_UNITS, - "tabs": { - "Database and Session": { - "order": 1, - "tablist": ["Database", "Session"] - }, - "Threads": { - "order": 2, - "tablist": METRICS_MAPPING["Threads"] - }, - "Handler": { - "order": 3, - "tablist": METRICS_MAPPING["Handler"] - }, - "Query and Storage": { - "order": 4, - "tablist": METRICS_MAPPING["Query and Storage"] - } - } + } self.connection = None self.cursor = None - def safe_int(self, val): - try: - return int(val) - except Exception: - return 0 - def safe_float(self, val): - try: - return float(val) - except Exception: - return 0.0 + def to_str(self, val): try: f = float(val) return f"{f:.2f}" except Exception: return str(val) + def connect(self): try: self.connection = pymysql.connect( @@ -319,27 +207,27 @@ def connect(self): user=self.username, password=self.password, cursorclass=pymysql.cursors.DictCursor, - autocommit=True, - connect_timeout=10 ) self.cursor = self.connection.cursor() return True except Exception as e: - self.maindata["availability"] = 0 + self.maindata["status"] = 0 self.maindata["msg"] = f"Connection error: {repr(e)}" return False + def close(self): try: if self.cursor: self.cursor.close() if self.connection: self.connection.close() - except Exception: - pass + except Exception as e: + self.maindata["status"] = 0 + self.maindata["msg"] = f"Error closing the connection: {repr(e)}" + def collect_database(self): dbs = [] try: - # Collect base db info self.cursor.execute(""" SELECT table_schema AS name, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS Db_size_MB, @@ -349,10 +237,8 @@ def collect_database(self): FROM information_schema.tables WHERE table_schema NOT IN ('mysql', 'performance_schema', 'sys', 'information_schema') GROUP BY table_schema - LIMIT 25 """) db_size_info = {row["name"]: row for row in self.cursor.fetchall()} - # Fetch Latency (average SELECT latency in ms) self.cursor.execute(""" SELECT ROUND(IFNULL(AVG_TIMER_WAIT/1000000000, 0)*1000, 2) AS avg_fetch_latency_ms FROM performance_schema.events_statements_summary_by_digest @@ -363,7 +249,6 @@ def collect_database(self): r = self.cursor.fetchone() if r and "avg_fetch_latency_ms" in r: fetch_latency = float(r["avg_fetch_latency_ms"]) - # Insert Latency (average INSERT latency in ms) self.cursor.execute(""" SELECT ROUND(IFNULL(AVG_TIMER_WAIT/1000000000, 0)*1000, 2) AS avg_insert_latency_ms FROM performance_schema.events_statements_summary_by_digest @@ -374,21 +259,17 @@ def collect_database(self): r = self.cursor.fetchone() if r and "avg_insert_latency_ms" in r: insert_latency = float(r["avg_insert_latency_ms"]) - # Throughput (queries per second) calculated over uptime self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Questions'") questions_row = self.cursor.fetchone() questions = int(questions_row['Value']) if questions_row else 0 self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Uptime'") uptime_row = self.cursor.fetchone() - uptime = int(uptime_row['Value']) if uptime_row else 1 # avoid zero division + uptime = int(uptime_row['Value']) if uptime_row else 1 throughput_qps = round(questions / uptime, 2) if uptime > 0 else 0.0 - # Queries executed - total queries count queries_executed = questions - # Errors - sum of errors from SHOW GLOBAL STATUS variables related self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Errors'") errors_row = self.cursor.fetchone() errors = int(errors_row['Value']) if errors_row else 0 - # Additional ManageEngine metrics examples fetched by direct SQL queries self.cursor.execute("SELECT VERSION() AS version") version_row = self.cursor.fetchone() version = version_row.get("version", "") if version_row else "" @@ -408,7 +289,6 @@ def collect_database(self): total = table_open_cache_hits + table_open_cache_misses if total > 0: table_open_cache_hit_ratio = round((table_open_cache_hits / total) * 100, 2) - # Buffer Pool Utilization: sum dirty+clean pages / total pages * 100 self.cursor.execute("SHOW GLOBAL STATUS LIKE 'Innodb_buffer_pool_pages_data'") pages_data_row = self.cursor.fetchone() pages_data = int(pages_data_row['Value']) if pages_data_row else 0 @@ -418,81 +298,63 @@ def collect_database(self): buffer_pool_utilization = 0.0 if pages_total > 0: buffer_pool_utilization = round((pages_data / pages_total) * 100, 2) - # Add fetched metrics to each DB record as demo; generally these are global but added for visibility for dbname, dbdata in db_size_info.items(): rec = { "name": dbname, "Db_size_MB": self.to_str(dbdata.get("Db_size_MB", 0)), "Index_MB": self.to_str(dbdata.get("Index_MB", 0)), "Number_of_Tables": self.to_str(dbdata.get("Number_of_Tables", 0)), - "status": 1, - "Open_tables": "0", "Fetch_Latency_ms": self.to_str(fetch_latency), "Insert_Latency_ms": self.to_str(insert_latency), "Throughput_qps": self.to_str(throughput_qps), "Queries_executed": str(queries_executed), "Errors": str(errors), - "Version": version, - "Connections_attempted": str(connections_attempted), - "Threads_connected": str(threads_connected), - "Table_open_cache_hit_ratio": str(table_open_cache_hit_ratio), - "Buffer_pool_utilization": str(buffer_pool_utilization) + + # "Version": version, + # "Connections_attempted": str(connections_attempted), + # "Threads_connected": str(threads_connected), + # "Table_open_cache_hit_ratio": str(table_open_cache_hit_ratio), + # "Buffer_pool_utilization": str(buffer_pool_utilization) } dbs.append(rec) except Exception as e: - self.maindata["availability"] = 0 + self.maindata["status"] = 0 self.maindata["msg"] = f"Database collection error: {e}" return dbs - def collect_session(self): - sess = [] + + def collect_sessions(self): try: self.cursor.execute(""" - SELECT THREAD_ID, PROCESSLIST_ID - FROM performance_schema.threads - WHERE TYPE = 'FOREGROUND' + SELECT + COUNT(*) AS total_sessions, + SUM(CASE WHEN COMMAND != 'Sleep' THEN 1 ELSE 0 END) AS active_sessions, + SUM(CASE WHEN COMMAND = 'Sleep' THEN 1 ELSE 0 END) AS idle_sessions, + ( + SELECT VARIABLE_VALUE + FROM performance_schema.global_status + WHERE VARIABLE_NAME = 'Aborted_clients' + ) AS killed_sessions + FROM information_schema.PROCESSLIST """) - thread_pid_map = {r["THREAD_ID"]: r["PROCESSLIST_ID"] for r in self.cursor.fetchall()} - self.cursor.execute("SHOW FULL PROCESSLIST") - sessions = self.cursor.fetchall() - pid_idx_map = {} - count = 0 - for s in sessions: - if count >= 25: - break - pid = s.get("Id", 0) - pid_str = f"PID_{pid}" - rec = { - "name": pid_str, - "User": str(s.get("User") or ""), - "db": str(s.get("db") or ""), - # "Memory": "0.00", # commented as requested - # "CPU": "0.00", # commented as requested - "Lock_Latency": self.to_str(s.get("Time", 0)) - } - sess.append(rec) - pid_idx_map[pid] = count - count += 1 - except Exception as e: - self.maindata["availability"] = 0 - self.maindata["msg"] = f"Session collection error: {e}" - return sess - def collect_performance(self): - perf = {} - try: - self.cursor.execute(""" - SELECT digest_text, count_star, sum_timer_wait, avg_timer_wait - FROM performance_schema.events_statements_summary_by_digest - ORDER BY count_star DESC - LIMIT 10 - """) - rows = self.cursor.fetchall() - for i, row in enumerate(rows, 1): - perf[f"PS_{i}_count"] = self.safe_int(row.get("count_star", 0)) - perf[f"PS_{i}_sum_timer"] = self.safe_float(row.get("sum_timer_wait", 0.0)) - perf[f"PS_{i}_avg_timer"] = self.safe_float(row.get("avg_timer_wait", 0.0)) + row = self.cursor.fetchone() + # Safely cast and supply session stats; handle None gracefully + sessions = { + "total_sessions": int(row.get('total_sessions', 0)), + "active_sessions": int(row.get('active_sessions', 0)), + "idle_sessions": int(row.get('idle_sessions', 0)), + "killed_sessions": int(row.get('killed_sessions', 0)) if row.get('killed_sessions') is not None else 0 + } + return sessions except Exception as e: - perf["perf_error"] = f"Performance collection error: {e}" - return perf + return { + "total_sessions": 0, + "active_sessions": 0, + "idle_sessions": 0, + "killed_sessions": 0, + "sessions_error": str(e) + } + + def collect_metrics(self): if not self.connect(): return self.maindata @@ -501,15 +363,14 @@ def collect_metrics(self): status = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} self.cursor.execute("SHOW GLOBAL VARIABLES") variables = {row["Variable_name"]: row["Value"] for row in self.cursor.fetchall()} - # Add Uptime explicitly self.maindata["Uptime"] = self.to_str(status.get("Uptime", 0)) - # Add threads_running old metric name as well - self.maindata["threads_running"] = self.to_str(status.get("Threads_running", 0)) for tab_name in ["Threads", "Handler", "Query and Storage"]: for metric in METRICS_MAPPING[tab_name]: val = status.get(metric) or variables.get(metric) if val is not None: - self.maindata[metric] = self.to_str(val) + display_name = metric.replace("Com_", "Command_") if metric.startswith("Com_") else metric + self.maindata[display_name] = self.to_str(val) + # self.maindata[metric] = self.to_str(val) max_used = status.get("Max_used_connections") self.maindata["Max_used_connections"] = self.to_str(int(max_used)) if max_used else "0.00" max_conn = int(variables.get("max_connections", "0")) @@ -519,27 +380,154 @@ def collect_metrics(self): self.maindata["Connection_usage"] = self.to_str(round(threads / max_conn * 100, 2)) if max_conn else "0.00" self.maindata["Open_files_usage"] = self.to_str(round(open_files / open_files_limit * 100, 2)) if open_files_limit else "0.00" self.maindata["Database"] = self.collect_database() - self.maindata["Session"] = self.collect_session() + # Fetch the session metrics (already in your code) + session_stats = self.collect_sessions() + # Add individually to maindata for summary display + self.maindata["total_sessions"] = session_stats["total_sessions"] + self.maindata["active_sessions"] = session_stats["active_sessions"] + self.maindata["idle_sessions"] = session_stats["idle_sessions"] + self.maindata["killed_sessions"] = session_stats["killed_sessions"] self.maindata["s247config"] = { - "childdiscovery": ["Session", "Database"] + "childdiscovery": ["Database"] + } + self.maindata["units"] = METRICS_UNITS + self.maindata["tabs"] = { + "Database": { + "order": 1, + "tablist": ["Database"] + }, + "Threads": { + "order": 2, + "tablist": [ + "Threads_running", + "Threads_connected", + "Threads_cached", + "Threads_created" + ], + }, + "Handler": { + "order": 3, + "tablist": [ + "Handler_read_first", + "Handler_read_key", + "Handler_read_rnd", + "Handler_read_rnd_next", + "Handler_write", + "Handler_update", + "Handler_delete", + "Handler_commit", + "Handler_rollback" + ], + }, + "Query and Storage": { + "order": 4, + "tablist": [ + "Queries", + "Questions", + "Slow_queries", + "Opened_tables", + "Opened_files", + "Binlog_cache_use", + "Binlog_cache_disk_use", + "Bytes_received", + "Bytes_sent", + "Table_locks_waited", + "Table_locks_immediate", + "Created_tmp_files", + "Created_tmp_tables", + "Created_tmp_disk_tables", + "Commands_per_second", + "Avg_query_time", + "Connection_usage", + "Open_files_usage", + "Innodb_buffer_pool_pages_data", + "Innodb_buffer_pool_pages_dirty", + "Innodb_buffer_pool_pages_free", + "Innodb_rows_deleted", + "Innodb_rows_inserted", + "Innodb_rows_updated", + "Slave_running", + "Slave_sql_running", + "Slave_io_running", + "Seconds_behind_master", + "Relay_log_space", + "Master_host", + "Master_user", + "Master_retry_count", + "Skip_counter", + "Open_files_limit", + "Open_files_used", + "Key_reads", + "Key_writes", + "Key_blocks_used", + "Key_blocks_unused", + "Key_blocks_not_flushed", + "Innodb_buffer_pool_pages_total", + "Innodb_pages_read", + "Innodb_pages_written", + "Innodb_log_writes", + "Innodb_log_waits", + "Innodb_buffer_pool_reads", + "Innodb_buffer_pool_write_requests", + "Innodb_data_reads", + "Innodb_data_writes", + "Innodb_pages_created", + "Innodb_row_lock_time_avg", + "Innodb_row_lock_time_max", + "Innodb_row_lock_waits", + "Innodb_buffer_pool_wait_free", + "Innodb_buffer_pool_pages_flushed", + "Innodb_buffer_pool_read_ahead_evicted", + "Innodb_buffer_pool_read_ahead", + "Innodb_buffer_pool_read_ahead_rnd", + "Innodb_buffer_pool_read_requests", + "Innodb_data_fsyncs", + "Innodb_data_pending_fsyncs", + "Innodb_data_pending_reads", + "Innodb_data_pending_writes", + "Innodb_log_write_requests", + "Innodb_os_log_fsyncs", + "Innodb_os_log_pending_fsyncs", + "Innodb_os_log_pending_writes", + "Innodb_os_log_written", + "Innodb_rows_read", + "Qcache_hits", + "Qcache_free_memory", + "Qcache_not_cached", + "Qcache_queries_in_cache", + "Qcache_free_blocks", + "Qcache_inserts", + "Qcache_lowmem_prunes", + "Qcache_total_blocks", + "Connection_errors_max_connections", + "max_connections", + "Select_full_join", + "Select_full_range_join", + "Select_range", + "Select_range_check", + "Select_scan", + "Max_execution_time_exceeded", + "Open_files", + "Table_open_cache_hits", + "Table_open_cache_misses", + "Table_open_cache_overflows", + "Prepared_stmt_count", + "Sort_merge_passes", + "Sort_range", + "Sort_rows", + "Sort_scan", + "Key_read_requests", + "Key_write_requests" + ] + } } - perf = self.collect_performance() - self.maindata.update(perf) - self.close() - return self.maindata except Exception as e: - self.maindata["availability"] = 0 + self.maindata["status"] = 0 self.maindata["msg"] = f"Metric collection error: {e}\n{traceback.format_exc()}" + finally: self.close() - return self.maindata -def normalize(obj): - if isinstance(obj, dict): - return {k: normalize(v) for k, v in obj.items()} - if isinstance(obj, list): - return [normalize(i) for i in obj] - if isinstance(obj, Decimal): - return float(obj) - return obj + return self.maindata + if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() @@ -547,8 +535,12 @@ def normalize(obj): parser.add_argument("--port", default=MYSQL_DEFAULTS["port"]) parser.add_argument("--username", default=MYSQL_DEFAULTS["username"]) parser.add_argument("--password", default=MYSQL_DEFAULTS["password"]) + + parser.add_argument('--logs_enabled', help='enable log collection for this plugin application',default="False") + parser.add_argument('--log_type_name', help='Display name of the log type', nargs='?', default=None) + parser.add_argument('--log_file_path', help='list of comma separated log file paths', nargs='?', default=None) + args = parser.parse_args() monitor = MySQLMonitor(args) result = monitor.collect_metrics() - result = normalize(result) print(json.dumps(result, indent=2)) From bf923007905ebf042bb2a84958b526119ca6195b Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Fri, 5 Sep 2025 18:33:48 +0530 Subject: [PATCH 04/17] Updated Oracle Plugin code --- oracle/oracle.py | 230 +++++++++++++++++++++++------------------------ 1 file changed, 113 insertions(+), 117 deletions(-) diff --git a/oracle/oracle.py b/oracle/oracle.py index 0be025ab..632022cf 100644 --- a/oracle/oracle.py +++ b/oracle/oracle.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python3.8 import json import os import warnings @@ -6,7 +6,6 @@ PLUGIN_VERSION=1 HEARTBEAT=True METRICS_UNITS={ - "Buffer Cache Hit Ratio":"%", "Cursor Cache Hit Ratio":"%", "Library Cache Hit Ratio":"%", @@ -32,31 +31,45 @@ "FRA Space Used":"mb", "FRA Space Reclaimable":"mb", "Response Time":"ms", - "Tablespace_Details":{ + "Tablespace_List":{ "Tablespace_Size":"mb", "Used_Percent":"%", "Used_Space":"mb" }, - "Tablespace_Datafile_Details":{ + "Tablespace_Datafile":{ "Data_File_Size":"mb", "Max_Data_File_Size":"mb", "Usable_Data_File_Size":"mb" }, - "PDB_Details":{ + "PDB":{ "PDB_Size":"mb" }, - "ASM_Details":{ + "ASM":{ "total_gb": "GB", "free_gb": "GB", "pct_free": "%" - } + }, + "CPU Usage":"seconds", + "Enqueue Deadlocks":"count", + "Exchange Deadlocks":"count", + "Logical Reads":"count", + "Transactions Per Second":"txn/sec", + "Queries Per Second":"query/sec", + "DB Time":"seconds", + "Rollback Segments": "count", + "Rollback Segment Initial Extent":"bytes", + "Rollback Segment Next Extent":"bytes", + "Rollback Segment Max Extents":"count", + "Slow Query Latency 95 Percentile":"ms", + "Full Table Scans Short":"count", + "Full Table Scans Long":"count", + "Full Table Scans Rowid":"count", + "Full Table Scans IM":"count", + "Failed Backups":"count", + "Alert Log Recent Errors":"count" } - - class oracle: - def __init__(self,args): - self.maindata={} self.maindata['plugin_version'] = PLUGIN_VERSION self.maindata['heartbeat_required']=HEARTBEAT @@ -68,7 +81,6 @@ def __init__(self,args): self.port=args.port self.tls=args.tls.lower() self.wallet_location=args.wallet_location - def connect(self, dsn): try: @@ -96,7 +108,7 @@ def execute_query_row_col(self, query, col_change=False): try: self.c.execute(query) col_names = [row[0] for row in self.c.description] - tot_cols=len(col_names) + tot_cols=len(col_names) if col_change: for row in self.c: for i in range(tot_cols): @@ -112,8 +124,6 @@ def execute_query_row_col(self, query, col_change=False): queried_data['msg']=str(e) return queried_data - - def execute_query_bulk(self, query, query_name): queried_data={} try: @@ -129,9 +139,8 @@ def execute_query_bulk(self, query, query_name): asm_list = [] for row in self.c: asm_name, asm_total_gb, asm_free_gb, asm_pct_free, asm_limit, asm_threshold = row - asm_list.append({"name": asm_name, "ASM_TOTAL_GB" :asm_total_gb, "ASM_FREE_GB": asm_free_gb, "ASM_PCT_FREE": asm_pct_free ,"ASM_LIMIT": asm_limit, "ASM_THRESHOLD": asm_threshold}) - queried_data['ASM_Details']=asm_list + queried_data['ASM']=asm_list else: for row in self.c: value,metric=row @@ -141,7 +150,6 @@ def execute_query_bulk(self, query, query_name): queried_data['msg']=str(e) return queried_data - def execute_query(self,metric_query_name): queried_data={} try: @@ -152,8 +160,7 @@ def execute_query(self,metric_query_name): queried_data["status"]=0 queried_data['msg']=str(e) return queried_data - - + def execute_waits_query(self,metric_query_name): queried_data={} wait_units={} @@ -161,24 +168,21 @@ def execute_waits_query(self,metric_query_name): self.c.execute(self.metric_queries[metric_query_name]) for row in self.c: name,time_waited,wait_count=row - queried_data[str.title(name)+" Time Waited"]=time_waited + queried_data[str.title(name)+" Time Waited (microsec)"]=time_waited queried_data[str.title(name)+" Wait Count"]=wait_count - wait_units[str.title(name)+" Time Waited"]='sec' + wait_units[str.title(name)+" Time Waited (microsec)"]='microseconds' except Exception as e: queried_data['status']=0 queried_data['msg']=str(e) METRICS_UNITS.update(wait_units) return queried_data - def execute_tablespace_metrics(self): - db_block_size=8192 self.c.execute("select value from v$parameter where name = 'db_block_size'") for row in self.c: db_block_size=row[0] break - queried_data={} try: self.c.execute(self.metric_queries["Tablespace Queries"]["Tablespace Metrics Query"]) @@ -188,10 +192,8 @@ def execute_tablespace_metrics(self): tbs_dict["name"]=row[0] if row[2]:tbs_dict['Used_Space']=int(row[2])*int(db_block_size)/1024/1024 else:tbs_dict['Used_Space']=0 - if row[3]:tbs_dict['Tablespace_Size']=int(row[3])*int(db_block_size)/1024/1024 else:tbs_dict['Tablespace_Size']=0 - if row[4]:tbs_dict['Used_Percent']=row[4] else:tbs_dict['Used_Percent']=0 if row[7]=="OFFLINE": @@ -200,16 +202,13 @@ def execute_tablespace_metrics(self): else: tbs_dict['TB_Status']=1 tbs_dict['status']=1 - tbs_list.append(tbs_dict) - queried_data['Tablespace_Details']=tbs_list - + queried_data['Tablespace_List']=tbs_list except Exception as e: queried_data["status"]=0 queried_data['msg']=str(e) - - return queried_data - + return queried_data + def execute_tablespace_datafile(self): queried_data={} try: @@ -233,12 +232,11 @@ def execute_tablespace_datafile(self): tb_dict["Usable_Data_File_Size"]=row[8] tb_dict["Usable_Data_File_Blocks"]=row[9] tbs_list.append(tb_dict) - queried_data["Tablespace_Datafile_Details"]=tbs_list - + queried_data["Tablespace_Datafile"]=tbs_list except Exception as e: queried_data["status"]=0 queried_data['msg']=str(e) - return queried_data + return queried_data def tablespace_complete(self): queried_data={} @@ -246,20 +244,17 @@ def tablespace_complete(self): query_output_data=self.execute_tablespace_metrics() queried_data.update(query_output_data) if 'status' in queried_data and queried_data['status']==0: - return queried_data + return queried_data query_output_data=self.execute_tablespace_datafile() queried_data.update(query_output_data) if 'status' in queried_data and queried_data['status']==0: - return queried_data - + return queried_data except Exception as e: queried_data["status"]=0 queried_data['msg']=str(e) - return queried_data - + return queried_data - def execute_pdb(self, metric_query_name): queried_data={} try: @@ -269,59 +264,69 @@ def execute_pdb(self, metric_query_name): pdb_dict={} pdb_dict['name']=row[0] pdb_dict['PDB_ID']=row[1] - pdb_dict['PDB_Size']=row[-2] - pdb_dict['Block_Size']=row[-1] - + pdb_dict['PDB_Size']=row[2] + pdb_dict['Block_Size']=row[3] pdb_list.append(pdb_dict) - queried_data['PDB_Details']=pdb_list - + queried_data['PDB']=pdb_list except Exception as e: queried_data['status']=0 queried_data['msg']=str(e) return queried_data - - def metriccollector(self): self.metric_queries={ "Bulk Queries":{ - "system_query":"SELECT VALUE, METRIC_NAME FROM GV$SYSMETRIC WHERE METRIC_NAME IN ( 'Soft Parse Ratio', 'Total Parse Count Per Sec', 'Total Parse Count Per Txn', 'Hard Parse Count Per Sec', 'Hard Parse Count Per Txn', 'Parse Failure Count Per Sec', 'Parse Failure Count Per Txn', 'Temp Space Used', 'Session Count', 'Session Limit %', 'Database Wait Time Ratio', 'Memory Sorts Ratio', 'Disk Sort Per Sec', 'Rows Per Sort', 'Total Sorts Per User Call', 'User Rollbacks Per Sec', 'SQL Service Response Time', 'Long Table Scans Per Sec', 'Average Active Sessions', 'Logons Per Sec', 'Global Cache Blocks Los', 'Global Cache Blocks Corrupted', 'GC CR Block Received Per Second', 'Enqueue Timeouts Per Sec', 'Physical Writes Per Sec', 'Physical Reads Per Sec', 'Shared Pool Free %', 'Library Cache Hit Ratio', 'Cursor Cache Hit Ratio', 'Buffer Cache Hit Ratio' )", "pga_query":"SELECT VALUE, NAME FROM gv$pgastat where NAME IN ('total PGA allocated', 'total freeable PGA memory', 'maximum PGA allocated','total PGA inuse')", "sga_query":"""SELECT sga.value, CONCAT('SGA ',sga.name) AS name FROM GV$SGA sga INNER JOIN GV$INSTANCE inst ON sga.inst_id = inst.inst_id""", "asm_query": "SELECT name AS asm_name, ROUND(total_mb / 1024, 2) AS asm_total_gb, ROUND(free_mb / 1024, 2) AS asm_free_gb, ROUND((free_mb / total_mb) * 100, 2) AS asm_pct_free, USABLE_FILE_MB AS asm_limit, REQUIRED_MIRROR_FREE_MB AS asm_threshold FROM v$asm_diskgroup" - }, - + }, "Single Queries":{ - - "Rman Failed Backup Count":"""SELECT COUNT(*) as "Rman Failed Backup Count" FROM v$rman_status WHERE operation = 'BACKUP' AND status = 'FAILED' AND END_TIME >= sysdate-(5/(24*60))""", - "Dict Cache Hit Ratio":"""select (1-(sum(getmisses)/sum(gets)))*100 as " DICT CACHE HIT RATIO" from gv$rowcache""", - "Long Running Queries":"""SELECT sum(num) AS total FROM (( SELECT i.inst_id, 1 AS num FROM gv$session s, gv$instance i WHERE i.inst_id=s.inst_id AND s.status='ACTIVE' AND s.type <>'BACKGROUND' AND s.last_call_et > 60 GROUP BY i.inst_id ) UNION ( SELECT i.inst_id, 0 AS num FROM gv$session s, gv$instance i WHERE i.inst_id=s.inst_id)) GROUP BY inst_id""", - "Blocking Locks":"""SELECT count(*) FROM gv$session WHERE blocking_session IS NOT NULL""", - "SGA Hit Ratio":"""SELECT (1 - (phy.value - lob.value - dir.value)/ses.value) as ratio FROM GV$SYSSTAT ses, GV$SYSSTAT lob, GV$SYSSTAT dir, GV$SYSSTAT phy, GV$INSTANCE inst WHERE ses.name='session logical reads' AND dir.name='physical reads direct' AND lob.name='physical reads direct (lob)' AND phy.name='physical reads' AND ses.inst_id=inst.inst_id AND lob.inst_id=inst.inst_id AND dir.inst_id=inst.inst_id AND phy.inst_id=inst.inst_id""", - "SGA Log Alloc Retries":"""SELECT (rbar.value/re.value) as ratio FROM GV$SYSSTAT rbar, GV$SYSSTAT re, GV$INSTANCE inst WHERE rbar.name like 'redo buffer allocation retries' AND re.name like 'redo entries' AND re.inst_id=inst.inst_id AND rbar.inst_id=inst.inst_id""", - "SGA Shared Pool Dict Cache Ratio":"""SELECT (SUM(rcache.getmisses)/SUM(rcache.gets)) as ratio FROM GV$rowcache rcache, GV$INSTANCE inst WHERE inst.inst_id=rcache.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Hit Ratio":"""SELECT libcache.gethitratio as ratio FROM GV$librarycache libcache, GV$INSTANCE inst WHERE namespace='SQL AREA' AND inst.inst_id=libcache.inst_id""", - "SGA Shared Pool Lib Cache Reload Ratio":"""SELECT (sum(libcache.reloads)/sum(libcache.pins)) AS ratio FROM GV$librarycache libcache, GV$INSTANCE inst WHERE inst.inst_id=libcache.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Sharable Statement":"""SELECT SUM(sqlarea.sharable_mem) AS sum FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE sqlarea.executions > 5 AND inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Shareable User":"""SELECT SUM(250 * sqlarea.users_opening) AS sum FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", - "Total Memory":"""SELECT SUM(value) AS sum FROM GV$sesstat, GV$statname, GV$INSTANCE inst WHERE name = 'session uga memory max' AND GV$sesstat.statistic#=GV$statname.statistic# AND GV$sesstat.inst_id=inst.inst_id AND GV$statname.inst_id=inst.inst_id GROUP BY inst.inst_id""", + "Rman Failed Backup Count":"""SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'""", + "Dict Cache Hit Ratio":"""SELECT (1 - (SUM(getmisses)/SUM(gets))) * 100 FROM gv$rowcache""", + "Long Running Queries":"""SELECT COUNT(*) FROM v$session WHERE status='ACTIVE' AND type<>'BACKGROUND' AND last_call_et > 60""", + "Blocking Locks":"""SELECT COUNT(*) FROM gv$session WHERE blocking_session IS NOT NULL""", + "SGA Hit Ratio":"""SELECT (1 - (phy.value - lob.value - dir.value)/ses.value) FROM GV$SYSSTAT ses, GV$SYSSTAT lob, GV$SYSSTAT dir, GV$SYSSTAT phy, GV$INSTANCE inst WHERE ses.name='session logical reads' AND dir.name='physical reads direct' AND lob.name='physical reads direct (lob)' AND phy.name='physical reads' AND ses.inst_id=inst.inst_id AND lob.inst_id=inst.inst_id AND dir.inst_id=inst.inst_id AND phy.inst_id=inst.inst_id""", + "SGA Log Alloc Retries":"""SELECT (rbar.value/re.value) FROM GV$SYSSTAT rbar, GV$SYSSTAT re, GV$INSTANCE inst WHERE rbar.name LIKE 'redo buffer allocation retries' AND re.name LIKE 'redo entries' AND re.inst_id=inst.inst_id AND rbar.inst_id=inst.inst_id""", + "SGA Shared Pool Dict Cache Ratio":"""SELECT (SUM(rcache.getmisses)/SUM(rcache.gets)) FROM GV$rowcache rcache, GV$INSTANCE inst WHERE inst.inst_id=rcache.inst_id GROUP BY inst.inst_id""", + "SGA Shared Pool Lib Cache Hit Ratio":"""SELECT libcache.gethitratio FROM GV$librarycache libcache, GV$INSTANCE inst WHERE namespace='SQL AREA' AND inst.inst_id=libcache.inst_id""", + "SGA Shared Pool Lib Cache Reload Ratio":"""SELECT (SUM(libcache.reloads)/SUM(libcache.pins)) FROM GV$librarycache libcache, GV$INSTANCE inst WHERE inst.inst_id=libcache.inst_id GROUP BY inst.inst_id""", + "SGA Shared Pool Lib Cache Sharable Statement":"""SELECT SUM(sqlarea.sharable_mem) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE sqlarea.executions > 5 AND inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", + "SGA Shared Pool Lib Cache Shareable User":"""SELECT SUM(250 * sqlarea.users_opening) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", + "Total Memory":"""SELECT SUM(value) FROM GV$sesstat, GV$statname, GV$INSTANCE inst WHERE name = 'session uga memory max' AND GV$sesstat.statistic#=GV$statname.statistic# AND GV$sesstat.inst_id=inst.inst_id AND GV$statname.inst_id=inst.inst_id GROUP BY inst.inst_id""", "Oracle Database Version":"SELECT version FROM PRODUCT_COMPONENT_VERSION WHERE product LIKE 'Oracle Database%'", - "Response Time":"""SELECT ROUND (VALUE * 10, 2) "Response Time (msecs)" FROM GV$SYSMETRIC WHERE 1 = 1 AND METRIC_NAME = 'SQL Service Response Time' ORDER BY INST_ID""", - "Number of Session Users":"""SELECT COUNT(DISTINCT username) AS num_session_users FROM v$session WHERE username IS NOT NULL""", - "Database Block Size":"""select value from v$parameter where name = 'db_block_size'""", - "Invalid Index Count":"""SELECT COUNT(*) AS invalid_index_count FROM dba_indexes WHERE status = 'INVALID'""" - }, + "Response Time":"""SELECT ROUND(VALUE * 10, 2) FROM GV$SYSMETRIC WHERE METRIC_NAME = 'SQL Service Response Time' ORDER BY INST_ID""", + "Number of Session Users":"""SELECT COUNT(DISTINCT username) FROM v$session WHERE username IS NOT NULL""", + "Database Block Size":"""SELECT value FROM v$parameter WHERE name = 'db_block_size'""", + "Invalid Index Count":"""SELECT COUNT(*) FROM dba_indexes WHERE status = 'INVALID'""", + "CPU Usage":"SELECT value FROM v$sysstat WHERE name='CPU used by this session'", + "Enqueue Deadlocks":"SELECT value FROM v$sysstat WHERE name='enqueue deadlocks'", + "Exchange Deadlocks":"SELECT value FROM v$sysstat WHERE name='exchange deadlocks'", + "Logical Reads":"SELECT value FROM v$sysstat WHERE name='session logical reads'", + "Queries Per Second":"""SELECT (value / 60) FROM v$sysstat WHERE name = 'execute count'""", + "Transactions Per Second":"""SELECT ((SELECT value FROM v$sysstat WHERE name = 'user commits') + (SELECT value FROM v$sysstat WHERE name = 'user rollbacks')) / 60 FROM dual""", + "DB Time":"""SELECT SUM(value)/100 FROM v$sys_time_model WHERE stat_name = 'DB time'""", + + "Slow Query Latency 95 Percentile":"""SELECT ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY elapsed_time/executions)) FROM v$sql WHERE executions > 0""", + "Full Table Scans Short":"""SELECT value FROM v$sysstat WHERE name = 'table scans (short tables)'""", + "Full Table Scans Long":"""SELECT value FROM v$sysstat WHERE name = 'table scans (long tables)'""", + "Full Table Scans Rowid":"""SELECT value FROM v$sysstat WHERE name = 'table scans (rowid ranges)'""", + "Full Table Scans IM":"""SELECT value FROM v$sysstat WHERE name = 'table scans (IM)'""", + "Failed Backups":"""SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'""", + "Alert Log Recent Errors":"""SELECT COUNT(*) FROM V$DIAG_ALERT_EXT WHERE ORIGINATING_TIMESTAMP > SYSDATE - 1""", + + "Rollback Segments":"""SELECT COUNT(*) FROM dba_rollback_segs""", + "Rollback Segment Details":"""SELECT segment_name, tablespace_name, status, initial_extent, next_extent, max_extents FROM dba_rollback_segs""" + }, "Tablespace Queries":{ - "Tablespace Metrics Query":""" SELECT b.TABLESPACE_NAME as "dba_tablespace", d.* , b.CONTENTS, b.LOGGING, b.STATUS FROM dba_tablespace_usage_metrics d FULL JOIN dba_tablespaces b ON d.TABLESPACE_NAME = b.TABLESPACE_NAME""", - "Tablespace Datafile Query":""" SELECT TABLESPACE_NAME, FILE_NAME, (BYTES/1024/1024) , BLOCKS, AUTOEXTENSIBLE, (MAXBYTES/1024/1024), (MAXBLOCKS/1024/1024), INCREMENT_BY, (USER_BYTES/1024/1024), USER_BLOCKS FROM DBA_DATA_FILES""" + "Tablespace Metrics Query":"""SELECT b.TABLESPACE_NAME as "dba_tablespace", d.* , b.CONTENTS, b.LOGGING, b.STATUS FROM dba_tablespace_usage_metrics d FULL JOIN dba_tablespaces b ON d.TABLESPACE_NAME = b.TABLESPACE_NAME""", + "Tablespace Datafile Query":"""SELECT TABLESPACE_NAME, FILE_NAME, (BYTES/1024/1024), BLOCKS, AUTOEXTENSIBLE, (MAXBYTES/1024/1024), (MAXBLOCKS/1024/1024), INCREMENT_BY, (USER_BYTES/1024/1024), USER_BLOCKS FROM DBA_DATA_FILES""" }, "FRA Query":"""SELECT name AS "FRA File Dest", space_limit / (1024 * 1024) AS "FRA Space Limit", space_used / (1024 * 1024) AS "FRA Space Used", space_reclaimable / (1024 * 1024) AS "FRA Space Reclaimable", number_of_files AS "FRA Number of Files" FROM V$RECOVERY_FILE_DEST""", - "Waits Query":"""select n.name , round(m.time_waited/100,3) time_waited, m.wait_count from v$eventmetric m, v$event_name n where m.event_id=n.event_id and n.name in ( 'free buffer waits' , 'buffer busy waits', 'latch free', 'library cache pin', 'library cache load lock', 'log buffer space', 'library object reloads count', 'enqueue waits', 'db file parallel read', 'db file parallel write', 'control file sequential read', 'control file parallel write', 'write complete waits', 'log file sync', 'sort segment request', 'direct path read', 'direct path write')""", - "PDB Query":"""SELECT a.PDB_NAME, a.PDB_ID, a.STATUS, b.OPEN_MODE, b.RESTRICTED, b.OPEN_TIME, b.total_size/1024/1024, b.BLOCK_SIZE FROM DBA_PDBS a join V$PDBS b on a.PDB_NAME=b.NAME""", - "DB Query":"""select cdb as "CDB", open_mode as "Open Mode", TO_CHAR(created, 'YYYY-MM-DD HH24:MI:SS') AS "Created Date", log_mode as "Log Mode", switchover_status as "Switchover Status", protection_mode as "Protection Mode", current_scn as "Current SCN" from v$database""" - } - + "Waits Query":"""SELECT event, time_waited_micro, total_waits FROM v$system_event WHERE wait_class != 'Idle' AND ROWNUM <= 20""", + "PDB Query":"""SELECT p.name, p.con_id, NVL(SUM(f.bytes),0)/1024/1024 AS pdb_size_mb, p.block_size FROM v$pdbs p LEFT JOIN cdb_data_files f ON f.con_id = p.con_id GROUP BY p.name, p.con_id, p.block_size""", + "DB Query":"SELECT 1 FROM dual" + } if self.tls=="True": dsn=f"""(DESCRIPTION= (ADDRESS=(PROTOCOL=tcps)(HOST={self.hostname})(PORT={self.port})) @@ -330,74 +335,64 @@ def metriccollector(self): )""" else: dsn=f"{self.hostname}:{self.port}/{self.sid}" - - connection_status=self.connect(dsn) if not connection_status[0]: self.maindata['status']=0 self.maindata['msg']=connection_status[1] self.close_connection() return self.maindata - for query_name ,bulk_query in self.metric_queries['Bulk Queries'].items(): query_output_data=self.execute_query_bulk(bulk_query, query_name=query_name) self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - for query_name in self.metric_queries['Single Queries']: query_output_data=self.execute_query(query_name) self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - query_output_data=self.tablespace_complete() self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - query_output_data=self.execute_waits_query("Waits Query") self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - query_output_data=self.execute_pdb("PDB Query") self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - query_output_data=self.execute_query_row_col(self.metric_queries["DB Query"]) self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - query_output_data=self.execute_query_row_col(self.metric_queries["FRA Query"]) self.maindata.update(query_output_data) if 'status' in self.maindata and self.maindata['status']==0: self.close_connection() return self.maindata - self.maindata['tabs']={ "Tablespace and PDB":{ "order":1, "tablist":[ - "Tablespace_Details", - "Tablespace_Datafile_Details", - "PDB_Details" - ]}, - + "Tablespace_List", + "Tablespace_Datafile", + "PDB" + ] + }, "Buffer Cache and Memory":{ "order":2, "tablist":[ "Buffer Cache Hit Ratio", "Database Block Size", - "Shared Pool Free %" + "Shared Pool Free %", "Total Freeable PGA Memory", "Maximum PGA Allocated", "Total PGA Allocated", @@ -412,7 +407,10 @@ def metriccollector(self): "SGA Shared Pool Lib Cache Hit Ratio", "SGA Shared Pool Lib Cache Reload Ratio", "SGA Shared Pool Lib Cache Sharable Statement", - "SGA Shared Pool Lib Cache Shareable User" + "SGA Shared Pool Lib Cache Shareable User", + "CPU Usage", + "Logical Reads", + "DB Time" ] }, "I/O Operations and ASM":{ @@ -425,11 +423,11 @@ def metriccollector(self): "Direct Path Write Time Waited", "Direct Path Write Wait Count", "Db File Parallel Read Time Waited", - "Db File Parallel Read Wait Count", + "Db File Parallel Read Count", "Db File Parallel Write Time Waited", - "Db File Parallel Write Wait Count", + "Db File Parallel Write Count", "Control File Parallel Write Time Waited", - "Control File Parallel Write Wait Count", + "Control File Parallel Write Count", "Control File Sequential Read Time Waited", "Control File Sequential Read Wait Count", "Log Buffer Space Time Waited", @@ -438,7 +436,7 @@ def metriccollector(self): "Log File Sync Wait Count", "Write Complete Waits Time Waited", "Write Complete Waits Wait Count", - "ASM_Details" + "ASM" ] }, "Parsing and Execution":{ @@ -451,7 +449,14 @@ def metriccollector(self): "Parse Failure Count Per Txn", "Soft Parse Ratio", "Total Parse Count Per Sec", - "Total Parse Count Per Txn" + "Total Parse Count Per Txn", + "Queries Per Second", + "Transactions Per Second", + "Slow Query Latency 95 Percentile", + "Full Table Scans Short", + "Full Table Scans Long", + "Full Table Scans Rowid", + "Full Table Scans IM" ] }, "Locks and Contention":{ @@ -464,29 +469,25 @@ def metriccollector(self): "Library Cache Load Lock Wait Count", "Latch Free Time Waited", "Latch Free Wait Count", - "Enqueue Timeouts Per Sec" + "Enqueue Timeouts Per Sec", + "Enqueue Deadlocks", + "Exchange Deadlocks" ] } - - } - - + } self.maindata['units']=METRICS_UNITS self.maindata['s247config']={ "childdiscovery":[ - "Tablespace_Details", - "Tablespace_Datafile_Details", - "PDB_Details", - "ASM_Details" + "Tablespace_List", + "Tablespace_Datafile", + "PDB", + "ASM" ] } self.close_connection() return self.maindata - - if __name__=="__main__": - hostname="localhost" port="1521" sid="ORCL" @@ -494,11 +495,9 @@ def metriccollector(self): password="ORACLE_USER" tls="False" wallet_location=None - oracle_home="/opt/oracle/product/19c/dbhome_1/" - + oracle_home="/opt/oracle/product/21c/dbhomeXE" import argparse parser=argparse.ArgumentParser() - parser.add_argument('--hostname', help='hostname for oracle',default=hostname) parser.add_argument('--port', help='port number for oracle',default=port) parser.add_argument('--sid', help='sid for oracle',default=sid) @@ -507,11 +506,8 @@ def metriccollector(self): parser.add_argument('--tls', help='tls support for oracle',default=tls) parser.add_argument('--wallet_location', help='oracle wallet location',default=wallet_location) parser.add_argument('--oracle_home',help='oracle home path',default=oracle_home) - args=parser.parse_args() - os.environ['ORACLE_HOME']=args.oracle_home obj=oracle(args) - result=obj.metriccollector() print(json.dumps(result)) From d4180a77b6a0fb6a70789216841003ff5282204f Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 10 Sep 2025 15:08:51 +0530 Subject: [PATCH 05/17] Final Oracle plugin code --- oracle/oracle.py | 843 ++++++++++++++++++++++++++++++----------------- 1 file changed, 539 insertions(+), 304 deletions(-) diff --git a/oracle/oracle.py b/oracle/oracle.py index 632022cf..e84cda8f 100644 --- a/oracle/oracle.py +++ b/oracle/oracle.py @@ -3,89 +3,128 @@ import os import warnings warnings.filterwarnings("ignore") -PLUGIN_VERSION=1 -HEARTBEAT=True -METRICS_UNITS={ - "Buffer Cache Hit Ratio":"%", - "Cursor Cache Hit Ratio":"%", - "Library Cache Hit Ratio":"%", - "Soft Parse Ratio":"%", - "Memory Sorts Ratio":"%", - "Session Limit %":"%", - "Shared Pool Free %":"%", - "SQL Service Response Time":"sec", - "Memory Sorts Ratio":"%", - "Database Wait Time Ratio":"%", - "Total PGA Allocated":"bytes", - "Total Freeable PGA Memory":"bytes", - "Maximum PGA Allocated":"bytes", - "Total PGA Inuse":"bytes", - "SGA Fixed Size":"bytes", - "SGA Variable Size":"bytes", - "SGA Database Buffers":"bytes", - "SGA Redo Buffers":"bytes", - "SGA Shared Pool Lib Cache Sharable Statement":"bytes", - "SGA Shared Pool Lib Cache Shareable User":"bytes", - "Total Memory":"bytes", - "FRA Space Limit":"mb", - "FRA Space Used":"mb", - "FRA Space Reclaimable":"mb", - "Response Time":"ms", - "Tablespace_List":{ - "Tablespace_Size":"mb", - "Used_Percent":"%", - "Used_Space":"mb" + +PLUGIN_VERSION = 1 +HEARTBEAT = True + +# Canonical metric names used in tabs and payload (keep these exact) +METRICS_UNITS = { + "Buffer Cache Hit Ratio": "%", + "Cursor Cache Hit Ratio": "%", + "Library Cache Hit Ratio": "%", + "Soft Parse Ratio": "%", + "Memory Sorts Ratio": "%", + "Session Limit %": "%", + "Shared Pool Free %": "%", + "SQL Service Response Time": "sec", + "Database Wait Time Ratio": "%", + "Total PGA Allocated": "bytes", + "Total Freeable PGA Memory": "bytes", + "Maximum PGA Allocated": "bytes", + "Total PGA Inuse": "bytes", + "SGA Fixed Size": "bytes", + "SGA Variable Size": "bytes", + "SGA Database Buffers": "bytes", + "SGA Redo Buffers": "bytes", + "SGA Shared Pool Lib Cache Sharable Statement": "bytes", + "SGA Shared Pool Lib Cache Shareable User": "bytes", + "Total Memory": "bytes", + "FRA Space Limit": "mb", + "FRA Space Used": "mb", + "FRA Space Reclaimable": "mb", + "Response Time": "ms", + "Tablespace_List": { + "Tablespace_Size": "mb", + "Used_Percent": "%", + "Used_Space": "mb" }, - "Tablespace_Datafile":{ - "Data_File_Size":"mb", - "Max_Data_File_Size":"mb", - "Usable_Data_File_Size":"mb" + "Tablespace_Datafile": { + "Data_File_Size": "mb", + "Max_Data_File_Size": "mb", + "Usable_Data_File_Size": "mb" }, - "PDB":{ - "PDB_Size":"mb" + "PDB": { + "PDB_Size": "mb" }, - "ASM":{ + "ASM": { "total_gb": "GB", "free_gb": "GB", "pct_free": "%" }, - "CPU Usage":"seconds", - "Enqueue Deadlocks":"count", - "Exchange Deadlocks":"count", - "Logical Reads":"count", - "Transactions Per Second":"txn/sec", - "Queries Per Second":"query/sec", - "DB Time":"seconds", - "Rollback Segments": "count", - "Rollback Segment Initial Extent":"bytes", - "Rollback Segment Next Extent":"bytes", - "Rollback Segment Max Extents":"count", - "Slow Query Latency 95 Percentile":"ms", - "Full Table Scans Short":"count", - "Full Table Scans Long":"count", - "Full Table Scans Rowid":"count", - "Full Table Scans IM":"count", - "Failed Backups":"count", - "Alert Log Recent Errors":"count" + + # Parsing / execution + "CPU Usage": "seconds", + "Transactions Per Second": "txn/sec", + "Queries Per Second": "query/sec", + "DB Time": "seconds", + "Rollback Segment Initial Extent": "bytes", + "Rollback Segment Next Extent": "bytes", + "Slow Query Latency 95 Percentile": "ms", + + # Waits (canonical names). Use seconds for time keys and count for wait counts. + "Disk File Operations I/O Time Waited (seconds)": "seconds", + "Control File Parallel Write Time Waited (seconds)": "seconds", + "Control File Sequential Read Time Waited (seconds)": "seconds", + "Db File Parallel Read Time Waited (seconds)": "seconds", + "Db File Parallel Write Time Waited (seconds)": "seconds", + "Db File Scattered Read Time Waited (seconds)": "seconds", + "Db File Sequential Read Time Waited (seconds)": "seconds", + "Direct Path Read Time Waited (seconds)": "seconds", + "Direct Path Write Time Waited (seconds)": "seconds", + "Direct Path Sync Time Waited (seconds)": "seconds", + "Log File Sync Time Waited (seconds)": "seconds", + "Log Buffer Space Time Waited (seconds)": "seconds", + "Write Complete Waits Time Waited (seconds)": "seconds", + "Library Cache Load Lock Time Waited (seconds)": "seconds", + "Library Cache Pin Time Waited (seconds)": "seconds", + "Latch Free Time Waited (seconds)": "seconds" } + +# mapping from lower-cased event name to canonical base label (used to build keys) +_EVENT_TO_CANONICAL = { + "direct path read": "Direct Path Read", + "direct path write": "Direct Path Write", + "db file parallel read": "Db File Parallel Read", + "db file parallel write": "Db File Parallel Write", + "control file parallel write": "Control File Parallel Write", + "control file sequential read": "Control File Sequential Read", + "log file sync": "Log File Sync", + "disk file operations i/o": "Disk File Operations I/O", + "db file sequential read": "Db File Sequential Read", + "db file scattered read": "Db File Scattered Read", + "direct path sync": "Direct Path Sync", + "write complete waits": "Write Complete Waits", + "library cache pin": "Library Cache Pin", + "library cache load lock": "Library Cache Load Lock", + "latch free": "Latch Free", + "log buffer space": "Log Buffer Space" +} + + class oracle: - def __init__(self,args): - self.maindata={} + def __init__(self, args): + self.maindata = {} self.maindata['plugin_version'] = PLUGIN_VERSION - self.maindata['heartbeat_required']=HEARTBEAT - self.maindata['units']=METRICS_UNITS - self.username=args.username - self.password=args.password - self.sid=args.sid - self.hostname=args.hostname - self.port=args.port - self.tls=args.tls.lower() - self.wallet_location=args.wallet_location + self.maindata['heartbeat_required'] = HEARTBEAT + self.maindata['units'] = METRICS_UNITS.copy() + self.username = args.username + self.password = args.password + self.sid = args.sid + self.hostname = args.hostname + self.port = args.port + self.tls = args.tls.lower() + self.wallet_location = args.wallet_location + self.conn = None + self.c = None def connect(self, dsn): try: import oracledb - oracledb.init_oracle_client() + try: + oracledb.init_oracle_client() + except Exception: + # init may fail in thin mode, ignore if so + pass self.conn = oracledb.connect(user=self.username, password=self.password, dsn=dsn) self.c = self.conn.cursor() return (True, "Connected") @@ -104,292 +143,462 @@ def close_connection(self): pass def execute_query_row_col(self, query, col_change=False): - queried_data={} + queried_data = {} try: self.c.execute(query) - col_names = [row[0] for row in self.c.description] - tot_cols=len(col_names) + col_names = [row[0] for row in self.c.description] if self.c.description else [] + tot_cols = len(col_names) if col_change: for row in self.c: for i in range(tot_cols): - queried_data[str.title(col_names[i])]=row[i] + queried_data[str(col_names[i]).title()] = row[i] break else: for row in self.c: for i in range(tot_cols): - queried_data[col_names[i]]=row[i] + queried_data[col_names[i]] = row[i] break except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data["status"] = 0 + queried_data['msg'] = str(e) return queried_data + def execute_table_query(self, query, col_aliases=None): + """ + Execute a query that returns multiple rows/columns and return a list of dicts. + If col_aliases is provided (list), use those as dict keys; otherwise use cursor.description names. + """ + try: + self.c.execute(query) + desc = [d[0] for d in self.c.description] if self.c.description else [] + results = [] + for row in self.c: + rowd = {} + for i, val in enumerate(row): + key = None + if col_aliases and i < len(col_aliases): + key = col_aliases[i] + elif i < len(desc): + key = desc[i] + else: + key = f"col_{i}" + rowd[key] = val + results.append(rowd) + return results + except Exception as e: + return {"status": 0, "msg": str(e)} + def execute_query_bulk(self, query, query_name): - queried_data={} + queried_data = {} try: self.c.execute(query) - if query_name=="pga_query": + if query_name == "pga_query": for row in self.c: - value,metric=row - metric=str.title(metric).replace("Pga","PGA") - if metric=="Maximum PGA Allocated": - value=str(value/1024/1024)+" MB" - queried_data[metric]=value - elif query_name=="asm_query": + value, metric = row + metric = str(metric).title().replace("Pga", "PGA") + if metric == "Maximum PGA Allocated": + value = str(value / 1024 / 1024) + " MB" + queried_data[metric] = value + elif query_name == "asm_query": asm_list = [] for row in self.c: asm_name, asm_total_gb, asm_free_gb, asm_pct_free, asm_limit, asm_threshold = row - asm_list.append({"name": asm_name, "ASM_TOTAL_GB" :asm_total_gb, "ASM_FREE_GB": asm_free_gb, "ASM_PCT_FREE": asm_pct_free ,"ASM_LIMIT": asm_limit, "ASM_THRESHOLD": asm_threshold}) - queried_data['ASM']=asm_list + asm_list.append({ + "name": asm_name, + "ASM_TOTAL_GB": asm_total_gb, + "ASM_FREE_GB": asm_free_gb, + "ASM_PCT_FREE": asm_pct_free, + "ASM_LIMIT": asm_limit, + "ASM_THRESHOLD": asm_threshold + }) + queried_data['ASM'] = asm_list else: + # generic bulk: expect VALUE, METRIC_NAME pairs for row in self.c: - value,metric=row - queried_data[metric]=value + try: + value, metric = row + queried_data[metric] = value + except Exception: + # fallback if ordering different + if len(row) >= 2: + queried_data[row[1]] = row[0] + elif len(row) == 1: + queried_data[query_name] = row[0] except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data["status"] = 0 + queried_data['msg'] = str(e) return queried_data - def execute_query(self,metric_query_name): - queried_data={} + def _canonical_for_event(self, event_name): + if not event_name: + return None + lk = event_name.strip().lower() + return _EVENT_TO_CANONICAL.get(lk, None) + + def execute_waits_query(self, metric_query_name): + """ + Runs waits query (left-joined event names to system_event), + converts time (microseconds) -> seconds and writes canonical keys. + """ + queried_data = {} + wait_units = {} try: - self.c.execute(self.metric_queries["Single Queries"][metric_query_name]) + self.c.execute(self.metric_queries[metric_query_name]) for row in self.c: - queried_data[metric_query_name]=row[0] + ename, time_waited_micro, total_waits = row + if ename is None: + continue + canonical = self._canonical_for_event(str(ename)) + # if not in our explicit map, try to derive a readable canonical name + if not canonical: + canonical = " ".join([w.capitalize() for w in str(ename).replace("/", " ").split()]) + # normalize micro -> seconds (safe guard None) + time_waited_micro = 0 if time_waited_micro is None else time_waited_micro + total_waits = 0 if total_waits is None else total_waits + time_seconds = float(time_waited_micro) / 1_000_000.0 + time_key = f"{canonical} Time Waited (seconds)" + count_key = f"{canonical} Wait Count" + queried_data[time_key] = time_seconds + queried_data[count_key] = total_waits + wait_units[time_key] = 'seconds' + wait_units[count_key] = 'count' except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data['status'] = 0 + queried_data['msg'] = str(e) + # update units so payload has units for these dynamic keys + self.maindata['units'].update(wait_units) return queried_data - def execute_waits_query(self,metric_query_name): - queried_data={} - wait_units={} + def execute_tablespace_metrics(self): + # Use aliased query to avoid ambiguous column names + db_block_size = 8192 try: - self.c.execute(self.metric_queries[metric_query_name]) + self.c.execute("select value from v$parameter where name = 'db_block_size'") for row in self.c: - name,time_waited,wait_count=row - queried_data[str.title(name)+" Time Waited (microsec)"]=time_waited - queried_data[str.title(name)+" Wait Count"]=wait_count - wait_units[str.title(name)+" Time Waited (microsec)"]='microseconds' - except Exception as e: - queried_data['status']=0 - queried_data['msg']=str(e) - METRICS_UNITS.update(wait_units) - return queried_data + db_block_size = row[0] + break + except Exception: + db_block_size = 8192 - def execute_tablespace_metrics(self): - db_block_size=8192 - self.c.execute("select value from v$parameter where name = 'db_block_size'") - for row in self.c: - db_block_size=row[0] - break - queried_data={} + queried_data = {} try: - self.c.execute(self.metric_queries["Tablespace Queries"]["Tablespace Metrics Query"]) - tbs_list=[] + # This query aliases b.STATUS etc to avoid ambiguous names + q = self.metric_queries["Tablespace Queries"]["Tablespace Metrics Query"] + self.c.execute(q) + desc = [d[0].upper() for d in self.c.description] + tbs_list = [] for row in self.c: - tbs_dict={} - tbs_dict["name"]=row[0] - if row[2]:tbs_dict['Used_Space']=int(row[2])*int(db_block_size)/1024/1024 - else:tbs_dict['Used_Space']=0 - if row[3]:tbs_dict['Tablespace_Size']=int(row[3])*int(db_block_size)/1024/1024 - else:tbs_dict['Tablespace_Size']=0 - if row[4]:tbs_dict['Used_Percent']=row[4] - else:tbs_dict['Used_Percent']=0 - if row[7]=="OFFLINE": - tbs_dict['TB_Status']=0 - tbs_dict['status']=0 + rowd = dict(zip(desc, row)) + # pick the tablespace name from known aliases + name = rowd.get('DBA_TABLESPACE') or rowd.get('TABLESPACE_NAME') or rowd.get('TABLESPACE') + tbs_dict = {"name": name} + # used_space and tablespace_size fields (these names come from dba_tablespace_usage_metrics) + used_space_blocks = rowd.get('USED_SPACE') or rowd.get('BYTES') or rowd.get('USER_BYTES') or rowd.get('USER_BLOCKS') + tablespace_size_blocks = rowd.get('TABLESPACE_SIZE') or rowd.get('TABLESPACE_SIZE_BYTES') or rowd.get('BLOCKS') + used_percent = rowd.get('USED_PERCENT') or rowd.get('USED_PCT') or rowd.get('USED_PCT_PERCENT') + if used_space_blocks is None: + try: + used_space_blocks = row[2] + except Exception: + used_space_blocks = 0 + if tablespace_size_blocks is None: + try: + tablespace_size_blocks = row[3] + except Exception: + tablespace_size_blocks = 0 + if used_percent is None: + try: + used_percent = row[4] + except Exception: + used_percent = 0 + try: + tbs_dict['Used_Space'] = int(used_space_blocks) * int(db_block_size) / 1024 / 1024 if used_space_blocks else 0 + except Exception: + tbs_dict['Used_Space'] = 0 + try: + tbs_dict['Tablespace_Size'] = int(tablespace_size_blocks) * int(db_block_size) / 1024 / 1024 if tablespace_size_blocks else 0 + except Exception: + tbs_dict['Tablespace_Size'] = 0 + tbs_dict['Used_Percent'] = used_percent or 0 + # determine status from aliased B_STATUS or STATUS + status_val = rowd.get('B_STATUS') or rowd.get('STATUS') or rowd.get('b_STATUS') + # map to Site24x7: 1=online, 0=offline + if isinstance(status_val, str): + tbs_dict['TB_Status'] = 0 if status_val.upper() == 'OFFLINE' else 1 + tbs_dict['status'] = 0 if status_val.upper() == 'OFFLINE' else 1 else: - tbs_dict['TB_Status']=1 - tbs_dict['status']=1 + try: + tbs_dict['TB_Status'] = 1 if int(status_val or 0) != 0 else 1 + tbs_dict['status'] = 1 if int(status_val or 0) != 0 else 1 + except Exception: + tbs_dict['TB_Status'] = 1 + tbs_dict['status'] = 1 tbs_list.append(tbs_dict) - queried_data['Tablespace_List']=tbs_list + queried_data['Tablespace_List'] = tbs_list except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data["status"] = 0 + queried_data['msg'] = str(e) return queried_data - + def execute_tablespace_datafile(self): - queried_data={} + queried_data = {} try: - self.c.execute(self.metric_queries["Tablespace Queries"]["Tablespace Datafile Query"]) - tbs_list=[] + self.c.execute(self.metric_queries["Tablespace Queries"]["Tablespace Datafile Query"]) + tbs_list = [] for row in self.c: - tb_dict={} - tbs_name=row[0] - tbs_datafile=row[1].split("/")[-1] - name=tbs_datafile - tb_dict["name"]=name - tb_dict["Data_File_Size"]=row[2] - tb_dict["Data_File_Blocks"]=row[3] - if row[4]=="YES": - tb_dict["Autoextensible"]=1 - else: - tb_dict["Autoextensible"]=0 - tb_dict["Max_Data_File_Size"]=row[5] - tb_dict["Max_Data_File_Blocks"]=row[6] - tb_dict["Increment_By"]=row[7] - tb_dict["Usable_Data_File_Size"]=row[8] - tb_dict["Usable_Data_File_Blocks"]=row[9] + tb_dict = {} + tbs_name = row[0] + tbs_datafile = row[1].split("/")[-1] + name = tbs_datafile + tb_dict["name"] = name + tb_dict["Data_File_Size"] = row[2] + tb_dict["Data_File_Blocks"] = row[3] + tb_dict["Autoextensible"] = 1 if row[4] == "YES" else 0 + tb_dict["Max_Data_File_Size"] = row[5] + tb_dict["Max_Data_File_Blocks"] = row[6] + tb_dict["Increment_By"] = row[7] + tb_dict["Usable_Data_File_Size"] = row[8] + tb_dict["Usable_Data_File_Blocks"] = row[9] tbs_list.append(tb_dict) - queried_data["Tablespace_Datafile"]=tbs_list + queried_data["Tablespace_Datafile"] = tbs_list except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data["status"] = 0 + queried_data['msg'] = str(e) return queried_data def tablespace_complete(self): - queried_data={} + queried_data = {} try: - query_output_data=self.execute_tablespace_metrics() + query_output_data = self.execute_tablespace_metrics() queried_data.update(query_output_data) - if 'status' in queried_data and queried_data['status']==0: + if 'status' in queried_data and queried_data['status'] == 0: return queried_data - - query_output_data=self.execute_tablespace_datafile() + query_output_data = self.execute_tablespace_datafile() queried_data.update(query_output_data) - if 'status' in queried_data and queried_data['status']==0: + if 'status' in queried_data and queried_data['status'] == 0: return queried_data except Exception as e: - queried_data["status"]=0 - queried_data['msg']=str(e) + queried_data["status"] = 0 + queried_data['msg'] = str(e) return queried_data - + def execute_pdb(self, metric_query_name): - queried_data={} + queried_data = {} try: - self.c.execute(self.metric_queries[metric_query_name]) - pdb_list=[] + self.c.execute(self.metric_queries[metric_query_name]) + pdb_list = [] for row in self.c: - pdb_dict={} - pdb_dict['name']=row[0] - pdb_dict['PDB_ID']=row[1] - pdb_dict['PDB_Size']=row[2] - pdb_dict['Block_Size']=row[3] + pdb_dict = {} + pdb_dict['name'] = row[0] + pdb_dict['PDB_ID'] = row[1] + pdb_dict['PDB_Size'] = row[2] + pdb_dict['Block_Size'] = row[3] pdb_list.append(pdb_dict) - queried_data['PDB']=pdb_list + queried_data['PDB'] = pdb_list except Exception as e: - queried_data['status']=0 - queried_data['msg']=str(e) + queried_data['status'] = 0 + queried_data['msg'] = str(e) return queried_data def metriccollector(self): - self.metric_queries={ - "Bulk Queries":{ - "system_query":"SELECT VALUE, METRIC_NAME FROM GV$SYSMETRIC WHERE METRIC_NAME IN ( 'Soft Parse Ratio', 'Total Parse Count Per Sec', 'Total Parse Count Per Txn', 'Hard Parse Count Per Sec', 'Hard Parse Count Per Txn', 'Parse Failure Count Per Sec', 'Parse Failure Count Per Txn', 'Temp Space Used', 'Session Count', 'Session Limit %', 'Database Wait Time Ratio', 'Memory Sorts Ratio', 'Disk Sort Per Sec', 'Rows Per Sort', 'Total Sorts Per User Call', 'User Rollbacks Per Sec', 'SQL Service Response Time', 'Long Table Scans Per Sec', 'Average Active Sessions', 'Logons Per Sec', 'Global Cache Blocks Los', 'Global Cache Blocks Corrupted', 'GC CR Block Received Per Second', 'Enqueue Timeouts Per Sec', 'Physical Writes Per Sec', 'Physical Reads Per Sec', 'Shared Pool Free %', 'Library Cache Hit Ratio', 'Cursor Cache Hit Ratio', 'Buffer Cache Hit Ratio' )", - "pga_query":"SELECT VALUE, NAME FROM gv$pgastat where NAME IN ('total PGA allocated', 'total freeable PGA memory', 'maximum PGA allocated','total PGA inuse')", - "sga_query":"""SELECT sga.value, CONCAT('SGA ',sga.name) AS name FROM GV$SGA sga INNER JOIN GV$INSTANCE inst ON sga.inst_id = inst.inst_id""", + self.metric_queries = { + "Bulk Queries": { + "system_query": "SELECT VALUE, METRIC_NAME FROM GV$SYSMETRIC WHERE METRIC_NAME IN ( 'Soft Parse Ratio', 'Total Parse Count Per Sec', 'Total Parse Count Per Txn', 'Hard Parse Count Per Sec', 'Hard Parse Count Per Txn', 'Parse Failure Count Per Sec', 'Parse Failure Count Per Txn', 'Temp Space Used', 'Session Count', 'Session Limit %', 'Database Wait Time Ratio', 'Memory Sorts Ratio', 'Disk Sort Per Sec', 'Rows Per Sort', 'Total Sorts Per User Call', 'User Rollbacks Per Sec', 'SQL Service Response Time', 'Long Table Scans Per Sec', 'Average Active Sessions', 'Logons Per Sec', 'Global Cache Blocks Los', 'Global Cache Blocks Corrupted', 'GC CR Block Received Per Second', 'Enqueue Timeouts Per Sec', 'Physical Writes Per Sec', 'Physical Reads Per Sec', 'Shared Pool Free %', 'Library Cache Hit Ratio', 'Cursor Cache Hit Ratio', 'Buffer Cache Hit Ratio' )", + "pga_query": "SELECT VALUE, NAME FROM gv$pgastat where NAME IN ('total PGA allocated', 'total freeable PGA memory', 'maximum PGA allocated','total PGA inuse')", + "sga_query": """SELECT sga.value, CONCAT('SGA ',sga.name) AS name FROM GV$SGA sga INNER JOIN GV$INSTANCE inst ON sga.inst_id = inst.inst_id""", "asm_query": "SELECT name AS asm_name, ROUND(total_mb / 1024, 2) AS asm_total_gb, ROUND(free_mb / 1024, 2) AS asm_free_gb, ROUND((free_mb / total_mb) * 100, 2) AS asm_pct_free, USABLE_FILE_MB AS asm_limit, REQUIRED_MIRROR_FREE_MB AS asm_threshold FROM v$asm_diskgroup" }, - "Single Queries":{ - "Rman Failed Backup Count":"""SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'""", - "Dict Cache Hit Ratio":"""SELECT (1 - (SUM(getmisses)/SUM(gets))) * 100 FROM gv$rowcache""", - "Long Running Queries":"""SELECT COUNT(*) FROM v$session WHERE status='ACTIVE' AND type<>'BACKGROUND' AND last_call_et > 60""", - "Blocking Locks":"""SELECT COUNT(*) FROM gv$session WHERE blocking_session IS NOT NULL""", - "SGA Hit Ratio":"""SELECT (1 - (phy.value - lob.value - dir.value)/ses.value) FROM GV$SYSSTAT ses, GV$SYSSTAT lob, GV$SYSSTAT dir, GV$SYSSTAT phy, GV$INSTANCE inst WHERE ses.name='session logical reads' AND dir.name='physical reads direct' AND lob.name='physical reads direct (lob)' AND phy.name='physical reads' AND ses.inst_id=inst.inst_id AND lob.inst_id=inst.inst_id AND dir.inst_id=inst.inst_id AND phy.inst_id=inst.inst_id""", - "SGA Log Alloc Retries":"""SELECT (rbar.value/re.value) FROM GV$SYSSTAT rbar, GV$SYSSTAT re, GV$INSTANCE inst WHERE rbar.name LIKE 'redo buffer allocation retries' AND re.name LIKE 'redo entries' AND re.inst_id=inst.inst_id AND rbar.inst_id=inst.inst_id""", - "SGA Shared Pool Dict Cache Ratio":"""SELECT (SUM(rcache.getmisses)/SUM(rcache.gets)) FROM GV$rowcache rcache, GV$INSTANCE inst WHERE inst.inst_id=rcache.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Hit Ratio":"""SELECT libcache.gethitratio FROM GV$librarycache libcache, GV$INSTANCE inst WHERE namespace='SQL AREA' AND inst.inst_id=libcache.inst_id""", - "SGA Shared Pool Lib Cache Reload Ratio":"""SELECT (SUM(libcache.reloads)/SUM(libcache.pins)) FROM GV$librarycache libcache, GV$INSTANCE inst WHERE inst.inst_id=libcache.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Sharable Statement":"""SELECT SUM(sqlarea.sharable_mem) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE sqlarea.executions > 5 AND inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", - "SGA Shared Pool Lib Cache Shareable User":"""SELECT SUM(250 * sqlarea.users_opening) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id""", - "Total Memory":"""SELECT SUM(value) FROM GV$sesstat, GV$statname, GV$INSTANCE inst WHERE name = 'session uga memory max' AND GV$sesstat.statistic#=GV$statname.statistic# AND GV$sesstat.inst_id=inst.inst_id AND GV$statname.inst_id=inst.inst_id GROUP BY inst.inst_id""", - "Oracle Database Version":"SELECT version FROM PRODUCT_COMPONENT_VERSION WHERE product LIKE 'Oracle Database%'", - "Response Time":"""SELECT ROUND(VALUE * 10, 2) FROM GV$SYSMETRIC WHERE METRIC_NAME = 'SQL Service Response Time' ORDER BY INST_ID""", - "Number of Session Users":"""SELECT COUNT(DISTINCT username) FROM v$session WHERE username IS NOT NULL""", - "Database Block Size":"""SELECT value FROM v$parameter WHERE name = 'db_block_size'""", - "Invalid Index Count":"""SELECT COUNT(*) FROM dba_indexes WHERE status = 'INVALID'""", - "CPU Usage":"SELECT value FROM v$sysstat WHERE name='CPU used by this session'", - "Enqueue Deadlocks":"SELECT value FROM v$sysstat WHERE name='enqueue deadlocks'", - "Exchange Deadlocks":"SELECT value FROM v$sysstat WHERE name='exchange deadlocks'", - "Logical Reads":"SELECT value FROM v$sysstat WHERE name='session logical reads'", - "Queries Per Second":"""SELECT (value / 60) FROM v$sysstat WHERE name = 'execute count'""", - "Transactions Per Second":"""SELECT ((SELECT value FROM v$sysstat WHERE name = 'user commits') + (SELECT value FROM v$sysstat WHERE name = 'user rollbacks')) / 60 FROM dual""", - "DB Time":"""SELECT SUM(value)/100 FROM v$sys_time_model WHERE stat_name = 'DB time'""", - - "Slow Query Latency 95 Percentile":"""SELECT ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY elapsed_time/executions)) FROM v$sql WHERE executions > 0""", - "Full Table Scans Short":"""SELECT value FROM v$sysstat WHERE name = 'table scans (short tables)'""", - "Full Table Scans Long":"""SELECT value FROM v$sysstat WHERE name = 'table scans (long tables)'""", - "Full Table Scans Rowid":"""SELECT value FROM v$sysstat WHERE name = 'table scans (rowid ranges)'""", - "Full Table Scans IM":"""SELECT value FROM v$sysstat WHERE name = 'table scans (IM)'""", - "Failed Backups":"""SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'""", - "Alert Log Recent Errors":"""SELECT COUNT(*) FROM V$DIAG_ALERT_EXT WHERE ORIGINATING_TIMESTAMP > SYSDATE - 1""", - - "Rollback Segments":"""SELECT COUNT(*) FROM dba_rollback_segs""", - "Rollback Segment Details":"""SELECT segment_name, tablespace_name, status, initial_extent, next_extent, max_extents FROM dba_rollback_segs""" + "Single Queries": { + "Rman Failed Backup Count": "SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'", + "Dict Cache Hit Ratio": "SELECT (1 - (SUM(getmisses)/SUM(gets))) * 100 FROM gv$rowcache", + "Long Running Queries": "SELECT COUNT(*) FROM v$session WHERE status='ACTIVE' AND type<>'BACKGROUND' AND last_call_et > 60", + "Blocking Locks": "SELECT COUNT(*) FROM gv$session WHERE blocking_session IS NOT NULL", + "SGA Hit Ratio": "SELECT (1 - (phy.value - lob.value - dir.value)/ses.value) FROM GV$SYSSTAT ses, GV$SYSSTAT lob, GV$SYSSTAT dir, GV$SYSSTAT phy, GV$INSTANCE inst WHERE ses.name='session logical reads' AND dir.name='physical reads direct' AND lob.name='physical reads direct (lob)' AND phy.name='physical reads' AND ses.inst_id=inst.inst_id AND lob.inst_id=inst.inst_id AND dir.inst_id=inst.inst_id AND phy.inst_id=inst.inst_id", + "SGA Log Alloc Retries": "SELECT (rbar.value/re.value) FROM GV$SYSSTAT rbar, GV$SYSSTAT re, GV$INSTANCE inst WHERE rbar.name LIKE 'redo buffer allocation retries' AND re.name LIKE 'redo entries' AND re.inst_id=inst.inst_id AND rbar.inst_id=inst.inst_id", + "SGA Shared Pool Dict Cache Ratio": "SELECT (SUM(rcache.getmisses)/SUM(rcache.gets)) FROM GV$rowcache rcache, GV$INSTANCE inst WHERE inst.inst_id=rcache.inst_id GROUP BY inst.inst_id", + "SGA Shared Pool Lib Cache Hit Ratio": "SELECT libcache.gethitratio FROM GV$librarycache libcache, GV$INSTANCE inst WHERE namespace='SQL AREA' AND inst.inst_id=libcache.inst_id", + "SGA Shared Pool Lib Cache Reload Ratio": "SELECT (SUM(libcache.reloads)/SUM(libcache.pins)) FROM GV$librarycache libcache, GV$INSTANCE inst WHERE inst.inst_id=libcache.inst_id GROUP BY inst.inst_id", + "SGA Shared Pool Lib Cache Sharable Statement": "SELECT SUM(sqlarea.sharable_mem) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE sqlarea.executions > 5 AND inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id", + "SGA Shared Pool Lib Cache Shareable User": "SELECT SUM(250 * sqlarea.users_opening) FROM GV$sqlarea sqlarea, GV$INSTANCE inst WHERE inst.inst_id=sqlarea.inst_id GROUP BY inst.inst_id", + "Total Memory": "SELECT SUM(value) FROM GV$sesstat, GV$statname, GV$INSTANCE inst WHERE name = 'session uga memory max' AND GV$sesstat.statistic#=GV$statname.statistic# AND GV$sesstat.inst_id=inst.inst_id AND GV$statname.inst_id=inst.inst_id GROUP BY inst.inst_id", + "Oracle Database Version": "SELECT version FROM PRODUCT_COMPONENT_VERSION WHERE product LIKE 'Oracle Database%'", + #"DB ID": "SELECT dbid FROM v$database", + "Response Time": "SELECT ROUND(VALUE * 10, 2) FROM GV$SYSMETRIC WHERE METRIC_NAME = 'SQL Service Response Time' ORDER BY INST_ID", + "Number of Session Users": "SELECT COUNT(DISTINCT username) FROM v$session WHERE username IS NOT NULL", + "Database Block Size": "SELECT value FROM v$parameter WHERE name = 'db_block_size'", + "Invalid Index Count": "SELECT COUNT(*) FROM dba_indexes WHERE status = 'INVALID'", + "CPU Usage": "SELECT value FROM v$sysstat WHERE name='CPU used by this session'", + "Enqueue Deadlocks": "SELECT value FROM v$sysstat WHERE name='enqueue deadlocks'", + "Exchange Deadlocks": "SELECT value FROM v$sysstat WHERE name='exchange deadlocks'", + "Logical Reads": "SELECT value FROM v$sysstat WHERE name='session logical reads'", + "Queries Per Second": "SELECT (value / 60) FROM v$sysstat WHERE name = 'execute count'", + "Transactions Per Second": "SELECT ((SELECT value FROM v$sysstat WHERE name = 'user commits') + (SELECT value FROM v$sysstat WHERE name = 'user rollbacks')) / 60 FROM dual", + "DB Time": "SELECT SUM(value)/100 FROM v$sys_time_model WHERE stat_name = 'DB time'", + "Slow Query Latency 95 Percentile": "SELECT ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY elapsed_time/executions)) FROM v$sql WHERE executions > 0", + "Full Table Scans Short": "SELECT value FROM v$sysstat WHERE name = 'table scans (short tables)'", + "Full Table Scans Long": "SELECT value FROM v$sysstat WHERE name = 'table scans (long tables)'", + "Full Table Scans Rowid": "SELECT value FROM v$sysstat WHERE name = 'table scans (rowid ranges)'", + "Full Table Scans IM": "SELECT value FROM v$sysstat WHERE name = 'table scans (IM)'", + "Failed Backups": "SELECT COUNT(*) FROM v$rman_status WHERE operation='BACKUP' AND status='FAILED'", + "Alert Log Recent Errors": "SELECT COUNT(*) FROM V$DIAG_ALERT_EXT WHERE ORIGINATING_TIMESTAMP > SYSDATE - 1", + "Rollback Segments": "SELECT COUNT(*) FROM dba_rollback_segs", + # detailed rollback segment table (alias segment_name as name) + "Rollback_Segment_Details": "SELECT segment_name AS name, tablespace_name, status, initial_extent, next_extent, max_extents FROM dba_rollback_segs" }, - "Tablespace Queries":{ - "Tablespace Metrics Query":"""SELECT b.TABLESPACE_NAME as "dba_tablespace", d.* , b.CONTENTS, b.LOGGING, b.STATUS FROM dba_tablespace_usage_metrics d FULL JOIN dba_tablespaces b ON d.TABLESPACE_NAME = b.TABLESPACE_NAME""", - "Tablespace Datafile Query":"""SELECT TABLESPACE_NAME, FILE_NAME, (BYTES/1024/1024), BLOCKS, AUTOEXTENSIBLE, (MAXBYTES/1024/1024), (MAXBLOCKS/1024/1024), INCREMENT_BY, (USER_BYTES/1024/1024), USER_BLOCKS FROM DBA_DATA_FILES""" + "Tablespace Queries": { + # alias b.STATUS etc to avoid ambiguous column names that caused ORA-00918 + "Tablespace Metrics Query": """SELECT b.TABLESPACE_NAME AS DBA_TABLESPACE, d.* , b.CONTENTS AS B_CONTENTS, b.LOGGING AS B_LOGGING, b.STATUS AS B_STATUS +FROM dba_tablespace_usage_metrics d +FULL JOIN dba_tablespaces b ON d.TABLESPACE_NAME = b.TABLESPACE_NAME""", + "Tablespace Datafile Query": "SELECT TABLESPACE_NAME, FILE_NAME, (BYTES/1024/1024), BLOCKS, AUTOEXTENSIBLE, (MAXBYTES/1024/1024), (MAXBLOCKS/1024/1024), INCREMENT_BY, (USER_BYTES/1024/1024), USER_BLOCKS FROM DBA_DATA_FILES" }, - "FRA Query":"""SELECT name AS "FRA File Dest", space_limit / (1024 * 1024) AS "FRA Space Limit", space_used / (1024 * 1024) AS "FRA Space Used", space_reclaimable / (1024 * 1024) AS "FRA Space Reclaimable", number_of_files AS "FRA Number of Files" FROM V$RECOVERY_FILE_DEST""", - "Waits Query":"""SELECT event, time_waited_micro, total_waits FROM v$system_event WHERE wait_class != 'Idle' AND ROWNUM <= 20""", - "PDB Query":"""SELECT p.name, p.con_id, NVL(SUM(f.bytes),0)/1024/1024 AS pdb_size_mb, p.block_size FROM v$pdbs p LEFT JOIN cdb_data_files f ON f.con_id = p.con_id GROUP BY p.name, p.con_id, p.block_size""", - "DB Query":"SELECT 1 FROM dual" + # FRA + "FRA Query": "SELECT name AS \"FRA File Dest\", space_limit / (1024 * 1024) AS \"FRA Space Limit\", space_used / (1024 * 1024) AS \"FRA Space Used\", space_reclaimable / (1024 * 1024) AS \"FRA Space Reclaimable\", number_of_files AS \"FRA Number of Files\" FROM V$RECOVERY_FILE_DEST", + # waits: select canonical events from v$event_name left joined with v$system_event + "Waits Query": """ +SELECT en.name AS event, + NVL(se.time_waited_micro,0) AS time_waited_micro, + NVL(se.total_waits,0) AS total_waits +FROM v$event_name en +LEFT JOIN v$system_event se + ON LOWER(se.event) = LOWER(en.name) +WHERE LOWER(en.name) IN ( + 'direct path read', + 'direct path write', + 'db file parallel read', + 'db file parallel write', + 'control file parallel write', + 'control file sequential read', + 'log file sync', + 'disk file operations i/o', + 'db file sequential read', + 'db file scattered read', + 'direct path sync', + 'write complete waits', + 'library cache pin', + 'library cache load lock', + 'latch free', + 'log buffer space' +) +ORDER BY en.name +""", + "PDB Query": "SELECT p.name, p.con_id, NVL(SUM(f.bytes),0)/1024/1024 AS pdb_size_mb, p.block_size FROM v$pdbs p LEFT JOIN cdb_data_files f ON f.con_id = p.con_id GROUP BY p.name, p.con_id, p.block_size" } - if self.tls=="True": - dsn=f"""(DESCRIPTION= + + # Build DSN + if self.tls == "True": + dsn = f"""(DESCRIPTION= (ADDRESS=(PROTOCOL=tcps)(HOST={self.hostname})(PORT={self.port})) (CONNECT_DATA=(SERVICE_NAME={self.sid})) (SECURITY=(MY_WALLET_DIRECTORY={self.wallet_location})) )""" else: - dsn=f"{self.hostname}:{self.port}/{self.sid}" - connection_status=self.connect(dsn) + dsn = f"{self.hostname}:{self.port}/{self.sid}" + + connection_status = self.connect(dsn) if not connection_status[0]: - self.maindata['status']=0 - self.maindata['msg']=connection_status[1] + self.maindata['status'] = 0 + self.maindata['msg'] = connection_status[1] self.close_connection() return self.maindata - for query_name ,bulk_query in self.metric_queries['Bulk Queries'].items(): - query_output_data=self.execute_query_bulk(bulk_query, query_name=query_name) + + # Bulk queries (system metrics, pga, sga, asm) + for query_name, bulk_query in self.metric_queries['Bulk Queries'].items(): + query_output_data = self.execute_query_bulk(bulk_query, query_name=query_name) self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata + + # Single scalar queries (skip rollback details here) for query_name in self.metric_queries['Single Queries']: - query_output_data=self.execute_query(query_name) + if query_name == "Rollback_Segment_Details": + continue + query_output_data = self.execute_query(query_name) self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata - query_output_data=self.tablespace_complete() + + # Tablespace lists and datafiles + query_output_data = self.tablespace_complete() self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata - query_output_data=self.execute_waits_query("Waits Query") + + # Waits (I/O + locks) - convert microsec -> seconds inside handler + query_output_data = self.execute_waits_query("Waits Query") self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata - query_output_data=self.execute_pdb("PDB Query") + + # PDBs + query_output_data = self.execute_pdb("PDB Query") self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata - query_output_data=self.execute_query_row_col(self.metric_queries["DB Query"]) + + # FRA + query_output_data = self.execute_query_row_col(self.metric_queries["FRA Query"]) self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + if 'status' in self.maindata and self.maindata['status'] == 0: self.close_connection() return self.maindata - query_output_data=self.execute_query_row_col(self.metric_queries["FRA Query"]) - self.maindata.update(query_output_data) - if 'status' in self.maindata and self.maindata['status']==0: + + # Rollback Segment Details (table) - we use execute_table_query and alias first column as 'name' + rollback_rows = self.execute_table_query(self.metric_queries["Single Queries"]["Rollback_Segment_Details"], + col_aliases=['name', 'tablespace_name', 'status', 'initial_extent', 'next_extent', 'max_extents']) + if isinstance(rollback_rows, dict) and rollback_rows.get('status') == 0: + self.maindata.update(rollback_rows) self.close_connection() return self.maindata - self.maindata['tabs']={ - "Tablespace and PDB":{ - "order":1, - "tablist":[ + + # Process rollback rows: + # - remove 'status' column from each rollback row + # - strip leading underscores from the name + processed_rollback = [] + for r in rollback_rows: + try: + name_val = r.get('name') if isinstance(r, dict) else None + if name_val is None: + name_val = r.get('segment_name') if isinstance(r, dict) else None + # strip leading underscores + if isinstance(name_val, str): + cleaned_name = name_val.lstrip('_') + else: + cleaned_name = name_val + rr = { + "name": cleaned_name, + "tablespace_name": r.get('tablespace_name'), + "initial_extent": r.get('initial_extent'), + "next_extent": r.get('next_extent'), + "max_extents": r.get('max_extents') + } + processed_rollback.append(rr) + except Exception: + # in case of unexpected shape, skip the row gracefully + continue + + # Attach rollback details without 'status' field under the new key name + self.maindata["Rollback_Segment_Details"] = processed_rollback + + # Tabs & organization: make sure tab metric names exactly match keys produced above + self.maindata['tabs'] = { + "Tablespace and PDB": { + "order": 1, + "tablist": [ "Tablespace_List", "Tablespace_Datafile", - "PDB" + "PDB", + "Rollback_Segment_Details" ] }, - "Buffer Cache and Memory":{ - "order":2, - "tablist":[ + "Buffer Cache and Memory": { + "order": 2, + "tablist": [ "Buffer Cache Hit Ratio", "Database Block Size", "Shared Pool Free %", @@ -413,35 +622,41 @@ def metriccollector(self): "DB Time" ] }, - "I/O Operations and ASM":{ - "order":3, - "tablist":[ + "I/O Operations and ASM": { + "order": 3, + "tablist": [ "Physical Reads Per Sec", "Physical Writes Per Sec", - "Direct Path Read Time Waited", + "Direct Path Read Time Waited (seconds)", "Direct Path Read Wait Count", - "Direct Path Write Time Waited", + "Direct Path Write Time Waited (seconds)", "Direct Path Write Wait Count", - "Db File Parallel Read Time Waited", - "Db File Parallel Read Count", - "Db File Parallel Write Time Waited", - "Db File Parallel Write Count", - "Control File Parallel Write Time Waited", + "Db File Parallel Read Time Waited (seconds)", + "Db File Parallel Read Wait Count", + "Db File Parallel Write Time Waited (seconds)", + "Db File Parallel Write Wait Count", + "Control File Parallel Write Time Waited (seconds)", "Control File Parallel Write Count", - "Control File Sequential Read Time Waited", + "Control File Sequential Read Time Waited (seconds)", "Control File Sequential Read Wait Count", - "Log Buffer Space Time Waited", + "Log Buffer Space Time Waited (seconds)", "Log Buffer Space Wait Count", - "Log File Sync Time Waited", + "Log File Sync Time Waited (seconds)", "Log File Sync Wait Count", - "Write Complete Waits Time Waited", + "Write Complete Waits Time Waited (seconds)", "Write Complete Waits Wait Count", + "Disk File Operations I/O Time Waited (seconds)", + "Disk File Operations I/O Wait Count", + "Db File Scattered Read Time Waited (seconds)", + "Db File Scattered Read Wait Count", + "Direct Path Sync Time Waited (seconds)", + "Direct Path Sync Wait Count", "ASM" ] }, - "Parsing and Execution":{ - "order":4, - "tablist":[ + "Parsing and Execution": { + "order": 4, + "tablist": [ "Cursor Cache Hit Ratio", "Hard Parse Count Per Sec", "Hard Parse Count Per Txn", @@ -459,15 +674,15 @@ def metriccollector(self): "Full Table Scans IM" ] }, - "Locks and Contention":{ - "order":5, - "tablist":[ + "Locks and Contention": { + "order": 5, + "tablist": [ "Blocking Locks", - "Library Cache Pin Time Waited", + "Library Cache Pin Time Waited (seconds)", "Library Cache Pin Wait Count", - "Library Cache Load Lock Time Waited", + "Library Cache Load Lock Time Waited (seconds)", "Library Cache Load Lock Wait Count", - "Latch Free Time Waited", + "Latch Free Time Waited (seconds)", "Latch Free Wait Count", "Enqueue Timeouts Per Sec", "Enqueue Deadlocks", @@ -475,39 +690,59 @@ def metriccollector(self): ] } } - self.maindata['units']=METRICS_UNITS - self.maindata['s247config']={ - "childdiscovery":[ + + # finalize units (already updated dynamically) + self.maindata['units'] = self.maindata.get('units', METRICS_UNITS.copy()) + + # child discovery + self.maindata['s247config'] = { + "childdiscovery": [ "Tablespace_List", "Tablespace_Datafile", "PDB", - "ASM" + "ASM", + "Rollback_Segment_Details" ] } + self.close_connection() return self.maindata -if __name__=="__main__": - hostname="localhost" - port="1521" - sid="ORCL" - username="ORACLE_USER" - password="ORACLE_USER" - tls="False" - wallet_location=None - oracle_home="/opt/oracle/product/21c/dbhomeXE" + def execute_query(self, metric_query_name): + queried_data = {} + try: + self.c.execute(self.metric_queries["Single Queries"][metric_query_name]) + for row in self.c: + queried_data[metric_query_name] = row[0] + except Exception as e: + queried_data["status"] = 0 + queried_data['msg'] = str(e) + return queried_data + + +if __name__ == "__main__": + hostname = "localhost" + port = "1521" + sid = "ORCL" + username = "ORACLE_USER" + password = "ORACLE_USER" + tls = "False" + wallet_location = None + oracle_home = "/opt/oracle/product/21c/dbhomeXE" + import argparse - parser=argparse.ArgumentParser() - parser.add_argument('--hostname', help='hostname for oracle',default=hostname) - parser.add_argument('--port', help='port number for oracle',default=port) - parser.add_argument('--sid', help='sid for oracle',default=sid) - parser.add_argument('--username', help='username for oracle',default=username) - parser.add_argument('--password', help='password for oracle',default=password) - parser.add_argument('--tls', help='tls support for oracle',default=tls) - parser.add_argument('--wallet_location', help='oracle wallet location',default=wallet_location) - parser.add_argument('--oracle_home',help='oracle home path',default=oracle_home) - args=parser.parse_args() - os.environ['ORACLE_HOME']=args.oracle_home - obj=oracle(args) - result=obj.metriccollector() + parser = argparse.ArgumentParser() + parser.add_argument('--hostname', help='hostname for oracle', default=hostname) + parser.add_argument('--port', help='port number for oracle', default=port) + parser.add_argument('--sid', help='sid for oracle', default=sid) + parser.add_argument('--username', help='username for oracle', default=username) + parser.add_argument('--password', help='password for oracle', default=password) + parser.add_argument('--tls', help='tls support for oracle', default=tls) + parser.add_argument('--wallet_location', help='oracle wallet location', default=wallet_location) + parser.add_argument('--oracle_home', help='oracle home path', default=oracle_home) + args = parser.parse_args() + + os.environ['ORACLE_HOME'] = args.oracle_home + obj = oracle(args) + result = obj.metriccollector() print(json.dumps(result)) From 434db56b708bfd0f30fa83521088f5c30940b41d Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Tue, 7 Oct 2025 12:13:42 +0530 Subject: [PATCH 06/17] dropwizard-combined --- .../dropwizard-combined | 366 ++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 dropwizard_connection_metrics/dropwizard-combined diff --git a/dropwizard_connection_metrics/dropwizard-combined b/dropwizard_connection_metrics/dropwizard-combined new file mode 100644 index 00000000..ebc60894 --- /dev/null +++ b/dropwizard_connection_metrics/dropwizard-combined @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +""" +Dropwizard plugin for Site24x7 + +Behavior changes in this version: +- Any output key that starts with 'jvm' (case-insensitive) will be placed into the JVM tab. +- Any output key that appears Jetty-related (contains 'jetty', 'org_eclipse', or is Active_Connections_*) will be placed into the Jetty tab. +- Other tabs remain Connection, Events, Memory; Health metrics still emitted at top-level (appear in Summary). +- Plugin version kept here (bump to notify monitor of structure changes). +""" + +import urllib.request +import json +import argparse +import time +import configparser +import os +import re + +# --- CONFIG / VERSION --- +PLUGIN_VERSION = 8 +HEARTBEAT = True + +DEFAULT_PROTOCOL = "http" +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "8081" +DEFAULT_TIMEOUT = 10 + +# --- MAPPINGS (core) --- +TIMERS_MAP = { + "io.dropwizard.jetty.MutableServletContextHandler.requests": "total_requests_", + "io.dropwizard.jetty.MutableServletContextHandler.get-requests": "get_", + "io.dropwizard.jetty.MutableServletContextHandler.post-requests": "post_", + "io.dropwizard.jetty.MutableServletContextHandler.put-requests": "put_", + "io.dropwizard.jetty.MutableServletContextHandler.delete-requests": "delete_", + "org.eclipse.jetty.server.HttpConnectionFactory.8080.connections": "Active_Connections_8080", + "org.eclipse.jetty.server.HttpConnectionFactory.8081.connections": "Active_Connections_8081", + "org.eclipse.jetty.server.HttpConnectionFactory.8443.connections": "Active_Connections_8443", + "org.eclipse.jetty.server.HttpConnectionFactory.8444.connections": "Active_Connections_8444", +} + +METERS_MAP = { + "ch.qos.logback.core.Appender.all": "log_count", + "ch.qos.logback.core.Appender.debug": "debug_", + "ch.qos.logback.core.Appender.error": "error_", + "ch.qos.logback.core.Appender.info": "info_", + "ch.qos.logback.core.Appender.trace": "trace_", + "ch.qos.logback.core.Appender.warn": "warn_", + "io.dropwizard.jetty.MutableServletContextHandler.1xx-responses": "_1xx_", + "io.dropwizard.jetty.MutableServletContextHandler.2xx-responses": "_2xx_", + "io.dropwizard.jetty.MutableServletContextHandler.3xx-responses": "_3xx_", + "io.dropwizard.jetty.MutableServletContextHandler.4xx-responses": "_4xx_", + "io.dropwizard.jetty.MutableServletContextHandler.5xx-responses": "_5xx_", + # Health meters + "TimeBoundHealthCheck-pool.created": "hc_pool_created", + "TimeBoundHealthCheck-pool.terminated": "hc_pool_terminated", +} + +GAUGES_MAP = { + # JVM basics + "jvm.attribute.uptime": "uptime", + "jvm.threads.count": "threads_count", + "jvm.threads.runnable.count": "threads_runnable_count", + "jvm.classloader.loaded": "classloader_loaded", + "jvm.classloader.unloaded": "classloader_unloaded", + "jvm.filedescriptor": "file_descriptor", + # Memory + "jvm.memory.heap.used": "heap_used", + "jvm.memory.heap.max": "heap_max", + "jvm.memory.non-heap.used": "nonheap_used", + "jvm.memory.non-heap.max": "non_heap_max", + "jvm.memory.total.max": "max_memory", + "jvm.memory.total.used": "used_memory", + "jvm.memory.total.committed": "memory_total_committed", + "jvm.memory.pools.Metaspace.used": "metaspace_used", + "jvm.memory.pools.Compressed-Class-Space.used": "compressedclassspace_used", + "jvm.memory.pools.Code-Cache.used": "jvm_pool_codecache_used", + # Jetty gauges + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.size": "jetty_qtp_dw_size", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.utilization": "jetty_qtp_dw_utilization", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.utilization-max": "jetty_qtp_dw_utilization_max", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs": "jetty_qtp_dw_jobs", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs-queue-utilization": "jetty_qtp_dw_jobs_queue_utilization", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.size": "jetty_qtp_dwadmin_size", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.utilization": "jetty_qtp_dwadmin_utilization", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.utilization-max": "jetty_qtp_dwadmin_utilization_max", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.jobs": "jetty_qtp_dwadmin_jobs", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.jobs-queue-utilization": "jetty_qtp_dwadmin_jobs_queue_utilization", + # Health context + "io.dropwizard.jetty.MutableServletContextHandler.active-requests": "active_requests", + "io.dropwizard.jetty.MutableServletContextHandler.active-dispatches": "active_dispatches", + "io.dropwizard.jetty.MutableServletContextHandler.active-suspended": "active_suspended", +} + +# --- UNITS --- +METRIC_UNITS = { + "total_requests_": "count", + "get_": "count", + "post_": "count", + "put_": "count", + "delete_": "count", + "Active_Connections_8080": "connections", + "Active_Connections_8081": "connections", + "Active_Connections_8443": "connections", + "Active_Connections_8444": "connections", + "log_count": "count", + "debug_": "count", + "error_": "count", + "info_": "count", + "trace_": "count", + "warn_": "count", + "_1xx_": "count", + "_2xx_": "count", + "_3xx_": "count", + "_4xx_": "count", + "_5xx_": "count", + "uptime": "ms", + "threads_count": "units", + "threads_runnable_count": "units", + "classloader_loaded": "count", + "classloader_unloaded": "count", + "file_descriptor": "ratio", + "heap_used": "MB", + "heap_max": "MB", + "nonheap_used": "MB", + "non_heap_max": "MB", + "max_memory": "MB", + "used_memory": "MB", + "memory_total_committed": "MB", + "metaspace_used": "MB", + "compressedclassspace_used": "MB", + "jvm_pool_codecache_used": "MB", + "jetty_qtp_dw_size": "units", + "jetty_qtp_dw_utilization": "ratio", + "jetty_qtp_dw_utilization_max": "ratio", + "jetty_qtp_dw_jobs": "count", + "jetty_qtp_dw_jobs_queue_utilization": "ratio", + "jetty_qtp_dwadmin_size": "units", + "jetty_qtp_dwadmin_utilization": "ratio", + "jetty_qtp_dwadmin_utilization_max": "ratio", + "jetty_qtp_dwadmin_jobs": "count", + "jetty_qtp_dwadmin_jobs_queue_utilization": "ratio", + "hc_pool_created": "count", + "hc_pool_terminated": "count", + "active_requests": "count", + "active_dispatches": "count", + "active_suspended": "count", +} + +# --- Helpers --- +def to_mb(v): + try: + return round(float(v) / (1024.0 * 1024.0), 2) + except Exception: + return v + +def shorten_key(k: str) -> str: + s = k.replace("'", "").replace('"', "") + subs = [ + (r"org\.eclipse\.jetty\.server\.HttpConnectionFactory\.", "jetty_conn_"), + (r"org\.eclipse\.jetty\.util\.thread\.QueuedThreadPool\.", "jetty_qtp_"), + (r"jvm\.memory\.pools\.", "jvm_pool_"), + (r"\.", "_"), (r"-", "_"), (r"\s+", "_"), + ] + for pat, rep in subs: + s = re.sub(pat, rep, s) + return re.sub(r"_+", "_", s).strip("_").lower() + +def load_config(cfg_path="dropwiz.cfg"): + cfg = {} + parser = configparser.ConfigParser() + if os.path.exists(cfg_path): + parser.read(cfg_path) + if "dw" in parser: + sec = parser["dw"] + cfg["protocol"] = sec.get("protocol", DEFAULT_PROTOCOL) + cfg["host"] = sec.get("host", DEFAULT_HOST) + cfg["port"] = sec.get("port", DEFAULT_PORT) + cfg["timeout"] = sec.getint("timeout", DEFAULT_TIMEOUT) + return cfg + +def fetch_metrics(url, timeout): + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=timeout) as res: + return json.loads(res.read().decode()) + +# --- Main plugin class --- +class DropwizardPlugin: + def __init__(self, protocol, host, port, timeout): + self.url = f"{protocol}://{host}:{port}/metrics" + self.timeout = timeout + + def collect(self): + out = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "units": METRIC_UNITS, + "collected_at": int(time.time()), + } + + try: + payload = fetch_metrics(self.url, self.timeout) + except Exception as e: + out["error"] = f"Failed to fetch metrics: {e}" + out["error_details"] = repr(e) + return out + + # Timers: counts; keep p95 for total_requests only + timers = payload.get("timers", {}) if isinstance(payload, dict) else {} + for src, name in TIMERS_MAP.items(): + if src in timers: + t = timers[src] + out[name] = t.get("count", 0) + if name == "total_requests_": + out["total_requests_p95"] = t.get("p95", 0.0) + else: + out[name] = 0 + if name == "total_requests_": + out["total_requests_p95"] = 0.0 + + # Meters + meters = payload.get("meters", {}) if isinstance(payload, dict) else {} + for src, name in METERS_MAP.items(): + out[name] = meters.get(src, {}).get("count", 0) + + # Gauges + gauges = payload.get("gauges", {}) if isinstance(payload, dict) else {} + for src, name in GAUGES_MAP.items(): + val = gauges.get(src, {}).get("value") + if METRIC_UNITS.get(name) == "MB": + val = to_mb(val) + # keep None as None for missing non-numeric gauges + out[name] = val if val is not None else 0 + + # Dynamic: emit GC count/time and memory pool gauges (shortened keys) + for key, g in gauges.items(): + if key.startswith("jvm.gc."): + base = key.rsplit(".", 1)[0] + last = key.rsplit(".", 1)[1] + short = shorten_key(base) + if last == "count": + out[f"{short}_count"] = g.get("value") + elif last == "time": + out[f"{short}_time"] = g.get("value") + elif key.startswith("jvm.memory.pools."): + short = shorten_key(key) + val = g.get("value") + out[short] = to_mb(val) if isinstance(val, (int, float)) and val > 1000 else val + + # Add dynamic meters/gauges that look like db/cache/pool to root (sanitized) + dynamic_keywords = ("db", "datasource", "connection", "pool", "cache", "jdbc") + for mk, mv in meters.items(): + if any(k in mk.lower() for k in dynamic_keywords): + out[shorten_key(mk)] = mv.get("count", 0) + for gk, gv in gauges.items(): + if any(k in gk.lower() for k in dynamic_keywords): + key = shorten_key(gk) + val = gv.get("value") + out[key] = to_mb(val) if isinstance(val, (int, float)) and val > 1000 else val + + # Build tabs dynamically: + # - Connection and Events and Memory have predefined core lists + # - For JVM: include every top-level key that starts with 'jvm' (case-insensitive) + # - For Jetty: include keys that contain 'jetty' or 'org_eclipse' or are Active_Connections_* + connection_tab = [ + "total_requests_", "get_", "post_", "put_", "delete_", + "Active_Connections_8080", "Active_Connections_8081", + "Active_Connections_8443", "Active_Connections_8444" + ] + events_tab = [ + "log_count", "debug_", "error_", "info_", "trace_", "warn_", + "_1xx_", "_2xx_", "_3xx_", "_4xx_", "_5xx_" + ] + memory_tab_core = [ + "heap_used", "heap_max", "nonheap_used", "non_heap_max", + "max_memory", "used_memory", "memory_total_committed", + "metaspace_used", "compressedclassspace_used", "jvm_pool_codecache_used" + ] + + jvm_keys = [] + jetty_keys = [] + + # Scan all collected keys and classify + for key in list(out.keys()): + if key in ("plugin_version", "heartbeat_required", "units", "collected_at", "error", "error_details", "tabs"): + continue + # skip the tabs building keys themselves + low = key.lower() + # Already explicitly put in Connection or Events or Memory core - we'll not duplicate + if key in connection_tab or key in events_tab or key in memory_tab_core: + continue + # identify jvm keys: startwith jvm_ or key startswith 'jvm' + if low.startswith("jvm") or low.startswith("jvm_"): + if key not in jvm_keys: + jvm_keys.append(key) + continue + # identify jetty: contains 'jetty' or 'org_eclipse' or key starts with 'active_connections' or 'active_connections_' or key starts with 'jetty_' + if ("jetty" in low) or ("org_eclipse" in low) or low.startswith("active_connections_") or low.startswith("active_connections") or low.startswith("jetty_"): + if key not in jetty_keys: + jetty_keys.append(key) + continue + # Some metrics from Dropwizard context are named dw_ctx_* or similar; they are app/jetty context related + if low.startswith("dw_") or low.startswith("dwctx") or "queuedthreadpool" in low: + if key not in jetty_keys: + jetty_keys.append(key) + continue + # anything else that looks like a memory pool (jvm.memory.pools shortened) would have jvm prefix due to shortening above + # otherwise skip + + # Make deterministic order: keep core lists first, then sorted dynamic keys + jvm_tablist = sorted(set(jvm_keys), key=lambda k: (not k.startswith("jvm"), k)) + jetty_tablist = sorted(set(jetty_keys)) + + # Final tablists - ensure no duplicates across tabs + def dedupe_preserve_order(seq): + seen = set() + out_list = [] + for s in seq: + if s not in seen: + seen.add(s) + out_list.append(s) + return out_list + + connection_tab = dedupe_preserve_order(connection_tab) + events_tab = dedupe_preserve_order(events_tab) + memory_tab = dedupe_preserve_order(memory_tab_core) + jvm_tab = dedupe_preserve_order(jvm_tablist) + jetty_tab = dedupe_preserve_order(jetty_tablist) + + # Attach computed tabs (JVM includes jvm_tab keys + also keep some important JVM keys in the tab) + out["tabs"] = { + "Connection": {"order": 1, "tablist": connection_tab}, + "Events": {"order": 2, "tablist": events_tab}, + "JVM": {"order": 3, "tablist": dedupe_preserve_order([ + # prefer some stable JVM keys first + "uptime", "threads_count", "threads_runnable_count", + "classloader_loaded", "classloader_unloaded", "file_descriptor" + ] + jvm_tab)}, + "Memory": {"order": 4, "tablist": memory_tab}, + "Jetty": {"order": 5, "tablist": jetty_tab} + } + + return out + +# --- CLI --- +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--protocol", default=None) + p.add_argument("--host", default=None) + p.add_argument("--port", default=None) + p.add_argument("--timeout", type=int, default=None) + return p.parse_args() + +def main(): + cfg = load_config() + args = parse_args() + protocol = args.protocol or cfg.get("protocol") or DEFAULT_PROTOCOL + host = args.host or cfg.get("host") or DEFAULT_HOST + port = args.port or cfg.get("port") or DEFAULT_PORT + timeout = args.timeout or cfg.get("timeout") or DEFAULT_TIMEOUT + plugin = DropwizardPlugin(protocol, host, port, timeout) + print(json.dumps(plugin.collect(), indent=2, sort_keys=False, default=str)) + + +if __name__ == "__main__": + main() From 94c9e0a9c2f4acf1c02a64881b224200c2e21cd8 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Tue, 7 Oct 2025 12:21:11 +0530 Subject: [PATCH 07/17] thread-locks-windows-os.ps1 --- thread-locks-windows-os.ps1 | 125 ++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 thread-locks-windows-os.ps1 diff --git a/thread-locks-windows-os.ps1 b/thread-locks-windows-os.ps1 new file mode 100644 index 00000000..b42f66b9 --- /dev/null +++ b/thread-locks-windows-os.ps1 @@ -0,0 +1,125 @@ +<# +threadlocks.ps1 +Collects only OS-level thread/lock metrics and outputs a single flat JSON object. +No .NET counters, no timestamps. Suitable for Site24x7 plugin ingestion. + +Metrics (flat keys): + - total_threads + - blocked_threads_count + - threads_waiting_on_sync_count + - processor_queue_length + - context_switches_per_sec + +Also includes: + - plugin_version (int) + - heartbeat_required ("true") + - units (map) + - msg (string) for non-fatal errors/warnings + +Run as-is. Get-CimInstance Win32_Thread enumeration can be slow on systems with many threads. +#> + +# Helper conversions +function Safe-Int([object]$v) { + try { + if ($null -eq $v) { return $null } + return [int]$v + } catch { + return $null + } +} +function Safe-Double([object]$v) { + try { + if ($null -eq $v) { return $null } + return [double]$v + } catch { + return $null + } +} + +# Base output +$output = @{ + plugin_version = 1 + heartbeat_required = "true" + + total_threads = $null + blocked_threads_count = $null + threads_waiting_on_sync_count = $null + processor_queue_length = $null + context_switches_per_sec = $null + + units = @{ + total_threads = "count" + blocked_threads_count = "count" + threads_waiting_on_sync_count = "count" + processor_queue_length = "count" + context_switches_per_sec = "per_sec" + } + + msg = "" +} + +# --- 1) Fast perf counters (system-level) --- +try { + $sys = Get-CimInstance -Namespace root\cimv2 -ClassName Win32_PerfFormattedData_PerfOS_System -ErrorAction Stop + if ($sys -ne $null) { + if ($sys.Threads -ne $null) { $output.total_threads = Safe-Int($sys.Threads) } + if ($sys.ProcessorQueueLength -ne $null) { $output.processor_queue_length = Safe-Int($sys.ProcessorQueueLength) } + if ($sys.ContextSwitchesPersec -ne $null) { $output.context_switches_per_sec = Safe-Double($sys.ContextSwitchesPersec) } + } +} catch { + $err = $_.Exception.Message + if ($output.msg -ne "") { $output.msg = $output.msg + " | perf_read_err:" + $err } else { $output.msg = "perf_read_err:" + $err } +} + +# --- 2) Win32_Thread enumeration for blocked counts and sync waits (best-effort) --- +try { + $threads = Get-CimInstance -ClassName Win32_Thread -ErrorAction Stop + + $total = 0 + $blocked = 0 + $syncWaits = 0 + + foreach ($t in $threads) { + $total += 1 + + # ThreadWaitReason may be null/0 when not waiting on a reason + $waitReason = $null + try { $waitReason = $t.ThreadWaitReason } catch { $waitReason = $null } + + $hasWaitReason = ($waitReason -ne $null -and $waitReason -ne 0 -and $waitReason -ne "") + if ($hasWaitReason) { $syncWaits += 1 } + + # ThreadState: 5 means Waiting (per Win32_Thread doc); treat Waiting or having a wait reason as blocked + $isWaiting = $false + try { + $stateInt = 0 + if ($t.ThreadState -ne $null -and [int]::TryParse([string]$t.ThreadState, [ref]$stateInt) -and $stateInt -eq 5) { + $isWaiting = $true + } + } catch {} + + if ($hasWaitReason) { $isWaiting = $true } + if ($isWaiting) { $blocked += 1 } + } + + # Fill totals if not provided by perf above + if ($output.total_threads -eq $null) { $output.total_threads = $total } + $output.blocked_threads_count = $blocked + $output.threads_waiting_on_sync_count = $syncWaits + +} catch { + $err = $_.Exception.Message + if ($output.msg -ne "") { $output.msg = $output.msg + " | wmi_thread_err:" + $err } else { $output.msg = "wmi_thread_err:" + $err } +} + +# Ensure keys exist and are single scalar values +foreach ($k in @('total_threads','blocked_threads_count','threads_waiting_on_sync_count','processor_queue_length','context_switches_per_sec')) { + if (-not $output.Contains($k)) { $output[$k] = $null } + if (-not $output.units.ContainsKey($k)) { $output.units[$k] = "count" } +} + +# Output compact JSON (flat) +$output | ConvertTo-Json -Depth 4 -Compress + +exit 0 From 71f412efd543cc2697dfd551e548be34973dd535 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:49:12 +0530 Subject: [PATCH 08/17] Create README.md --- dropwizard/README.md | 256 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 dropwizard/README.md diff --git a/dropwizard/README.md b/dropwizard/README.md new file mode 100644 index 00000000..91742889 --- /dev/null +++ b/dropwizard/README.md @@ -0,0 +1,256 @@ +# Dropwizard Monitoring + +## Prerequisites +- Download and install the latest version of the Site24x7 agent on the server where you plan to run the plugin. +- Python 3 must be installed. + +## Quick installation + +To quickly test the plugin against a local Dropwizard app: + +```bash + +python3 dropwiz.py –host localhost –port 8081 + +``` + +To test the raw metrics endpoint from the agent host: + +```bash + +curl \-sS http://localhost:8081/metrics | head \-n 2 + +``` + +## Standard Installation +If you're not using Linux servers or want to install the plugin manually, follow the steps below. + +### **Installation** + +- Create a directory named `dropwizard`. + + ```bash + mkdir dropwizard + cd dropwizard/ + ``` + + - Download the below files [oracle.cfg](https://github.com/site24x7/plugins/blob/master/oracle/oracle.cfg) and [oracle.py](https://github.com/site24x7/plugins/blob/master/oracle/oracle.py) place it under the `oracle` directory. + + ```bash + wget + wget + ``` +### Execute the plugin + +- Execute the below command with appropriate arguments to check for the valid json output: + + ```bash + python3 /opt/site24x7/monagent/plugins/dropwiz/dropwiz.py –host \ –port \ + ``` +### Move Plugin to Agent Directory + + #### Linux + +- Place the `dropwizard` under the Site24x7 Linux Agent plugin directory: + + ```bash + mv dropwizard /opt/site24x7/monagent/plugins + ``` + +#### Windows + +- Since it's a Python plugin, to run the plugin in a Windows server please follow the steps in [this link](https://support.site24x7.com/portal/en/kb/articles/run-python-plugin-scripts-in-windows-servers). The remaining configuration steps are the same. + +- Further, move the folder `dropwizard` into the Site24x7 Windows Agent plugin directory: + + C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins\dropwizard + + +The agent will automatically execute the plugin within five minutes and send performance data to the Site24x7 data center. + +--- + + +## **Configuration (dropwizard.cfg)** + +Put dropwizard.cfg beside dropwizard.py (or edit values via CLI). +```bash + +[dropwizard] + +protocol = http +host = localhost +port = 8081 +timeout = 30 + +``` + +* protocol — http or https. + +* host — hostname or IP for the Dropwizard app. + +* port — port where /metrics is served. + +* timeout — seconds to wait for the /metrics response before failing. + +CLI overrides config file: \--host, \--port, \--protocol, \--timeout. + +--- + +## **How to install (Linux agent)** + +1. Create plugin directory on the agent host (if not present) and copy files: + +```bash + +sudo mkdir \-p /opt/site24x7/monagent/plugins/dropwizard + +sudo cp dropwizard.py dropwizard.cfg /opt/site24x7/monagent/plugins/dropwizard/ + +sudo chmod \+x /opt/site24x7/monagent/plugins/dropwizard/dropwizard.py + +``` + +2. Test locally: + +```bash + +python3 /opt/site24x7/monagent/plugins/dropwizard/dropwizard.py –host \ –port \ + +``` + +3. The Site24x7 agent will pick up the plugin output automatically (typically within 5 minutes). If not, check agent logs and file permissions. + +--- + +## Supported Metrics + +Below are the metrics grouped exactly as the plugin uses them. Each table uses the Name | Description format so you can copy/paste into the repo. + +### **Summary tab** + +| Name | Description | +| ----- | ----- | +| HealthCheck Pool Created | Health check thread-pool ‘created’ events (count) | +| HealthCheck Pool Terminated | Health check thread-pool ‘terminated’ events (count) | +| collected_at | The UNIX timestamp (in seconds) indicating when the Dropwizard metrics were collected by the plugin. It helps correlate data collection time and detect delays in polling or transmission | + +### **Connection tab** + +| Name | Description | +| ----- | ----- | +| Total Requests | Total requests served by the app since start (count) | +| Get Requests | HTTP GET request count | +| Post Requests | HTTP POST request count | +| Put Requests | HTTP PUT request count | +| Delete Requests | HTTP DELETE request count | +| Connections at 8080 | Active Jetty connections on port 8080 (connections) | +| Connections at 8081 | Active Jetty connections on port 8081 (connections) | +| Connections at 8443 | Active Jetty connections on port 8443 (connections) | +| Connections at 8444 | Active Jetty connections on port 8444 (connections) | + +### **Events tab** + +| Name | Description | +| ----- | ----- | +| Log Count | Total number of log entries (all levels) | +| Debug Logs | Count of DEBUG-level log entries | +| Error Logs | Count of ERROR-level log entries | +| Info Logs | Count of INFO-level log entries | +| Trace Logs | Count of TRACE-level log entries | +| Warn Logs | Count of WARN-level log entries | +| 1xx Responses | Count of 1xx informational HTTP responses | +| 2xx Responses | Count of 2xx successful HTTP responses | +| 3xx Responses | Count of 3xx redirection HTTP responses | +| 4xx Responses | Count of 4xx client error HTTP responses | +| 5xx Responses | Count of 5xx server error HTTP responses | + +### **JVM tab** + +| Name | Description | +| ----- | ----- | +| JVM Uptime | JVM uptime in milliseconds | +| Threads Count | Total number of JVM threads | +| Threads Runnable Count | Number of threads currently runnable | +| Classloader Loaded | Total classes loaded by JVM (count) | +| Classloader Unloaded | Total classes unloaded by JVM (count) | +| File Descriptor Ratio | File descriptor usage metric (ratio/gauge) | +| GC G1 Young Generation Count | Number of G1 young-generation (minor) GC events | +| GC G1 Young Generation Time | Time spent in young-generation GC (ms) | +| GC G1 Old Generation Count | Number of G1 old-generation (major) GC events | +| GC G1 Old Generation Time | Time spent in old-generation GC (ms) | +| GC G1 Concurrent GC Count | Number of concurrent G1 GC cycles | +| GC G1 Concurrent GC Time | Time spent in concurrent G1 GC (ms) | + +--- + +### **Memory tab** + +| Name | Description | +| ----- | ----- | +| Heap Used | Heap memory currently used (MB) | +| Heap Max | Maximum heap memory (MB) | +| Non-Heap Used | Non-heap memory currently used (MB) | +| Non-Heap Max | Maximum non-heap memory (MB) | +| Max Memory | JVM total max memory (MB) | +| Used Memory | JVM total used memory (MB) | +| Memory Total Committed | Total committed memory (MB) | +| Metaspace Used | Metaspace used (MB) | +| Compressed Class Space Used | Compressed class space used (MB) | +| Code Cache Used | Code cache used (MB) | +| G1 Eden Space Used | G1 Eden pool used (MB) | +| G1 Old Gen Used | G1 Old Gen pool used (MB) | +| G1 Survivor Space Used | G1 Survivor pool used (MB) | + +### **Jetty tab** + +| Name | Description | +| ----- | ----- | +| Jetty DW Pool Size | Worker thread pool size (units) for main/dropwizard pool | +| Jetty DW Utilization | Worker pool utilization (ratio) | +| Jetty DW Utilization Max | Observed max worker pool utilization (ratio) | +| Jetty DW Jobs | Pending jobs in worker queue (count) | +| Jetty DW Queue Utilization | Worker queue utilization ratio | +| Jetty DW Admin Pool Size | Admin thread pool size (units) | +| Jetty DW Admin Utilization | Admin pool utilization (ratio) | +| Jetty DW Admin Utilization Max | Observed max admin pool utilization (ratio) | +| Jetty DW Admin Jobs | Pending admin jobs (count) | +| Jetty DW Admin Queue Utilization | Admin queue utilization ratio | +| Active Requests | Active requests in the servlet context (count) | +| Active Dispatches | Active dispatch handlers in the servlet context (count) | +| Active Suspended | Active suspended (async) requests (count) | + +## **Units mapping (as used in the plugin)** + +| Name | Unit | +| ----- | ----- | +| Connections at 8080 / 8081 / 8443 / 8444 | connections | +| JVM Uptime | ms | +| Threads Count / Threads Runnable Count | units | +| File Descriptor Ratio | ratio | +| Heap Used / Heap Max / Non-Heap Used / Non-Heap Max / Max Memory / Used Memory / Memory Total Committed / Metaspace Used / Compressed Class Space Used / Code Cache Used / G1 pool metrics | MB | + +## **Why these metrics matter (short)** + +* **Total Requests & per-method counts** — measure throughput and traffic composition; useful to detect traffic spikes or drops. + +* **Active Connections** — indicates current client concurrency on Jetty connectors; helps spot connection saturation. + +* **Log counts & response codes** — detect errors and client issues quickly (rising 4xx/5xx or ERROR logs). + +* **JVM / Memory / GC** — detect memory leaks, GC pressure, increased pause times, OOM risk. + +* **Jetty threadpool & queue** — detect thread starvation or queue buildup (slow downstream, blocked requests). + +--- + +## **Gauges vs Meters vs Timers (concise)** + +* **Gauges**: instantaneous values (e.g., heap used, thread counts). + +* **Meters**: measures of throughput/rate (e.g., counts that imply a rate). + +* **Timers**: measure duration distributions — in this explicit-mapping plugin we collect timer counts; percentiles are not emitted except if you extend mappings. + +--- + From 984c447990bd331b118e9ba3a9269f2dfe15b532 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:50:13 +0530 Subject: [PATCH 09/17] Create dropwizard.py --- dropwizard/dropwizard.py | 279 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 dropwizard/dropwizard.py diff --git a/dropwizard/dropwizard.py b/dropwizard/dropwizard.py new file mode 100644 index 00000000..3e07cb44 --- /dev/null +++ b/dropwizard/dropwizard.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 + +import urllib.request +import json +import argparse +import time +import configparser +import os + +PLUGIN_VERSION = 1 +HEARTBEAT = True + +DEFAULT_PROTOCOL = "http" +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "8081" +DEFAULT_TIMEOUT = 30 + +# ------------------------------- +# TIMER METRICS (Requests, Connections) +# ------------------------------- +TIMERS_MAP = { + "io.dropwizard.jetty.MutableServletContextHandler.requests": "Total Requests", + "io.dropwizard.jetty.MutableServletContextHandler.get-requests": "Get Requests", + "io.dropwizard.jetty.MutableServletContextHandler.post-requests": "Post Requests", + "io.dropwizard.jetty.MutableServletContextHandler.put-requests": "Put Requests", + "io.dropwizard.jetty.MutableServletContextHandler.delete-requests": "Delete Requests", + "org.eclipse.jetty.server.HttpConnectionFactory.8080.connections": "Connections at 8080", + "org.eclipse.jetty.server.HttpConnectionFactory.8081.connections": "Connections at 8081", + "org.eclipse.jetty.server.HttpConnectionFactory.8443.connections": "Connections at 8443", + "org.eclipse.jetty.server.HttpConnectionFactory.8444.connections": "Connections at 8444", +} + +# ------------------------------- +# METER METRICS (Logs, Status Codes, Health) +# ------------------------------- +METERS_MAP = { + # Log-related + "ch.qos.logback.core.Appender.all": "Log Count", + "ch.qos.logback.core.Appender.debug": "Debug Logs", + "ch.qos.logback.core.Appender.error": "Error Logs", + "ch.qos.logback.core.Appender.info": "Info Logs", + "ch.qos.logback.core.Appender.trace": "Trace Logs", + "ch.qos.logback.core.Appender.warn": "Warn Logs", + + # Response Codes + "io.dropwizard.jetty.MutableServletContextHandler.1xx-responses": "1xx Responses", + "io.dropwizard.jetty.MutableServletContextHandler.2xx-responses": "2xx Responses", + "io.dropwizard.jetty.MutableServletContextHandler.3xx-responses": "3xx Responses", + "io.dropwizard.jetty.MutableServletContextHandler.4xx-responses": "4xx Responses", + "io.dropwizard.jetty.MutableServletContextHandler.5xx-responses": "5xx Responses", + + # Health Check metrics + "TimeBoundHealthCheck-pool.created": "HealthCheck Pool Created", + "TimeBoundHealthCheck-pool.terminated": "HealthCheck Pool Terminated", +} + +# ------------------------------- +# GAUGE METRICS (JVM, Memory, Jetty) +# ------------------------------- +GAUGES_MAP = { + # JVM - Threads, Classloader + "jvm.attribute.uptime": "JVM Uptime", + "jvm.threads.count": "Threads Count", + "jvm.threads.runnable.count": "Threads Runnable Count", + "jvm.classloader.loaded": "Classloader Loaded", + "jvm.classloader.unloaded": "Classloader Unloaded", + "jvm.filedescriptor": "File Descriptor Ratio", + + # JVM - Memory (Heap, Non-Heap, Total) + "jvm.memory.heap.used": "Heap Used", + "jvm.memory.heap.max": "Heap Max", + "jvm.memory.non-heap.used": "Non-Heap Used", + "jvm.memory.non-heap.max": "Non-Heap Max", + "jvm.memory.total.max": "Max Memory", + "jvm.memory.total.used": "Used Memory", + "jvm.memory.total.committed": "Memory Total Committed", + + # JVM - Memory Pools + "jvm.memory.pools.Metaspace.used": "Metaspace Used", + "jvm.memory.pools.Compressed-Class-Space.used": "Compressed Class Space Used", + "jvm.memory.pools.Code-Cache.used": "Code Cache Used", + "jvm.memory.pools.G1-Eden-Space.used": "G1 Eden Space Used", + "jvm.memory.pools.G1-Old-Gen.used": "G1 Old Gen Used", + "jvm.memory.pools.G1-Survivor-Space.used": "G1 Survivor Space Used", + + # JVM - Garbage Collection + "jvm.gc.G1-Young-Generation.count": "GC G1 Young Generation Count", + "jvm.gc.G1-Young-Generation.time": "GC G1 Young Generation Time", + "jvm.gc.G1-Old-Generation.count": "GC G1 Old Generation Count", + "jvm.gc.G1-Old-Generation.time": "GC G1 Old Generation Time", + "jvm.gc.G1-Concurrent-GC.count": "GC G1 Concurrent GC Count", + "jvm.gc.G1-Concurrent-GC.time": "GC G1 Concurrent GC Time", + + # Jetty Thread Pools (DW) + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.size": "Jetty DW Pool Size", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.utilization": "Jetty DW Utilization", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.utilization-max": "Jetty DW Utilization Max", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs": "Jetty DW Jobs", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw.jobs-queue-utilization": "Jetty DW Queue Utilization", + + # Jetty Thread Pools (DW Admin) + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.size": "Jetty DW Admin Pool Size", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.utilization": "Jetty DW Admin Utilization", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.utilization-max": "Jetty DW Admin Utilization Max", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.jobs": "Jetty DW Admin Jobs", + "org.eclipse.jetty.util.thread.QueuedThreadPool.dw-admin.jobs-queue-utilization": "Jetty DW Admin Queue Utilization", + + # Jetty - Active requests / dispatches + "io.dropwizard.jetty.MutableServletContextHandler.active-requests": "Active Requests", + "io.dropwizard.jetty.MutableServletContextHandler.active-dispatches": "Active Dispatches", + "io.dropwizard.jetty.MutableServletContextHandler.active-suspended": "Active Suspended", +} + +# ------------------------------- +# UNITS +# ------------------------------- +METRIC_UNITS = { + "Connections at 8080": "connections", + "Connections at 8081": "connections", + "Connections at 8443": "connections", + "Connections at 8444": "connections", + "JVM Uptime": "ms", + "Threads Count": "units", + "Threads Runnable Count": "units", + "File Descriptor Ratio": "ratio", + "Heap Used": "MB", + "Heap Max": "MB", + "Non-Heap Used": "MB", + "Non-Heap Max": "MB", + "Max Memory": "MB", + "Used Memory": "MB", + "Memory Total Committed": "MB", + "Metaspace Used": "MB", + "Compressed Class Space Used": "MB", + "Code Cache Used": "MB", + "G1 Eden Space Used": "MB", + "G1 Old Gen Used": "MB", + "G1 Survivor Space Used": "MB", +} + +# ------------------------------- +# Helper Functions +# ------------------------------- +def to_mb(v): + try: + return round(float(v) / (1024.0 * 1024.0), 2) + except Exception: + return v + + +def load_config(cfg_path="dropwiz.cfg"): + cfg = {} + parser = configparser.ConfigParser() + if os.path.exists(cfg_path): + parser.read(cfg_path) + if "dw" in parser: + sec = parser["dw"] + cfg["protocol"] = sec.get("protocol", DEFAULT_PROTOCOL) + cfg["host"] = sec.get("host", DEFAULT_HOST) + cfg["port"] = sec.get("port", DEFAULT_PORT) + cfg["timeout"] = sec.getint("timeout", DEFAULT_TIMEOUT) + return cfg + + +def fetch_metrics(url, timeout): + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=timeout) as res: + return json.loads(res.read().decode()) + +# ------------------------------- +# Main Plugin +# ------------------------------- +class DropwizardPlugin: + def __init__(self, protocol, host, port, timeout): + self.url = f"{protocol}://{host}:{port}/metrics" + self.timeout = timeout + + def collect(self): + out = { + "plugin_version": PLUGIN_VERSION, + "heartbeat_required": HEARTBEAT, + "units": METRIC_UNITS, + "collected_at": int(time.time()), + } + + try: + payload = fetch_metrics(self.url, self.timeout) + except Exception as e: + out["error"] = f"Failed to fetch metrics: {e}" + out["error_details"] = repr(e) + return out + + # TIMERS + timers = payload.get("timers", {}) if isinstance(payload, dict) else {} + for src, name in TIMERS_MAP.items(): + val = timers.get(src, {}).get("count", 0) + out[name] = val + + # METERS + meters = payload.get("meters", {}) if isinstance(payload, dict) else {} + for src, name in METERS_MAP.items(): + val = meters.get(src, {}).get("count", 0) + out[name] = val + + # GAUGES + gauges = payload.get("gauges", {}) if isinstance(payload, dict) else {} + for src, name in GAUGES_MAP.items(): + val = gauges.get(src, {}).get("value") + if METRIC_UNITS.get(name) == "MB" and isinstance(val, (int, float)): + val = to_mb(val) + out[name] = val if val is not None else 0 + + # ------------------------------- + # Tabs + # ------------------------------- + connection_tab = [ + "Total Requests", "Get Requests", "Post Requests", "Put Requests", "Delete Requests", + "Connections at 8080", "Connections at 8081", "Connections at 8443", "Connections at 8444" + ] + events_tab = [ + "Log Count", "Debug Logs", "Error Logs", "Info Logs", "Trace Logs", "Warn Logs", + "1xx Responses", "2xx Responses", "3xx Responses", "4xx Responses", "5xx Responses" + ] + jvm_tab = [ + "JVM Uptime", "Threads Count", "Threads Runnable Count", + "Classloader Loaded", "Classloader Unloaded", "File Descriptor Ratio", + "GC G1 Young Generation Count", "GC G1 Young Generation Time", + "GC G1 Old Generation Count", "GC G1 Old Generation Time", + "GC G1 Concurrent GC Count", "GC G1 Concurrent GC Time" + ] + memory_tab = [ + "Heap Used", "Heap Max", "Non-Heap Used", "Non-Heap Max", + "Max Memory", "Used Memory", "Memory Total Committed", + "Metaspace Used", "Compressed Class Space Used", "Code Cache Used", + "G1 Eden Space Used", "G1 Old Gen Used", "G1 Survivor Space Used" + ] + jetty_tab = [ + "Jetty DW Pool Size", "Jetty DW Utilization", "Jetty DW Utilization Max", + "Jetty DW Jobs", "Jetty DW Queue Utilization", + "Jetty DW Admin Pool Size", "Jetty DW Admin Utilization", + "Jetty DW Admin Utilization Max", "Jetty DW Admin Jobs", + "Jetty DW Admin Queue Utilization", + "Active Requests", "Active Dispatches", "Active Suspended" + ] + + out["tabs"] = { + "Connection": {"order": 1, "tablist": connection_tab}, + "Events": {"order": 2, "tablist": events_tab}, + "JVM": {"order": 3, "tablist": jvm_tab}, + "Memory": {"order": 4, "tablist": memory_tab}, + "Jetty": {"order": 5, "tablist": jetty_tab}, + } + + return out + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--protocol", default=None) + p.add_argument("--host", default=None) + p.add_argument("--port", default=None) + p.add_argument("--timeout", type=int, default=None) + return p.parse_args() + + +def main(): + cfg = load_config() + args = parse_args() + protocol = args.protocol or cfg.get("protocol") or DEFAULT_PROTOCOL + host = args.host or cfg.get("host") or DEFAULT_HOST + port = args.port or cfg.get("port") or DEFAULT_PORT + timeout = args.timeout or cfg.get("timeout") or DEFAULT_TIMEOUT + + plugin = DropwizardPlugin(protocol, host, port, timeout) + print(json.dumps(plugin.collect(), indent=2, sort_keys=False, default=str)) + + +if __name__ == "__main__": + main() From 4a9cfc9a6b739edecb1db32b815d8b22788d739e Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:51:03 +0530 Subject: [PATCH 10/17] Create dropwizard.cfg --- dropwizard/dropwizard.cfg | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 dropwizard/dropwizard.cfg diff --git a/dropwizard/dropwizard.cfg b/dropwizard/dropwizard.cfg new file mode 100644 index 00000000..7fdb15b5 --- /dev/null +++ b/dropwizard/dropwizard.cfg @@ -0,0 +1,5 @@ +[dropwizard] +protocol = http +host = localhost +port = 8081 +timeout = 30 From 186a7af5e39c38a2fec53fec64ea486ad92d4178 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:52:29 +0530 Subject: [PATCH 11/17] Update README.md --- dropwizard/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index 91742889..f9569f1e 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -37,8 +37,8 @@ If you're not using Linux servers or want to install the plugin manually, follow - Download the below files [oracle.cfg](https://github.com/site24x7/plugins/blob/master/oracle/oracle.cfg) and [oracle.py](https://github.com/site24x7/plugins/blob/master/oracle/oracle.py) place it under the `oracle` directory. ```bash - wget - wget + wget https://raw.githubusercontent.com/Deepak-Bhuvaneswaran/plugins/refs/heads/deepakbhuvaneswaran/dropwizard/dropwizard.py + wget https://raw.githubusercontent.com/Deepak-Bhuvaneswaran/plugins/refs/heads/deepakbhuvaneswaran/dropwizard/dropwizard.cfg ``` ### Execute the plugin From 927911933aaa4ca75d503c0ef0500c325ae91e9d Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 16:53:08 +0530 Subject: [PATCH 12/17] Update README.md --- dropwizard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index f9569f1e..cc8bf909 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -34,7 +34,7 @@ If you're not using Linux servers or want to install the plugin manually, follow cd dropwizard/ ``` - - Download the below files [oracle.cfg](https://github.com/site24x7/plugins/blob/master/oracle/oracle.cfg) and [oracle.py](https://github.com/site24x7/plugins/blob/master/oracle/oracle.py) place it under the `oracle` directory. + - Download the below files [dropwizard.cfg](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.cfg) and [dropwizard.py](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.py) place it under the `dropwizard` directory. ```bash wget https://raw.githubusercontent.com/Deepak-Bhuvaneswaran/plugins/refs/heads/deepakbhuvaneswaran/dropwizard/dropwizard.py From e647583d859fc90972fc49e50c58cfb0a1af6be5 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:19:19 +0530 Subject: [PATCH 13/17] Update README.md --- dropwizard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index cc8bf909..d233528b 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -10,7 +10,7 @@ To quickly test the plugin against a local Dropwizard app: ```bash -python3 dropwiz.py –host localhost –port 8081 +python3 dropwizard.py –host localhost –port 8081 ``` From 0e402fe7e1cef54fb188d61ab1416cdfa1f97b57 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:27:30 +0530 Subject: [PATCH 14/17] Update README.md --- dropwizard/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index d233528b..d6a53193 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -33,7 +33,7 @@ If you're not using Linux servers or want to install the plugin manually, follow mkdir dropwizard cd dropwizard/ ``` - +### Download Plugin Script - Download the below files [dropwizard.cfg](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.cfg) and [dropwizard.py](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.py) place it under the `dropwizard` directory. ```bash From b2c6327161cbf7572591e1fd96b2a89661c2740f Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:28:20 +0530 Subject: [PATCH 15/17] Update README.md --- dropwizard/README.md | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index d6a53193..9ee13cea 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -220,35 +220,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | Active Dispatches | Active dispatch handlers in the servlet context (count) | | Active Suspended | Active suspended (async) requests (count) | -## **Units mapping (as used in the plugin)** -| Name | Unit | -| ----- | ----- | -| Connections at 8080 / 8081 / 8443 / 8444 | connections | -| JVM Uptime | ms | -| Threads Count / Threads Runnable Count | units | -| File Descriptor Ratio | ratio | -| Heap Used / Heap Max / Non-Heap Used / Non-Heap Max / Max Memory / Used Memory / Memory Total Committed / Metaspace Used / Compressed Class Space Used / Code Cache Used / G1 pool metrics | MB | - -## **Why these metrics matter (short)** - -* **Total Requests & per-method counts** — measure throughput and traffic composition; useful to detect traffic spikes or drops. - -* **Active Connections** — indicates current client concurrency on Jetty connectors; helps spot connection saturation. - -* **Log counts & response codes** — detect errors and client issues quickly (rising 4xx/5xx or ERROR logs). - -* **JVM / Memory / GC** — detect memory leaks, GC pressure, increased pause times, OOM risk. - -* **Jetty threadpool & queue** — detect thread starvation or queue buildup (slow downstream, blocked requests). - ---- - -## **Gauges vs Meters vs Timers (concise)** - -* **Gauges**: instantaneous values (e.g., heap used, thread counts). - -* **Meters**: measures of throughput/rate (e.g., counts that imply a rate). * **Timers**: measure duration distributions — in this explicit-mapping plugin we collect timer counts; percentiles are not emitted except if you extend mappings. From edef5f291992df8e55cb0cccb4a75fa15ba35cb8 Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:28:40 +0530 Subject: [PATCH 16/17] Update README.md --- dropwizard/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index 9ee13cea..02c8b626 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -220,9 +220,5 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | Active Dispatches | Active dispatch handlers in the servlet context (count) | | Active Suspended | Active suspended (async) requests (count) | - - -* **Timers**: measure duration distributions — in this explicit-mapping plugin we collect timer counts; percentiles are not emitted except if you extend mappings. - --- From 5b4a4954b19b17da943fd225202337b03299fa9a Mon Sep 17 00:00:00 2001 From: Deepak Bhuvaneswaran <151528030+Deepak-Bhuvaneswaran@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:57:11 +0530 Subject: [PATCH 17/17] Update README.md --- dropwizard/README.md | 112 ++++++++++++------------------------------- 1 file changed, 30 insertions(+), 82 deletions(-) diff --git a/dropwizard/README.md b/dropwizard/README.md index 02c8b626..fd8e28c5 100644 --- a/dropwizard/README.md +++ b/dropwizard/README.md @@ -4,23 +4,6 @@ - Download and install the latest version of the Site24x7 agent on the server where you plan to run the plugin. - Python 3 must be installed. -## Quick installation - -To quickly test the plugin against a local Dropwizard app: - -```bash - -python3 dropwizard.py –host localhost –port 8081 - -``` - -To test the raw metrics endpoint from the agent host: - -```bash - -curl \-sS http://localhost:8081/metrics | head \-n 2 - -``` ## Standard Installation If you're not using Linux servers or want to install the plugin manually, follow the steps below. @@ -34,100 +17,60 @@ If you're not using Linux servers or want to install the plugin manually, follow cd dropwizard/ ``` ### Download Plugin Script - - Download the below files [dropwizard.cfg](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.cfg) and [dropwizard.py](https://github.com/site24x7/plugins/blob/master/dropwizard/dropwizard.py) place it under the `dropwizard` directory. + - Download the below files and place it under the `dropwizard` directory. ```bash - wget https://raw.githubusercontent.com/Deepak-Bhuvaneswaran/plugins/refs/heads/deepakbhuvaneswaran/dropwizard/dropwizard.py - wget https://raw.githubusercontent.com/Deepak-Bhuvaneswaran/plugins/refs/heads/deepakbhuvaneswaran/dropwizard/dropwizard.cfg + wget https://raw.githubusercontent.com/site24x7/plugins/master/dropwizard/dropwizard.py && sed -i "1s|^.*|#! $(which python3)|" dropwizard.py + wget https://raw.githubusercontent.com/site24x7/plugins/master/dropwizard/dropwizard.cfg ``` ### Execute the plugin - Execute the below command with appropriate arguments to check for the valid json output: ```bash - python3 /opt/site24x7/monagent/plugins/dropwiz/dropwiz.py –host \ –port \ + python3 dropwizard.py –-host "localhost" -–port "8080" ``` -### Move Plugin to Agent Directory - - #### Linux - -- Place the `dropwizard` under the Site24x7 Linux Agent plugin directory: - - ```bash - mv dropwizard /opt/site24x7/monagent/plugins - ``` - -#### Windows - -- Since it's a Python plugin, to run the plugin in a Windows server please follow the steps in [this link](https://support.site24x7.com/portal/en/kb/articles/run-python-plugin-scripts-in-windows-servers). The remaining configuration steps are the same. - -- Further, move the folder `dropwizard` into the Site24x7 Windows Agent plugin directory: - - C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins\dropwizard - - -The agent will automatically execute the plugin within five minutes and send performance data to the Site24x7 data center. - --- - ## **Configuration (dropwizard.cfg)** -Put dropwizard.cfg beside dropwizard.py (or edit values via CLI). ```bash [dropwizard] - protocol = http host = localhost -port = 8081 +port = 8080 timeout = 30 ``` - -* protocol — http or https. - -* host — hostname or IP for the Dropwizard app. - -* port — port where /metrics is served. - -* timeout — seconds to wait for the /metrics response before failing. - -CLI overrides config file: \--host, \--port, \--protocol, \--timeout. - --- -## **How to install (Linux agent)** - -1. Create plugin directory on the agent host (if not present) and copy files: - -```bash - -sudo mkdir \-p /opt/site24x7/monagent/plugins/dropwizard - -sudo cp dropwizard.py dropwizard.cfg /opt/site24x7/monagent/plugins/dropwizard/ - -sudo chmod \+x /opt/site24x7/monagent/plugins/dropwizard/dropwizard.py +### Move Plugin to Agent Directory + + #### Linux -``` +- Place the `dropwizard` folder under the Site24x7 Linux Agent plugins directory: + + ```bash + mv dropwizard /opt/site24x7/monagent/plugins + ``` + +#### Windows -2. Test locally: +- Since it's a Python plugin, to run the plugin in a Windows server please follow the steps in [this link](https://support.site24x7.com/portal/en/kb/articles/run-python-plugin-scripts-in-windows-servers). The remaining configuration steps are the same. -```bash +- Further, move the folder `dropwizard` into the Site24x7 Windows Agent plugin directory: -python3 /opt/site24x7/monagent/plugins/dropwizard/dropwizard.py –host \ –port \ + C:\Program Files (x86)\Site24x7\WinAgent\monitoring\Plugins -``` -3. The Site24x7 agent will pick up the plugin output automatically (typically within 5 minutes). If not, check agent logs and file permissions. +The agent will automatically execute the plugin within five minutes and send performance data to the Site24x7 data center. --- ## Supported Metrics -Below are the metrics grouped exactly as the plugin uses them. Each table uses the Name | Description format so you can copy/paste into the repo. - -### **Summary tab** +### **Summary** | Name | Description | | ----- | ----- | @@ -135,7 +78,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | HealthCheck Pool Terminated | Health check thread-pool ‘terminated’ events (count) | | collected_at | The UNIX timestamp (in seconds) indicating when the Dropwizard metrics were collected by the plugin. It helps correlate data collection time and detect delays in polling or transmission | -### **Connection tab** +### **Connection** | Name | Description | | ----- | ----- | @@ -149,7 +92,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | Connections at 8443 | Active Jetty connections on port 8443 (connections) | | Connections at 8444 | Active Jetty connections on port 8444 (connections) | -### **Events tab** +### **Events** | Name | Description | | ----- | ----- | @@ -165,7 +108,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | 4xx Responses | Count of 4xx client error HTTP responses | | 5xx Responses | Count of 5xx server error HTTP responses | -### **JVM tab** +### **JVM** | Name | Description | | ----- | ----- | @@ -184,7 +127,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t --- -### **Memory tab** +### **Memory** | Name | Description | | ----- | ----- | @@ -202,7 +145,7 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t | G1 Old Gen Used | G1 Old Gen pool used (MB) | | G1 Survivor Space Used | G1 Survivor pool used (MB) | -### **Jetty tab** +### **Jetty** | Name | Description | | ----- | ----- | @@ -222,3 +165,8 @@ Below are the metrics grouped exactly as the plugin uses them. Each table uses t --- +### Sample Image +Screenshot 2025-10-08 at 5 49 32 PM + + +