From 0a98192ed239419540f3f7505753389ad1e17475 Mon Sep 17 00:00:00 2001 From: Damian Zaremba Date: Thu, 23 Jul 2026 13:19:03 +0200 Subject: [PATCH 1/3] getCbData - refactor logic - Gate on page metadata - Split queries into seperate functions - Use common function for exception/metric handling --- cbng.php | 12 +- mysql_functions.php | 422 +++++++++++++++++++------------------------- 2 files changed, 189 insertions(+), 245 deletions(-) diff --git a/cbng.php b/cbng.php index 2aeb435..ffafc4e 100644 --- a/cbng.php +++ b/cbng.php @@ -116,17 +116,19 @@ function parseFeedData($feedData) } $cutoff_timestamp = $feedData['timestamp'] - (14 * 86400); - $cb = getCbData( + $cb = ReplicaDb::getCbData( $feedData['user'], $feedData['namespaceid'], $feedData['title'], $cutoff_timestamp ); if ( - !(isset($cb['user_edit_count']) - and isset($cb['user_distinct_pages']) - and isset($cb['user_warns']) - and isset($cb['user_reg_time'])) + $cb == null or !( + isset($cb['user_edit_count']) + and isset($cb['user_distinct_pages']) + and isset($cb['user_warns']) + and isset($cb['user_reg_time']) + ) ) { $logger->warning( "Failed to get user info", diff --git a/mysql_functions.php b/mysql_functions.php index 24fb78c..44266d8 100644 --- a/mysql_functions.php +++ b/mysql_functions.php @@ -22,82 +22,97 @@ * You should have received a copy of the GNU General Public License * along with ClueBot NG. If not, see . */ -function connect_to_mysql($exclude_users = []) + +class ReplicaDb { - global $logger; + private static function connect($exclude_users = []) + { + global $logger; - if (count(Config::$mw_mysql_credentials) > 0) { - $candidate_credentials = array_filter( - Config::$mw_mysql_credentials, - fn($cred) => !in_array($cred['user'], $exclude_users, true) - ); - if (empty($candidate_credentials)) { + if (count(Config::$mw_mysql_credentials) > 0) { + $candidate_credentials = array_filter( + Config::$mw_mysql_credentials, + fn($cred) => !in_array($cred['user'], $exclude_users, true) + ); + if (empty($candidate_credentials)) { + $logger->error("ran out of database credentials"); + Metrics::increment('bot_mysql_mw_credentials_exhausted_total'); + die(); + } + $selected = $candidate_credentials[array_rand($candidate_credentials)]; + $mw_mysql_user = $selected['user']; + $mw_mysql_pass = $selected['pass']; + } elseif (!empty($exclude_users)) { $logger->error("ran out of database credentials"); - Metrics::increment('bot_mysql_mw_credentials_exhausted_total'); die(); + } else { + $mw_mysql_user = Config::$mw_mysql_user; + $mw_mysql_pass = Config::$mw_mysql_pass; } - $selected = $candidate_credentials[array_rand($candidate_credentials)]; - $mw_mysql_user = $selected['user']; - $mw_mysql_pass = $selected['pass']; - } elseif (!empty($exclude_users)) { - $logger->error("ran out of database credentials"); - die(); - } else { - $mw_mysql_user = Config::$mw_mysql_user; - $mw_mysql_pass = Config::$mw_mysql_pass; - } - try { - $mw_mysql = mysqli_init(); - $mw_mysql->options(MYSQLI_OPT_CONNECT_TIMEOUT, 10); - $mw_mysql->real_connect( - Config::$mw_mysql_host, - $mw_mysql_user, - $mw_mysql_pass, - Config::$mw_mysql_db, - Config::$mw_mysql_port - ); - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1226) { - $logger->debug("ran out of database connections for " . $mw_mysql_user); - Metrics::increment('bot_mysql_mw_credential_conn_limit_total', [$mw_mysql_user]); - Metrics::increment('bot_mysql_mw_connection_retries_total'); - usleep(5000); - $exclude_users[] = $mw_mysql_user; - return connect_to_mysql($exclude_users); + try { + $mw_mysql = mysqli_init(); + $mw_mysql->options(MYSQLI_OPT_CONNECT_TIMEOUT, 10); + $mw_mysql->real_connect( + Config::$mw_mysql_host, + $mw_mysql_user, + $mw_mysql_pass, + Config::$mw_mysql_db, + Config::$mw_mysql_port + ); + } catch (mysqli_sql_exception $e) { + if ($e->getCode() == 1226) { + $logger->debug("ran out of database connections for " . $mw_mysql_user); + Metrics::increment('bot_mysql_mw_credential_conn_limit_total', [$mw_mysql_user]); + Metrics::increment('bot_mysql_mw_connection_retries_total'); + usleep(5000); + $exclude_users[] = $mw_mysql_user; + return self::connect($exclude_users); + } + die('replica mysql error: ' . $e->getMessage()); } - die('replica mysql error: ' . $e->getMessage()); - } - if (!$mw_mysql) { - die('replica mysql error: ' . mysqli_connect_error()); + if (!$mw_mysql) { + die('replica mysql error: ' . mysqli_connect_error()); + } + + mysqli_select_db($mw_mysql, Config::$mw_mysql_db); + return $mw_mysql; } - mysqli_select_db($mw_mysql, Config::$mw_mysql_db); - return $mw_mysql; -} + private static function runQuery($mw_mysql, string $identifier, string $context, string $sql) + { + global $logger; + try { + $res = mysqli_query($mw_mysql, $sql); + if ($res === false) { + $logger->warning("$identifier query returned no data for $context: " . mysqli_error($mw_mysql)); + Metrics::increment('bot_mysql_mw_query_failures_total', [$identifier, 'no_data']); + return null; + } + return $res; + } catch (mysqli_sql_exception $e) { + if ($e->getCode() == 1969) { + $logger->warning("$identifier query timed out for $context"); + Metrics::increment('bot_mysql_mw_query_failures_total', [$identifier, 'timeout']); + } else { + $logger->error("$identifier query returned an error for $context: " . $e->getMessage()); + Metrics::increment('bot_mysql_mw_query_failures_total', [$identifier, 'error']); + } + return null; + } + } -function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') -{ - global $logger; - $mw_mysql = connect_to_mysql(); - $userPage = str_replace(' ', '_', $user); - $title = str_replace(' ', '_', $title); - $data = [ - 'common' => [ - 'creator' => null, - 'page_made_time' => null, - 'num_recent_edits' => null, - 'num_recent_reversions' => null, - ], - 'user_reg_time' => null, - 'user_warns' => null, - 'user_edit_count' => null, - 'user_distinct_pages' => null, - ]; + private static function parseMwTimestamp(string $timestamp): int + { + return \DateTime::createFromFormat('YmdHis', $timestamp, new \DateTimeZone('UTC'))->getTimestamp(); + } - try { - $res = mysqli_query( + private static function getPageMetadata($mw_mysql, $nsid, $title) + { + $res = self::runQuery( $mw_mysql, + 'page_metadata', + "$title ($nsid)", 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT `rev_timestamp`, `actor_name` FROM `revision`' . ' JOIN `actor_revision` ON `actor_id` = `rev_actor`' . @@ -110,31 +125,22 @@ function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') ' `page_title` = "' . mysqli_real_escape_string($mw_mysql, $title) . '"' . ')' ); - if ($res === false) { - $logger->warning("page metadata query returned no data for " . $title . - " (" . $nsid . "): " . mysqli_error($mw_mysql)); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_metadata', 'no_data']); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['common']['page_made_time'] = $d['rev_timestamp']; - $data['common']['creator'] = $d['actor_name']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("page metadata query timed out for " . $title . " (" . $nsid . ")"); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_metadata', 'timeout']); - } else { - $logger->error("page metadata query returned an error for " . $title . - " (" . $nsid . "): " . $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_metadata', 'error']); + $d = $res !== null ? mysqli_fetch_assoc($res) : null; + if ($d === null) { + return ['page_made_time' => null, 'creator' => null]; } + return [ + 'page_made_time' => self::parseMwTimestamp($d['rev_timestamp']), + 'creator' => $d['actor_name'], + ]; } - try { - $res = mysqli_query( + private static function getPageRecentEdits($mw_mysql, $nsid, $title, $timestamp) + { + $res = self::runQuery( $mw_mysql, + 'page_recent_edits', + "$title ($nsid) > $timestamp", 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT COUNT(*) as count FROM `page`' . ' JOIN `revision` ON `rev_page` = `page_id`' . @@ -145,31 +151,19 @@ function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') '" AND `rev_timestamp` > "' . mysqli_real_escape_string($mw_mysql, gmdate('YmdHis', $timestamp)) . '"' ); - if ($res === false) { - $logger->warning("page recent edits query returned no data for " . $title . - " (" . $nsid . ") > " . $timestamp . ": " . mysqli_error($mw_mysql)); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_edits', 'no_data']); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['common']['num_recent_edits'] = $d['count']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("page recent edits query timed out for " . $title . " (" . $nsid . ") > " . - $timestamp); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_edits', 'timeout']); - } else { - $logger->error("page recent edits query returned an error for " . $title . - " (" . $nsid . ") > " . $timestamp . ": " . $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_edits', 'error']); + if ($res === null) { + return null; } + $d = mysqli_fetch_assoc($res); + return $d !== null ? $d['count'] : null; } - try { - $res = mysqli_query( + private static function getPageRecentReverts($mw_mysql, $nsid, $title, $timestamp) + { + $res = self::runQuery( $mw_mysql, + 'page_recent_reverts', + "$title ($nsid) > $timestamp", 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT COUNT(*) as count FROM `page`' . ' JOIN `revision` ON `rev_page` = `page_id`' . @@ -182,104 +176,58 @@ function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') mysqli_real_escape_string($mw_mysql, gmdate('YmdHis', $timestamp)) . "' AND `comment_text` LIKE 'Revert%'" ); - - if ($res === false) { - $logger->warning("page recent reverts query returned no data for " . $title . - " (" . $nsid . ") > " . $timestamp . ": " . mysqli_error($mw_mysql)); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_reverts', 'no_data']); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['common']['num_recent_reversions'] = $d['count']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("page recent reverts query timed out for " . $title . - " (" . $nsid . ") > " . $timestamp); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_reverts', 'timeout']); - } else { - $logger->error("page recent reverts query returned an error for " . $title . - " (" . $nsid . ") > " . $timestamp . ": " . $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['page_recent_reverts', 'error']); + if ($res === null) { + return null; } + $d = mysqli_fetch_assoc($res); + return $d !== null ? $d['count'] : null; } - try { - $res = mysqli_query( + private static function getUserRegistration($mw_mysql, $user) + { + $res = self::runQuery( $mw_mysql, + 'user_registration', + $user, 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT `user_registration`, `user_editcount` FROM `user` WHERE `user_name` = "' . mysqli_real_escape_string($mw_mysql, $user) . '"' ); - - if ($res === false) { - $logger->warning("user registration query returned no data for " . - $user . ": " . mysqli_error($mw_mysql)); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_registration', 'no_data']); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['user_reg_time'] = $d['user_registration']; - $data['user_edit_count'] = $d['user_editcount']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("user registration query timed out for " . $user); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_registration', 'timeout']); - } else { - $logger->error("user registration query returned an error for " . $user . ": " . - $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_registration', 'error']); + $d = $res !== null ? mysqli_fetch_assoc($res) : null; + if ($d === null) { + return ['user_reg_time' => null, 'user_edit_count' => null]; } + return [ + 'user_reg_time' => $d['user_registration'] ? self::parseMwTimestamp($d['user_registration']) : null, + 'user_edit_count' => $d['user_editcount'], + ]; } - if (!$data['user_reg_time']) { - try { - $res = mysqli_query( - $mw_mysql, - 'SET STATEMENT max_statement_time=120 FOR ' . - 'SELECT `rev_timestamp` FROM `revision_userindex` ' . - ' JOIN `actor_revision` ON `actor_id` = `rev_actor`' . - ' WHERE `actor_name` = "' . - mysqli_real_escape_string($mw_mysql, $user) . '" ORDER BY `rev_timestamp` LIMIT 0,1' - ); - - if ($res === false) { - $logger->warning("user registration via revision query returned no data for " . - $user . ": " . mysqli_error($mw_mysql)); - Metrics::increment( - 'bot_mysql_mw_query_failures_total', - ['user_registration_via_revision', 'no_data'] - ); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['user_reg_time'] = $d['rev_timestamp']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("user registration via revision query timed out for " . $user); - Metrics::increment( - 'bot_mysql_mw_query_failures_total', - ['user_registration_via_revision', 'timeout'] - ); - } else { - $logger->error("user registration via revision query returned an error for " . $user . - ": " . $e->getMessage()); - Metrics::increment( - 'bot_mysql_mw_query_failures_total', - ['user_registration_via_revision', 'error'] - ); - } + private static function getUserRegistrationViaRevision($mw_mysql, $user) + { + $res = self::runQuery( + $mw_mysql, + 'user_registration_via_revision', + $user, + 'SET STATEMENT max_statement_time=120 FOR ' . + 'SELECT `rev_timestamp` FROM `revision_userindex` ' . + ' JOIN `actor_revision` ON `actor_id` = `rev_actor`' . + ' WHERE `actor_name` = "' . + mysqli_real_escape_string($mw_mysql, $user) . '" ORDER BY `rev_timestamp` LIMIT 0,1' + ); + if ($res === null) { + return null; } + $d = mysqli_fetch_assoc($res); + return $d !== null ? self::parseMwTimestamp($d['rev_timestamp']) : null; } - try { - $res = mysqli_query( + private static function getUserWarningsCount($mw_mysql, $userPage) + { + $res = self::runQuery( $mw_mysql, + 'user_warnings_count', + $userPage, 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT COUNT(*) as count FROM `page`' . ' JOIN `revision` ON `rev_page` = `page_id`' . @@ -289,68 +237,62 @@ function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') "' AND (`comment_text` LIKE '%warning%' OR `comment_text`" . " LIKE 'General note: Nonconstructive%')" ); - - if ($res === false) { - $logger->warning("user warnings count query returned no data for " . - $userPage . ": " . mysqli_error($mw_mysql)); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_warnings_count', 'no_data']); - } else { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['user_warns'] = $d['count']; - } - } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("user warnings count query timed out for " . $user); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_warnings_count', 'timeout']); - } else { - $logger->error("user warnings count query returned an error for " . $user . ": " . - $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_warnings_count', 'error']); + if ($res === null) { + return null; } + $d = mysqli_fetch_assoc($res); + return $d !== null ? $d['count'] : null; } - try { - $res = mysqli_query( + private static function getUserDistinctPages($mw_mysql, $userPage) + { + $res = self::runQuery( $mw_mysql, + 'user_distinct_pages', + $userPage, 'SET STATEMENT max_statement_time=120 FOR ' . 'SELECT count(distinct rev_page) AS count FROM' . ' `revision_userindex` JOIN `actor_revision` ON `actor_id` = `rev_actor`' . " WHERE `actor_name` = '" . mysqli_real_escape_string($mw_mysql, $userPage) . "'" ); + if ($res === null) { + return null; + } + $d = mysqli_fetch_assoc($res); + return $d !== null ? $d['count'] : null; + } - if ($res !== false) { - $d = mysqli_fetch_assoc($res); - if ($d !== null) { - $data['user_distinct_pages'] = $d['count']; - } + public static function getCbData($user = '', $nsid = '', $title = '', $timestamp = '') + { + $mw_mysql = self::connect(); + $userPage = str_replace(' ', '_', $user); + $title = str_replace(' ', '_', $title); + + $pageMetadata = self::getPageMetadata($mw_mysql, $nsid, $title); + if ($pageMetadata['page_made_time'] === null) { + mysqli_close($mw_mysql); + return null; } - } catch (mysqli_sql_exception $e) { - if ($e->getCode() == 1969) { - $logger->warning("user distinct page edits query timed out for " . $userPage); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_distinct_pages', 'timeout']); - } else { - $logger->error("user distinct page edits query returned an error for " . $userPage . - ": " . $e->getMessage()); - Metrics::increment('bot_mysql_mw_query_failures_total', ['user_distinct_pages', 'error']); + + $userRegistration = self::getUserRegistration($mw_mysql, $user); + if ($userRegistration['user_reg_time'] === null) { + $userRegistration['user_reg_time'] = self::getUserRegistrationViaRevision($mw_mysql, $user); } - } - if ($data['common']['page_made_time']) { - $data['common']['page_made_time'] = \DateTime::createFromFormat( - 'YmdHis', - $data['common']['page_made_time'], - new \DateTimeZone('UTC') - )->getTimestamp(); - } - if ($data['user_reg_time']) { - $data['user_reg_time'] = \DateTime::createFromFormat( - 'YmdHis', - $data['user_reg_time'], - new \DateTimeZone('UTC') - )->getTimestamp(); - } - mysqli_close($mw_mysql); - return $data; + $data = [ + 'common' => [ + 'creator' => $pageMetadata['creator'], + 'page_made_time' => $pageMetadata['page_made_time'], + 'num_recent_edits' => self::getPageRecentEdits($mw_mysql, $nsid, $title, $timestamp), + 'num_recent_reversions' => self::getPageRecentReverts($mw_mysql, $nsid, $title, $timestamp), + ], + 'user_reg_time' => $userRegistration['user_reg_time'], + 'user_warns' => self::getUserWarningsCount($mw_mysql, $userPage), + 'user_edit_count' => $userRegistration['user_edit_count'], + 'user_distinct_pages' => self::getUserDistinctPages($mw_mysql, $userPage), + ]; + + mysqli_close($mw_mysql); + return $data; + } } From 45488ebc94f4b19f8b4509345f3f706bb2e19ca0 Mon Sep 17 00:00:00 2001 From: Damian Zaremba Date: Thu, 23 Jul 2026 14:39:12 +0200 Subject: [PATCH 2/3] ReplicaDb - add total queries metric --- metric_functions.php | 16 +++++++++++++++- mysql_functions.php | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/metric_functions.php b/metric_functions.php index c3aa504..83353f3 100644 --- a/metric_functions.php +++ b/metric_functions.php @@ -139,6 +139,20 @@ public static function init($seedMetrics = true): void ['friends_account'], ] ); + self::registerCounter( + 'bot_mysql_mw_query_total', + 'Total replica MySQL queries', + ['query'], + [ + ['page_metadata'], + ['page_recent_edits'], + ['page_recent_reverts'], + ['user_registration'], + ['user_registration_via_revision'], + ['user_warnings_count'], + ['user_distinct_pages'], + ] + ); self::registerCounter( 'bot_mysql_mw_query_failures_total', 'Total replica MySQL query failures', @@ -158,7 +172,7 @@ public static function init($seedMetrics = true): void ['user_registration', 'error'], ['user_registration_via_revision', 'no_data'], ['user_registration_via_revision', 'timeout'], - ['user_registration_via_revision'], + ['user_registration_via_revision', 'error'], ['user_warnings_count', 'no_data'], ['user_warnings_count', 'timeout'], ['user_warnings_count', 'error'], diff --git a/mysql_functions.php b/mysql_functions.php index 44266d8..f84538e 100644 --- a/mysql_functions.php +++ b/mysql_functions.php @@ -89,6 +89,7 @@ private static function runQuery($mw_mysql, string $identifier, string $context, Metrics::increment('bot_mysql_mw_query_failures_total', [$identifier, 'no_data']); return null; } + Metrics::increment('bot_mysql_mw_query_total', [$identifier]); return $res; } catch (mysqli_sql_exception $e) { if ($e->getCode() == 1969) { From cdcc73c2aa08acc70f19673658e64971419fceb0 Mon Sep 17 00:00:00 2001 From: Damian Zaremba Date: Thu, 23 Jul 2026 23:24:02 +0100 Subject: [PATCH 3/3] fetchRevisionData - increase wait --- cbng.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbng.php b/cbng.php index ffafc4e..880e01b 100644 --- a/cbng.php +++ b/cbng.php @@ -58,7 +58,7 @@ function fetchRevisionData($url) } // Try again after a short wait - hopefully the change has replicated - sleep(1); + sleep(10); } return $page;