From d1d0d102a32216fb29d162f2183b81cb2185f076 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:21:05 +0800 Subject: [PATCH 01/14] feat: migrate blocking executors to virtual threads --- hertzbeat-alerter/pom.xml | 5 + .../hertzbeat/alert/AlerterWorkerPool.java | 120 ++++- .../periodic/PeriodicAlertRuleScheduler.java | 163 ++++++- .../realtime/window/AlarmEvaluator.java | 62 ++- .../alert/notice/AlertNoticeDispatch.java | 32 +- .../alert/reduce/AlarmCommonReduce.java | 67 +-- .../alert/reduce/AlarmGroupReduce.java | 157 ++++++- .../alert/reduce/AlarmInhibitReduce.java | 166 ++++++- .../alert/AlerterWorkerPoolTest.java | 176 +++++-- .../PeriodicAlertRuleSchedulerTest.java | 202 +++++++++ .../realtime/window/AlarmEvaluatorTest.java | 105 +++++ .../alert/notice/AlertNoticeDispatchTest.java | 31 ++ .../alert/reduce/AlarmCommonReduceTest.java | 79 +++- .../alert/reduce/AlarmGroupReduceTest.java | 118 ++++- .../alert/reduce/AlarmInhibitReduceTest.java | 124 ++++- .../collector/dispatch/WorkerPoolTest.java | 70 ++- .../dispatch/entrance/CollectServerTest.java | 73 +++ .../collector/dispatch/CommonDispatcher.java | 4 +- .../src/main/resources/application.yml | 8 + .../hertzbeat-collector-common/pom.xml | 4 + .../common/cache/GlobalConnectionCache.java | 116 ++++- .../collect/common/http/CommonHttpClient.java | 131 +++++- .../collector/dispatch/WorkerPool.java | 74 ++- .../dispatch/entrance/CollectServer.java | 125 ++++- .../cache/GlobalConnectionCacheTest.java | 175 +++++++ .../CommonHttpClientVirtualThreadTest.java | 121 +++++ .../hertzbeat-collector-rocketmq/pom.xml | 8 +- .../rocketmq/RocketmqSingleCollectImpl.java | 43 +- .../rocketmq/RocketmqSingleCollectTest.java | 62 +++ .../common/concurrent/AdmissionMode.java | 39 ++ .../common/concurrent/ManagedExecutor.java | 36 ++ .../common/concurrent/ManagedExecutors.java | 428 ++++++++++++++++++ .../concurrent/ManagedExecutorsTest.java | 184 ++++++++ .../hertzbeat/common/config/CommonConfig.java | 2 +- .../config/VirtualThreadProperties.java | 168 +++++++ .../common/support/CommonThreadPool.java | 66 ++- .../common/support/CommonThreadPoolTest.java | 121 +++-- .../hertzbeat/log/notice/LogSseManager.java | 10 +- .../log/notice/LogSseManagerTest.java | 33 +- .../component/sd/ServiceDiscoveryWorker.java | 2 +- .../component/status/CalculateStatus.java | 428 ++++++++++++------ .../manager/scheduler/ManagerWorkerPool.java | 74 ++- .../manager/scheduler/netty/ManageServer.java | 123 ++++- .../component/status/CalculateStatusTest.java | 148 ++++++ .../scheduler/ManagerWorkerPoolTest.java | 105 +++++ .../scheduler/netty/ManageServerTest.java | 149 ++++++ .../remoting/netty/NettyRemotingClient.java | 2 +- .../remoting/netty/NettyRemotingServer.java | 2 +- .../apache/hertzbeat/startup/AsyncConfig.java | 41 ++ .../src/main/resources/application-test.yml | 31 ++ .../src/main/resources/application.yml | 31 ++ .../hertzbeat/startup/AsyncConfigTest.java | 83 ++++ .../warehouse/WarehouseWorkerPool.java | 72 ++- .../warehouse/store/DataStorageDispatch.java | 4 +- .../history/tsdb/doris/DorisDataStorage.java | 2 +- .../duckdb/DuckdbDatabaseDataStorage.java | 224 ++++++--- .../warehouse/WarehouseWorkerPoolTest.java | 67 ++- .../duckdb/DuckdbDatabaseDataStorageTest.java | 148 ++++++ 58 files changed, 4849 insertions(+), 595 deletions(-) create mode 100644 hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java create mode 100644 hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java diff --git a/hertzbeat-alerter/pom.xml b/hertzbeat-alerter/pom.xml index 7af87836e9d..c3a3890d2c1 100644 --- a/hertzbeat-alerter/pom.xml +++ b/hertzbeat-alerter/pom.xml @@ -40,6 +40,11 @@ hertzbeat-common-core provided + + org.apache.hertzbeat + hertzbeat-common-spring + provided + org.apache.hertzbeat diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java index 23426ddcf06..b6b743b8a2e 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java @@ -18,12 +18,20 @@ package org.apache.hertzbeat.alert; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,16 +39,25 @@ */ @Component @Slf4j -public class AlerterWorkerPool { +public class AlerterWorkerPool implements DisposableBean { private ThreadPoolExecutor workerExecutor; - private ThreadPoolExecutor notifyExecutor; - private ThreadPoolExecutor logWorkerExecutor; + private ManagedExecutor notifyExecutor; + private ManagedExecutor logWorkerExecutor; + private Map notifyChannelPermits; + private int notifyMaxConcurrentPerChannel; public AlerterWorkerPool() { + this(VirtualThreadProperties.defaults()); + } + + @Autowired + public AlerterWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; initWorkExecutor(); - initNotifyExecutor(); - initLogWorkerExecutor(); + initNotifyExecutor(properties); + initLogWorkerExecutor(properties); } private void initWorkExecutor() { @@ -61,16 +78,32 @@ private void initWorkExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initNotifyExecutor() { + private void initNotifyExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter notifyExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.AlerterProperties alerterProperties = properties.getAlerter(); + VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.getNotify(); + notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.getNotifyMaxConcurrentPerChannel()); + notifyChannelPermits = new ConcurrentHashMap<>(8); + notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-", + notifyProperties.getMode(), notifyProperties.getMaxConcurrentJobs(), handler); + return; + } + notifyMaxConcurrentPerChannel = 0; + notifyChannelPermits = null; + notifyExecutor = ManagedExecutors.wrap("notify-worker", createLegacyNotifyExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyNotifyExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter notifyExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("notify-worker-%d") .build(); - notifyExecutor = new ThreadPoolExecutor(6, + return new ThreadPoolExecutor(6, 6, 10, TimeUnit.SECONDS, @@ -79,16 +112,27 @@ private void initNotifyExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initLogWorkerExecutor() { + private void initLogWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter logWorkerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties logWorkerProperties = properties.getAlerter().getLogWorker(); + logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-", + logWorkerProperties.getMaxConcurrentJobs(), logWorkerProperties.getQueueCapacity(), handler); + return; + } + logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyLogWorkerExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter logWorkerExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("log-worker-%d") .build(); - logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, + return new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1000), threadFactory, new ThreadPoolExecutor.AbortPolicy()); @@ -113,6 +157,41 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { notifyExecutor.execute(runnable); } + /** + * Executes the given runnable task using the notify executor with per-channel concurrency control. + * + * @param channelType notification channel type + * @param runnable the task to be executed + * @throws RejectedExecutionException if the task cannot be accepted for execution + */ + public void executeNotify(byte channelType, Runnable runnable) throws RejectedExecutionException { + if (notifyChannelPermits == null) { + notifyExecutor.execute(runnable); + return; + } + Semaphore semaphore = notifyChannelPermits.computeIfAbsent(channelType, + key -> new Semaphore(notifyMaxConcurrentPerChannel)); + if (!semaphore.tryAcquire()) { + throw new RejectedExecutionException( + "notify-worker rejected task because channel concurrency limit was reached for type " + channelType); + } + boolean submitted = false; + try { + notifyExecutor.execute(() -> { + try { + runnable.run(); + } finally { + semaphore.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + semaphore.release(); + } + } + } + /** * Executes the given runnable task using the logWorkerExecutor. * @@ -122,4 +201,11 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { public void executeLogJob(Runnable runnable) throws RejectedExecutionException { logWorkerExecutor.execute(runnable); } + + @Override + public void destroy() { + workerExecutor.shutdownNow(); + notifyExecutor.close(); + logWorkerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 978fb3d234a..9ad94c4d4dd 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -25,12 +25,19 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; @@ -40,18 +47,33 @@ */ @Slf4j @Component -public class PeriodicAlertRuleScheduler implements CommandLineRunner { +public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean { private final MetricsPeriodicAlertCalculator metricsCalculator; private final LogPeriodicAlertCalculator logCalculator; private final AlertDefineDao alertDefineDao; private final ScheduledExecutorService scheduledExecutor; - private final Map> scheduledFutures; + private final ExecutorService periodicExecutor; + private final Semaphore periodicPermits; + private final boolean virtualThreadsEnabled; + private final Map scheduledTasks; public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, LogPeriodicAlertCalculator logCalculator, AlertDefineDao alertDefineDao) { + this(metricsCalculator, logCalculator, alertDefineDao, VirtualThreadProperties.defaults()); + } + + @Autowired + public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, + LogPeriodicAlertCalculator logCalculator, + AlertDefineDao alertDefineDao, + VirtualThreadProperties virtualThreadProperties) { this.metricsCalculator = metricsCalculator; this.logCalculator = logCalculator; this.alertDefineDao = alertDefineDao; + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Scheduled periodic alert threshold has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled periodic alert threshold has uncaughtException."); @@ -61,17 +83,27 @@ public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculat .setNameFormat("periodic-alert-threshold-worker-%d") .build(); this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); - this.scheduledFutures = new ConcurrentHashMap<>(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.virtualThreadsEnabled = properties.isEnabled(); + int maxConcurrentPeriodicTasks = Math.max(1, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + this.periodicExecutor = virtualThreadsEnabled + ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("periodic-alert-task-", 0) + .uncaughtExceptionHandler(handler) + .factory()) + : null; + this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null; + this.scheduledTasks = new ConcurrentHashMap<>(); } public void cancelSchedule(Long ruleId) { if (ruleId == null) { return; } - ScheduledFuture future = scheduledFutures.get(ruleId); - if (future != null) { - future.cancel(true); - scheduledFutures.remove(ruleId); + ScheduledTaskState state = scheduledTasks.remove(ruleId); + if (state != null) { + state.cancel(); } } @@ -83,14 +115,12 @@ public void updateSchedule(AlertDefine rule) { cancelSchedule(rule.getId()); if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) || rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate(() -> { - if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { - metricsCalculator.calculate(rule); - } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - logCalculator.calculate(rule); - } - }, 0, rule.getPeriod(), java.util.concurrent.TimeUnit.SECONDS); - scheduledFutures.put(rule.getId(), future); + ScheduledTaskState state = new ScheduledTaskState(rule); + ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate( + virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), + 0, rule.getPeriod(), TimeUnit.SECONDS); + state.setScheduledFuture(future); + scheduledTasks.put(rule.getId(), state); } } @@ -106,4 +136,107 @@ public void run(String... args) throws Exception { updateSchedule(rule); } } + + @Override + public void destroy() { + scheduledTasks.values().forEach(ScheduledTaskState::cancel); + scheduledTasks.clear(); + scheduledExecutor.shutdownNow(); + if (periodicExecutor != null) { + periodicExecutor.shutdownNow(); + } + } + + private void executeRule(AlertDefine rule) { + if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { + metricsCalculator.calculate(rule); + } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { + logCalculator.calculate(rule); + } + } + + private final class ScheduledTaskState { + + private final AlertDefine rule; + private ScheduledFuture scheduledFuture; + private Future runningFuture; + private boolean running; + private boolean pending; + private boolean cancelled; + + private ScheduledTaskState(AlertDefine rule) { + this.rule = rule; + } + + private synchronized void setScheduledFuture(ScheduledFuture scheduledFuture) { + this.scheduledFuture = scheduledFuture; + } + + private synchronized void trigger() { + if (cancelled) { + return; + } + if (running) { + pending = true; + return; + } + running = true; + submitLocked(); + } + + private synchronized void cancel() { + cancelled = true; + pending = false; + ScheduledFuture periodicFuture = scheduledFuture; + Future currentFuture = runningFuture; + if (periodicFuture != null) { + periodicFuture.cancel(true); + } + if (currentFuture != null) { + currentFuture.cancel(true); + } + } + + private void submitLocked() { + try { + runningFuture = periodicExecutor.submit(() -> { + boolean permitAcquired = false; + try { + periodicPermits.acquire(); + permitAcquired = true; + if (!Thread.currentThread().isInterrupted()) { + executeRule(rule); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); + } finally { + if (permitAcquired) { + periodicPermits.release(); + } + onComplete(); + } + }); + } catch (RuntimeException e) { + running = false; + throw e; + } + } + + private synchronized void onComplete() { + runningFuture = null; + if (cancelled) { + running = false; + pending = false; + return; + } + if (!pending) { + running = false; + return; + } + pending = false; + submitLocked(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java index 1e245631d4f..1d210ac47c8 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java @@ -20,6 +20,11 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.apache.hertzbeat.alert.util.AlertTemplateUtil; @@ -32,10 +37,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * Alarm Evaluator - Final alarm logic trigger @@ -47,36 +48,48 @@ */ @Component @Slf4j -public class AlarmEvaluator { +public class AlarmEvaluator implements DisposableBean { private static final String WINDOW_START_TIME = "window_start_time"; private static final String WINDOW_END_TIME = "window_end_time"; private static final String MATCHING_LOGS_COUNT = "matching_logs_count"; private final AlarmCommonReduce alarmCommonReduce; - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce) { + this(alarmCommonReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmCommonReduce = alarmCommonReduce; - initAlarmEvaluator(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initAlarmEvaluator(properties); } - public void initAlarmEvaluator() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getWindowEvaluator(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-", + queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2, 10, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } public void sendAndProcessWindowData(WindowAggregator.WindowData windowData) { @@ -314,4 +327,9 @@ private void addLogEntryToMap(LogEntry logEntry, Map context) { } } } -} \ No newline at end of file + + @Override + public void destroy() { + workerExecutor.close(); + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java index 6d9207216b7..b294e817cd8 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterWorkerPool; import org.apache.hertzbeat.alert.config.AlertSseManager; @@ -121,14 +122,27 @@ public void dispatchAlarm(GroupAlert groupAlert) { } private void sendNotify(GroupAlert alert) { - matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> workerPool.executeNotify(() -> rule.getReceiverId() - .forEach(receiverId -> { - try { - sendNoticeMsg(getOneReceiverById(receiverId), - getOneTemplateById(rule.getTemplateId()), alert); - } catch (AlertNoticeException e) { - log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); - } - })))); + matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> { + NoticeTemplate noticeTemplate = getOneTemplateById(rule.getTemplateId()); + rule.getReceiverId().forEach(receiverId -> { + NoticeReceiver receiver = getOneReceiverById(receiverId); + if (receiver == null || receiver.getType() == null) { + log.warn("DispatchTask skip invalid receiver, receiverId: {}, alertId: {}", receiverId, alert.getId()); + return; + } + try { + workerPool.executeNotify(receiver.getType(), () -> { + try { + sendNoticeMsg(receiver, noticeTemplate, alert); + } catch (AlertNoticeException e) { + log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); + } + }); + } catch (RejectedExecutionException e) { + log.warn("DispatchTask rejected notify task, receiverId: {}, type: {}, message: {}", + receiverId, receiver.getType(), e.getMessage()); + } + }); + })); } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 0ef336921ac..4d6b2edb0e5 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -18,16 +18,16 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; /** @@ -35,33 +35,45 @@ */ @Service @Slf4j -public class AlarmCommonReduce { +public class AlarmCommonReduce implements DisposableBean { private final AlarmGroupReduce alarmGroupReduce; - - private ThreadPoolExecutor workerExecutor; + + private final ManagedExecutor workerExecutor; public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) { - initWorkExecutor(); + this(alarmGroupReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmGroupReduce = alarmGroupReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initWorkExecutor(properties); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getReduce(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-", + queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2, 2, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } @@ -103,10 +115,6 @@ Runnable reduceAlarmTask(SingleAlert alert) { * Fingerprint is based on labels excluding timestamp related fields */ private String generateAlertFingerprint(Map labels) { - // Remove timestamp related fields - labels.remove("timestamp"); - labels.remove("start_at"); - labels.remove("active_at"); return labels.entrySet().stream() .filter(e -> !"timestamp".equals(e.getKey()) && !"starts_at".equals(e.getKey()) && !"actives_at".equals(e.getKey()) @@ -116,4 +124,9 @@ private String generateAlertFingerprint(Map labels) { .map(e -> e.getKey() + ":" + e.getValue()) .collect(Collectors.joining(",")); } + + @Override + public void destroy() { + workerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 94c6cb873e2..e43b2e43bcb 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -26,17 +26,22 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -45,7 +50,7 @@ */ @Component @Slf4j -public class AlarmGroupReduce { +public class AlarmGroupReduce implements DisposableBean { /** * Default initial group wait time 30s @@ -88,16 +93,60 @@ public class AlarmGroupReduce { */ private final Map groupCacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService workerExecutor; + + private final ScheduledDispatchTask checkTask; + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) { + this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties) { + this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true); + } + + AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.alarmInhibitReduce = alarmInhibitReduce; this.groupDefines = new ConcurrentHashMap<>(8); this.groupCacheMap = new ConcurrentHashMap<>(8); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.scheduledExecutor = createScheduler(); + this.workerExecutor = createVirtualExecutor(properties); + this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups); List groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue(); refreshGroupDefines(groupConverges); - startCheckAndSendGroups(); + if (autoStart) { + startCheckAndSendGroups(); + } } private void startCheckAndSendGroups() { + scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCheckAndSendGroups() { + checkTask.dispatch(); + } + + void beforeCheckAndSendGroupsRun() { + } + + @Override + public void destroy() { + scheduledExecutor.shutdownNow(); + if (workerExecutor != null) { + workerExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Check alarm groups calculate has uncaughtException."); @@ -106,21 +155,36 @@ private void startCheckAndSendGroups() { .setDaemon(true) .setNameFormat("alarm-group-calculate-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - long now = System.currentTimeMillis(); - groupCacheMap.forEach((groupKey, cache) -> { - if (shouldSendGroup(cache, now)) { - sendGroupAlert(cache); - cache.setLastSendTime(now); - cache.getAlertFingerprints().clear(); - } - }); - } catch (Exception e) { - log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); - } - }, 10000, CHECK_INTERVAL, java.util.concurrent.TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) { + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("alarm-group-calculate-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Check alarm groups calculate worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCheckAndSendGroups() { + beforeCheckAndSendGroupsRun(); + try { + long now = System.currentTimeMillis(); + groupCacheMap.forEach((groupKey, cache) -> { + if (shouldSendGroup(cache, now)) { + sendGroupAlert(cache); + cache.setLastSendTime(now); + cache.getAlertFingerprints().clear(); + } + }); + } catch (Exception e) { + log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); + } } /** @@ -339,4 +403,63 @@ private static class GroupAlertCache { private long lastSendTime; private long lastRepeatTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index 08cdf7b3b78..c8a51392692 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -18,25 +18,28 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.Collections; -import java.util.stream.Collectors; - import lombok.Data; import lombok.AllArgsConstructor; @@ -46,7 +49,7 @@ */ @Component @Slf4j -public class AlarmInhibitReduce { +public class AlarmInhibitReduce implements DisposableBean { /** * Interval for checking and cleaning up expired source alerts @@ -73,22 +76,69 @@ public class AlarmInhibitReduce { /** * Default TTL for source alerts (4 hours) */ - private static long SOURCE_ALERT_TTL = 4 * 60 * 60 * 1000L; + private final long sourceAlertTtl; + + private final ScheduledExecutorService cleanupScheduler; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao , AlerterProperties alerterProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true); + } + + AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties, + boolean autoStart) { this.alarmSilenceReduce = alarmSilenceReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) { - SOURCE_ALERT_TTL = alerterProperties.getInhibit().getTtl(); + this.sourceAlertTtl = alerterProperties.getInhibit().getTtl(); + } else { + this.sourceAlertTtl = 4 * 60 * 60 * 1000L; } inhibitRules = new ConcurrentHashMap<>(8); sourceAlertCache = new ConcurrentHashMap<>(8); + this.cleanupScheduler = createCleanupScheduler(); + this.cleanupExecutor = createCleanupExecutor(properties); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache); List inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue(); refreshInhibitRules(inhibits); - startScheduledCleanupCache(); + if (autoStart) { + startScheduledCleanupCache(); + } } private void startScheduledCleanupCache() { + cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanupCacheRun() { + } + + @Override + public void destroy() { + cleanupScheduler.shutdownNow(); + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createCleanupScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled clean up inhibit cache has uncaughtException."); @@ -97,17 +147,30 @@ private void startScheduledCleanupCache() { .setDaemon(true) .setNameFormat("inhibit-clean-up-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - // Scheduled cleanup of all expired source alerts - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - sourceAlertCache.values().forEach(this::cleanupExpiredEntries); - // Remove empty rule caches - sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); - } catch (Exception e) { - log.error("Error during scheduled cleanup", e); - } - }, CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) { + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("inhibit-clean-up-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Scheduled clean up inhibit cache worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCleanupCache() { + beforeCleanupCacheRun(); + try { + sourceAlertCache.values().forEach(this::cleanupExpiredEntries); + sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + } catch (Exception e) { + log.error("Error during scheduled cleanup", e); + } } /** @@ -264,7 +327,7 @@ private void cacheSourceAlert(SingleAlert alert, AlertInhibit rule) { SourceAlertEntry entry = new SourceAlertEntry( alert, System.currentTimeMillis(), - System.currentTimeMillis() + SOURCE_ALERT_TTL + System.currentTimeMillis() + sourceAlertTtl ); ruleCache.put(alert.getFingerprint(), entry); cleanupExpiredEntries(ruleCache); @@ -315,4 +378,63 @@ private static class SourceAlertEntry { private final long createTime; private final long expiryTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java index 444b85d79dc..434a999eb65 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java @@ -18,9 +18,18 @@ package org.apache.hertzbeat.alert; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.BeforeEach; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -28,60 +37,159 @@ */ class AlerterWorkerPoolTest { - private static final int NUMBER_OF_THREADS = 10; + private static final int NUMBER_OF_TASKS = 10; + private AlerterWorkerPool pool; - private AtomicInteger counter; - private CountDownLatch latch; - @BeforeEach - void setUp() { - pool = new AlerterWorkerPool(); - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); + @AfterEach + void tearDown() { + if (pool != null) { + pool.destroy(); + } } @Test void executeJob() throws InterruptedException { - for (int i = 0; i < NUMBER_OF_THREADS; i++) { + pool = new AlerterWorkerPool(); + AtomicInteger counter = new AtomicInteger(); + CountDownLatch latch = new CountDownLatch(NUMBER_OF_TASKS); + + for (int i = 0; i < NUMBER_OF_TASKS; i++) { pool.executeJob(() -> { counter.incrementAndGet(); latch.countDown(); }); } - latch.await(); - assertEquals(NUMBER_OF_THREADS, counter.get()); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals(NUMBER_OF_TASKS, counter.get()); } @Test - void executeNotify() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeNotify(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); + void executeNotifyRunsOnVirtualThread() throws Exception { + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeNotify((byte) 1, () -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); + notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + notifyProperties.setMaxConcurrentJobs(1); + alerterProperties.setNotify(notifyProperties); + alerterProperties.setNotifyMaxConcurrentPerChannel(8); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 2, () -> { + })); + } finally { + release.countDown(); } - latch.await(); + } + + @Test + void executeNotifyRejectsWhenChannelLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); + notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + notifyProperties.setMaxConcurrentJobs(8); + alerterProperties.setNotify(notifyProperties); + alerterProperties.setNotifyMaxConcurrentPerChannel(1); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertEquals(NUMBER_OF_THREADS, counter.get()); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 1, () -> { + })); + } finally { + release.countDown(); + } } @Test void executeLogJob() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeLogJob(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); - } - latch.await(); + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeLogJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } - assertEquals(NUMBER_OF_THREADS, counter.get()); + @Test + void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.QueueProperties logWorkerProperties = new VirtualThreadProperties.QueueProperties(); + logWorkerProperties.setMaxConcurrentJobs(1); + logWorkerProperties.setQueueCapacity(1); + alerterProperties.setLogWorker(logWorkerProperties); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + pool.executeLogJob(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + pool.executeLogJob(secondStarted::countDown); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeLogJob(() -> { + })); + } finally { + releaseFirst.countDown(); + } + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java new file mode 100644 index 00000000000..a21f66fc2a8 --- /dev/null +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.calculate.periodic; + +import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link PeriodicAlertRuleScheduler}. + */ +@ExtendWith(MockitoExtension.class) +class PeriodicAlertRuleSchedulerTest { + + @Mock + private MetricsPeriodicAlertCalculator metricsCalculator; + + @Mock + private LogPeriodicAlertCalculator logCalculator; + + @Mock + private AlertDefineDao alertDefineDao; + + private PeriodicAlertRuleScheduler scheduler; + + @BeforeEach + void setUp() { + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + VirtualThreadProperties.defaults()); + } + + @AfterEach + void tearDown() { + if (scheduler != null) { + scheduler.destroy(); + } + } + + @Test + void updateScheduleRunsPeriodicCalculationOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(1L); + scheduler.updateSchedule(rule); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void updateScheduleKeepsSingleInFlightExecutionPerRule() throws InterruptedException { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + int count = invocations.incrementAndGet(); + try { + if (count == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (count == 2) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(2L)); + + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + Thread.sleep(1200L); + assertEquals(1, maxConcurrent.get()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void cancelScheduleInterruptsRunningVirtualTask() throws InterruptedException { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + doAnswer(invocation -> { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(3L); + scheduler.updateSchedule(rule); + + assertTrue(started.await(5, TimeUnit.SECONDS)); + scheduler.cancelSchedule(rule.getId()); + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + } + + @Test + void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException { + scheduler.destroy(); + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + periodicProperties(1)); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + AlertDefine rule = invocation.getArgument(0); + try { + if (rule.getId().equals(4L)) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (rule.getId().equals(5L)) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(4L)); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + scheduler.updateSchedule(metricRule(5L)); + Thread.sleep(200L); + assertEquals(1, maxConcurrent.get()); + assertEquals(1L, secondStarted.getCount()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private AlertDefine metricRule(Long id) { + return AlertDefine.builder() + .id(id) + .name("periodic-rule-" + id) + .type(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) + .period(1) + .enable(true) + .build(); + } + + private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { + VirtualThreadProperties properties = VirtualThreadProperties.defaults(); + properties.getAlerter().setPeriodicMaxConcurrentJobs(maxConcurrentJobs); + return properties; + } +} diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java index e27cb833f32..394b9b77d02 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java @@ -19,9 +19,11 @@ import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.apache.hertzbeat.common.entity.log.LogEntry; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,14 +34,20 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -98,6 +106,13 @@ void setUp() { windowData.addMatchingLog(matchingEvent); } + @AfterEach + void tearDown() { + if (alarmEvaluator != null) { + alarmEvaluator.destroy(); + } + } + @Test void testProcessWindowDataWithIndividualMode() throws InterruptedException { // Given - alert define with individual mode @@ -343,4 +358,94 @@ void testMultipleMatchingLogsInWindow() throws InterruptedException { assertEquals(2, alerts.size()); assertEquals(2, alerts.get(0).getTriggerTimes()); // Each alert should have trigger times = total count } + + @Test + void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + alertDefine.setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + alarmEvaluator.sendAndProcessWindowData(windowData); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); + queueProperties.setMaxConcurrentJobs(1); + properties.getAlerter().setWindowEvaluator(queueProperties); + alarmEvaluator.destroy(); + alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + WindowAggregator.WindowData firstWindow = cloneWindowDataWithBody("first"); + WindowAggregator.WindowData secondWindow = cloneWindowDataWithBody("second"); + firstWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + secondWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + + alarmEvaluator.sendAndProcessWindowData(firstWindow); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmEvaluator.sendAndProcessWindowData(secondWindow); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + + private WindowAggregator.WindowData cloneWindowDataWithBody(String body) { + LogEntry logEntry = LogEntry.builder() + .timeUnixNano(System.currentTimeMillis() * 1_000_000L) + .severityText("ERROR") + .body(body) + .build(); + AlertDefine define = AlertDefine.builder() + .id(alertDefine.getId()) + .name(alertDefine.getName()) + .type(alertDefine.getType()) + .expr(alertDefine.getExpr()) + .times(alertDefine.getTimes()) + .template(alertDefine.getTemplate()) + .labels(alertDefine.getLabels()) + .annotations(alertDefine.getAnnotations()) + .enable(alertDefine.isEnable()) + .build(); + MatchingLogEvent event = MatchingLogEvent.builder() + .logEntry(logEntry) + .alertDefine(define) + .eventTimestamp(System.currentTimeMillis()) + .workerTimestamp(System.currentTimeMillis()) + .build(); + WindowAggregator.WindowData clonedWindowData = new WindowAggregator.WindowData( + new WindowAggregator.WindowKey(define.getId(), System.currentTimeMillis() - 60000, System.currentTimeMillis()), + define); + clonedWindowData.addMatchingLog(event); + return clonedWindowData; + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java index 8102afcdc0b..ff4c92ef4cb 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java @@ -19,7 +19,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyByte; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -149,4 +152,32 @@ void testSendNoticeMsgNoHandler() { assertFalse(alertNoticeDispatch.sendNoticeMsg(receiver, null, alert)); } + + @Test + void testDispatchAlarmUsesTypedNotifyExecution() { + NoticeTemplate template = new NoticeTemplate(); + template.setId(1L); + template.setName("default-template"); + + when(alertStoreHandler.store(alert)).thenReturn(alert); + when(noticeConfigService.getReceiverFilterRule(alert)).thenReturn(Collections.singletonList( + org.apache.hertzbeat.common.entity.alerter.NoticeRule.builder() + .receiverId(Collections.singletonList(1L)) + .templateId(1L) + .build())); + when(noticeConfigService.getReceiverById(1L)).thenReturn(receiver); + when(noticeConfigService.getOneTemplateById(1L)).thenReturn(template); + doNothing().when(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(1); + task.run(); + return null; + }).when(workerPool).executeNotify(anyByte(), any(Runnable.class)); + + alertNoticeDispatch.dispatchAlarm(alert); + + verify(workerPool).executeNotify(eq((byte) 1), any(Runnable.class)); + verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + verify(emitterManager).broadcast(any(String.class)); + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index a98f36dfff8..8dae8999511 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -17,7 +17,20 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -40,14 +53,76 @@ class AlarmCommonReduceTest { @BeforeEach void setUp() { - - testAlert = SingleAlert.builder().build(); + testAlert = SingleAlert.builder().labels(new HashMap<>(Map.of("alertname", "test"))).build(); alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce); } + @AfterEach + void tearDown() { + if (alarmCommonReduce != null) { + alarmCommonReduce.destroy(); + } + } + @Test void testReduceAndSendAlarm() { alarmCommonReduce.reduceAndSendAlarm(testAlert); } + @Test + void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(testAlert); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); + queueProperties.setMaxConcurrentJobs(1); + properties.getAlerter().setReduce(queueProperties); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "first"))).build()); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "second"))).build()); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index a8e8dc05338..a042032cab4 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -38,6 +38,9 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; @@ -47,9 +50,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -73,7 +82,15 @@ void setUp() { MockitoAnnotations.openMocks(this); when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); - alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao); + alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmGroupReduce != null) { + alarmGroupReduce.destroy(); + } } @Test @@ -112,6 +129,42 @@ void whenMatchingGroupRule_shouldGroup() { verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait } + @Test + void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private Map createLabels(String... keyValues) { Map labels = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { @@ -119,4 +172,67 @@ private Map createLabels(String... keyValues) { } return labels; } + + private static final class TestAlarmGroupReduce extends AlarmGroupReduce { + + private final CountDownLatch virtualThreadLatch; + + private final AtomicBoolean virtualThread; + + private final CountDownLatch firstStarted; + + private final CountDownLatch releaseFirst; + + private final CountDownLatch secondStarted; + + private final AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent; + + private final AtomicInteger invocations; + + private TestAlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties properties, CountDownLatch virtualThreadLatch, + AtomicBoolean virtualThread, CountDownLatch firstStarted, + CountDownLatch releaseFirst, CountDownLatch secondStarted, + AtomicInteger maxConcurrent, AtomicInteger invocations) { + super(alarmInhibitReduce, alertGroupConvergeDao, properties, false); + this.virtualThreadLatch = virtualThreadLatch; + this.virtualThread = virtualThread; + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + this.invocations = invocations; + this.concurrent = maxConcurrent == null ? null : new AtomicInteger(); + } + + @Override + void beforeCheckAndSendGroupsRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null || invocations == null) { + return; + } + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1 && firstStarted != null && releaseFirst != null) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2 && secondStarted != null) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java index 18bc17eb578..7b264dffb7d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java @@ -39,6 +39,7 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -49,14 +50,19 @@ import java.util.Map; import java.util.ArrayList; import java.util.List; - +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -89,7 +95,15 @@ void setUp() { inhibitProperties.setTtl(60000); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmInhibitReduce != null) { + alarmInhibitReduce.destroy(); + } } @Test @@ -290,7 +304,9 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { AlerterProperties.InhibitProperties inhibitProperties = new AlerterProperties.InhibitProperties(); inhibitProperties.setTtl(100); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); AlertInhibit rule = AlertInhibit.builder() .id(1L) @@ -324,6 +340,42 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { verify(alarmSilenceReduce).silenceAlarm(targetGroupAlert); } + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmInhibitReduce.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmInhibitReduce.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmInhibitReduce.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private GroupAlert createGroupAlert(String status, Map labels, List alerts) { return GroupAlert.builder() .status(status) @@ -347,4 +399,68 @@ private SingleAlert createSingleAlert(String status, String fingerprint, Map { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - assertDoesNotThrow(() -> workerPool.executeJob(mockTask)); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); } @Test - void testExecuteJobThrowsException() { + void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties collectorProperties = new VirtualThreadProperties.PoolProperties(); + collectorProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + collectorProperties.setMaxConcurrentJobs(1); + properties.setCollector(collectorProperties); + workerPool = new WorkerPool(properties); - workerPool = mock(WorkerPool.class); - doThrow(new RejectedExecutionException()).when(workerPool).executeJob(mockTask); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + workerPool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(mockTask)); + try { + assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(() -> { + })); + } finally { + release.countDown(); + } } - - @Test - void testDestroy() { - assertDoesNotThrow(() -> workerPool.destroy()); - } - } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java index 3112e1e5baf..c56d53e948f 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java @@ -18,17 +18,25 @@ package org.apache.hertzbeat.collector.dispatch.entrance; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.netty.channel.Channel; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties; import org.apache.hertzbeat.collector.dispatch.DispatchProperties; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; @@ -142,4 +150,69 @@ void testOnChannelActive() { assertNotNull(scheduledExecutor); } + @Test + void testDispatchHeartbeatRunsOnVirtualThread() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testDispatchHeartbeatDoesNotRunConcurrently() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + collectServer.dispatchHeartbeat("collector1"); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private DispatchProperties properties() { + return properties; + } + } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java index c03709bad7f..f915f2ee4eb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java @@ -114,7 +114,7 @@ public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, public void start() { try { // Pull the collection task from the task queue and put it into the thread pool for execution - workerPool.executeJob(() -> { + workerPool.executeLongRunning(() -> { Thread.currentThread().setName("metrics-task-dispatcher"); while (!Thread.currentThread().isInterrupted()) { MetricsCollect metricsCollect = null; @@ -379,4 +379,4 @@ protected static class MetricsTime { private Metrics metrics; private Timeout timeout; } -} \ No newline at end of file +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index eb1b92e0de8..43057c241bf 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -75,3 +75,11 @@ push: common: queue: type: netty + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index d4cffe23fb7..b9a80815d3a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -51,6 +51,10 @@ org.apache.hertzbeat hertzbeat-common-core + + org.apache.hertzbeat + hertzbeat-common-spring + org.apache.sshd diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java index ed936372334..9b6699622b2 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java @@ -20,14 +20,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import lombok.extern.slf4j.Slf4j; - import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; /** * Singleton LRU global resource cache for client-server connections @@ -56,10 +57,20 @@ public class GlobalConnectionCache { */ private final ConcurrentLinkedHashMap> cacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; + /** * Private constructor to prevent instantiation */ private GlobalConnectionCache() { + this(true); + } + + GlobalConnectionCache(boolean autoStart) { cacheMap = new ConcurrentLinkedHashMap.Builder>() .maximumWeightedCapacity(Integer.MAX_VALUE) .listener((key, value) -> { @@ -72,7 +83,13 @@ private GlobalConnectionCache() { log.info("GlobalConnectionCache discarded key: {}, value: {}.", key, value); }) .build(); - initCacheMonitor(); + this.scheduledExecutor = createCacheMonitorScheduler(); + this.cleanupExecutor = createCleanupExecutor(); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanTimeoutOrUnHealthyCache); + if (autoStart) { + initCacheMonitor(); + Runtime.getRuntime().addShutdownHook(new Thread(this::destroy)); + } } /** @@ -95,18 +112,42 @@ public static GlobalConnectionCache getInstance() { * Initialize the cache monitor for cleaning up expired connections */ private void initCacheMonitor() { + scheduledExecutor.scheduleWithFixedDelay(this::dispatchCleanupCache, 2, 100, TimeUnit.SECONDS); + } + + /** + * Clean and remove timeout or unhealthy cache entries + */ + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanTimeoutOrUnHealthyCacheRun() { + } + + void destroy() { + scheduledExecutor.shutdownNow(); + cleanupExecutor.shutdownNow(); + } + + private ScheduledExecutorService createCacheMonitorScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("connection-cache-timeout-detector-%d") .setDaemon(true) .build(); - ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(this::cleanTimeoutOrUnHealthyCache, 2, 100, TimeUnit.SECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); } - /** - * Clean and remove timeout or unhealthy cache entries - */ - private void cleanTimeoutOrUnHealthyCache() { + private ExecutorService createCleanupExecutor() { + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("connection-cache-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("Connection cache cleanup has uncaughtException.", throwable)) + .factory()); + } + + private void runCleanTimeoutOrUnHealthyCache() { + beforeCleanTimeoutOrUnHealthyCacheRun(); try { cacheMap.forEach((key, value) -> { Long[] cacheTime = timeoutMap.get(key); @@ -204,4 +245,59 @@ public void removeCache(Object key) { log.error("Connection close error for key {}: {}", key, e.getMessage(), e); } } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java index 22cb0e8a1bd..54001cdae0d 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java @@ -22,6 +22,7 @@ import java.security.cert.CertificateExpiredException; import java.security.cert.X509Certificate; import java.util.Date; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -52,6 +53,14 @@ public class CommonHttpClient { private static PoolingHttpClientConnectionManager connectionManager; + private static ScheduledExecutorService scheduledExecutor; + + private static ExecutorService cleanupExecutor; + + private static ScheduledDispatchTask cleanupTask; + + private static volatile Runnable beforeCleanupHook; + /** * all max total connection */ @@ -137,15 +146,9 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr // clean up available but idle connections .evictIdleConnections(100, TimeUnit.SECONDS) .build(); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("http-connection-pool-cleaner-%d") - .setDaemon(true) - .build(); - ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(() -> { - connectionManager.closeExpiredConnections(); - connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); - }, 40L, 40L, TimeUnit.SECONDS); + initializeCleanupExecutors(); + scheduledExecutor.scheduleWithFixedDelay(CommonHttpClient::dispatchConnectionPoolCleanup, + 40L, 40L, TimeUnit.SECONDS); // shutdown hook Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close)); @@ -156,13 +159,119 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr public static CloseableHttpClient getHttpClient() { return httpClient; } + + public static PoolingHttpClientConnectionManager getConnectionManager() { + return connectionManager; + } + + static void dispatchConnectionPoolCleanup() { + if (cleanupTask != null) { + cleanupTask.dispatch(); + } + } + + static void setConnectionManagerForTest(PoolingHttpClientConnectionManager manager) { + connectionManager = manager; + } + + static void setBeforeCleanupHookForTest(Runnable hook) { + beforeCleanupHook = hook; + } public static void close() { try { - httpClient.close(); + if (httpClient != null) { + httpClient.close(); + } } catch (Exception e) { log.error("close http client error", e); } + if (scheduledExecutor != null) { + scheduledExecutor.shutdownNow(); + } + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private static void initializeCleanupExecutors() { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("http-connection-pool-cleaner-%d") + .setDaemon(true) + .build(); + scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); + cleanupExecutor = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("http-connection-pool-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("HTTP connection pool cleanup has uncaughtException.", throwable)) + .factory()); + cleanupTask = new ScheduledDispatchTask(cleanupExecutor, CommonHttpClient::runConnectionPoolCleanup); + } + + private static void runConnectionPoolCleanup() { + Runnable hook = beforeCleanupHook; + if (hook != null) { + hook.run(); + } + if (connectionManager == null) { + return; + } + connectionManager.closeExpiredConnections(); + connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } } - } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java index a11ff4cfe44..7ba179ce40d 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java @@ -18,15 +18,19 @@ package org.apache.hertzbeat.collector.dispatch; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.stereotype.Component; - +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * Collection task worker thread pool @@ -35,25 +39,55 @@ @Slf4j public class WorkerPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public WorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - // thread factory - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + @Autowired + public WorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getCollector(); + return ManagedExecutors.newVirtualExecutor("collector-worker", "collect-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("collector-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("collector-long-running", "collect-long-running-", + (thread, throwable) -> { + log.error("[Important] WorkerPool longRunningExecutor has uncaughtException.", throwable); log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("collect-worker-%d") .build(); int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors()); int maxSize = Runtime.getRuntime().availableProcessors() * 16; - workerExecutor = new ThreadPoolExecutor(coreSize, + return new ThreadPoolExecutor(coreSize, maxSize, 10, TimeUnit.SECONDS, @@ -72,10 +106,20 @@ public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + /** + * Run the long-lived dispatcher job outside of the collector admission limit. + * + * @param runnable dispatcher task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + @Override public void destroy() throws Exception { - if (workerExecutor != null) { - workerExecutor.shutdownNow(); + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); } } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 9b34a11d4bd..112209c8b76 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -32,6 +32,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProcessor; import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; @@ -40,11 +41,13 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -70,11 +73,29 @@ public class CollectServer implements CommandLineRunner { private ScheduledExecutorService scheduledExecutor; + private final ExecutorService heartbeatExecutor; + + private final Object heartbeatLock = new Object(); + + private boolean heartbeatRunning; + + private boolean heartbeatPending; + public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, final CommonThreadPool threadPool, final CollectorInfoProperties infoProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, VirtualThreadProperties.defaults()); + } + + @Autowired + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final CommonThreadPool threadPool, + final CollectorInfoProperties infoProperties, + final VirtualThreadProperties virtualThreadProperties) { if (properties == null || properties.getEntrance() == null || properties.getEntrance().getNetty() == null) { log.error("init error, please config dispatch entrance netty props in application.yml"); throw new IllegalArgumentException("please config dispatch entrance netty props"); @@ -87,6 +108,7 @@ public CollectServer(final CollectJobService collectJobService, this.timerDispatch = timerDispatch; this.collectJobService.setCollectServer(this); this.infoProperties = infoProperties; + this.heartbeatExecutor = createHeartbeatExecutor(virtualThreadProperties); this.init(properties, threadPool); } @@ -107,7 +129,12 @@ private void init(final DispatchProperties properties, final CommonThreadPool th } public void shutdown() { - this.scheduledExecutor.shutdownNow(); + if (this.scheduledExecutor != null) { + this.scheduledExecutor.shutdownNow(); + } + if (this.heartbeatExecutor != null) { + this.heartbeatExecutor.shutdownNow(); + } this.remotingClient.shutdown(); } @@ -120,6 +147,21 @@ public void sendMsg(final ClusterMsg.Message message) { this.remotingClient.sendMsg(message); } + void dispatchHeartbeat(String identity) { + if (heartbeatExecutor == null) { + sendHeartbeat(identity); + return; + } + synchronized (heartbeatLock) { + if (heartbeatRunning) { + heartbeatPending = true; + return; + } + heartbeatRunning = true; + } + submitHeartbeat(identity); + } + @Override public void run(String... args) throws Exception { this.remotingClient.start(); @@ -161,19 +203,8 @@ public void onChannelActive(Channel channel) { .build(); scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); // schedule send heartbeat message - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() - .setIdentity(identity) - .setDirection(ClusterMsg.Direction.REQUEST) - .setType(ClusterMsg.MessageType.HEARTBEAT) - .build(); - CollectServer.this.sendMsg(heartbeat); - log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); - } catch (Exception e) { - log.error("schedule send heartbeat to server error.{}", e.getMessage()); - } - }, 5, 5, TimeUnit.SECONDS); + scheduledExecutor.scheduleAtFixedRate(() -> CollectServer.this.dispatchHeartbeat(identity), + 5, 5, TimeUnit.SECONDS); } } @@ -182,4 +213,70 @@ public void onChannelIdle(Channel channel) { log.info("handle idle event triggered. collector is going offline."); } } + + private ExecutorService createHeartbeatExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("heartbeat-worker-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("HeartBeat worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void submitHeartbeat(String identity) { + boolean submitted = false; + try { + heartbeatExecutor.execute(() -> { + try { + sendHeartbeat(identity); + } finally { + onHeartbeatComplete(identity); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (heartbeatLock) { + heartbeatRunning = false; + heartbeatPending = false; + } + } + } + } + + private void onHeartbeatComplete(String identity) { + boolean shouldRunAgain; + synchronized (heartbeatLock) { + if (heartbeatPending) { + heartbeatPending = false; + shouldRunAgain = true; + } else { + heartbeatRunning = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submitHeartbeat(identity); + } + } + + private void sendHeartbeat(String identity) { + try { + ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() + .setIdentity(identity) + .setDirection(ClusterMsg.Direction.REQUEST) + .setType(ClusterMsg.MessageType.HEARTBEAT) + .build(); + CollectServer.this.sendMsg(heartbeat); + log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); + } catch (Exception e) { + log.error("schedule send heartbeat to server error.{}", e.getMessage()); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java new file mode 100644 index 00000000000..d066434d410 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link GlobalConnectionCache}. + */ +class GlobalConnectionCacheTest { + + private TestGlobalConnectionCache globalConnectionCache; + + @BeforeEach + void setUp() { + globalConnectionCache = new TestGlobalConnectionCache(); + } + + @AfterEach + void tearDown() { + if (globalConnectionCache != null) { + globalConnectionCache.destroy(); + } + } + + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + globalConnectionCache.setVirtualThreadHook(latch, virtualThread); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + globalConnectionCache.setConcurrencyHook(firstStarted, releaseFirst, secondStarted, maxConcurrent); + + globalConnectionCache.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + globalConnectionCache.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchCleanupCacheClosesExpiredConnections() throws Exception { + TestConnection connection = new TestConnection(); + globalConnectionCache.addCache("expired", connection, -1L); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(connection.closed.await(5, TimeUnit.SECONDS)); + assertEquals(1, connection.closeCount.get()); + assertTrue(globalConnectionCache.getCache("expired", false).isEmpty()); + } + + private static final class TestGlobalConnectionCache extends GlobalConnectionCache { + + private CountDownLatch virtualThreadLatch; + + private AtomicBoolean virtualThread; + + private CountDownLatch firstStarted; + + private CountDownLatch releaseFirst; + + private CountDownLatch secondStarted; + + private AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent = new AtomicInteger(); + + private final AtomicInteger invocations = new AtomicInteger(); + + private TestGlobalConnectionCache() { + super(false); + } + + private void setVirtualThreadHook(CountDownLatch latch, AtomicBoolean flag) { + this.virtualThreadLatch = latch; + this.virtualThread = flag; + } + + private void setConcurrencyHook(CountDownLatch firstStarted, CountDownLatch releaseFirst, + CountDownLatch secondStarted, AtomicInteger maxConcurrent) { + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + } + + @Override + void beforeCleanTimeoutOrUnHealthyCacheRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null) { + return; + } + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } + + private static final class TestConnection extends AbstractConnection { + + private final CountDownLatch closed = new CountDownLatch(1); + + private final AtomicInteger closeCount = new AtomicInteger(); + + @Override + public Object getConnection() { + return new Object(); + } + + @Override + public void closeConnection() { + closeCount.incrementAndGet(); + closed.countDown(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java new file mode 100644 index 00000000000..2ec28df4528 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for CommonHttpClient cleanup dispatch. + */ +class CommonHttpClientVirtualThreadTest { + + private final PoolingHttpClientConnectionManager originalConnectionManager = CommonHttpClient.getConnectionManager(); + + @AfterEach + void tearDown() { + CommonHttpClient.setBeforeCleanupHookForTest(null); + CommonHttpClient.setConnectionManagerForTest(originalConnectionManager); + } + + @Test + void dispatchConnectionPoolCleanupRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchConnectionPoolCleanupDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchConnectionPoolCleanupClosesExpiredAndIdleConnections() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(manager).closeExpiredConnections(); + CommonHttpClient.setConnectionManagerForTest(manager); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(manager, times(1)).closeExpiredConnections(); + verify(manager, times(1)).closeIdleConnections(40, TimeUnit.SECONDS); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml index ef3d9261cf3..92d05010fe1 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml @@ -28,12 +28,6 @@ hertzbeat-collector-rocketmq ${project.artifactId} - - 17 - 17 - UTF-8 - - org.apache.hertzbeat @@ -47,4 +41,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java index c8c3889dd85..a8077ca6091 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java @@ -19,7 +19,6 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -28,10 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -40,6 +35,8 @@ import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; import org.apache.hertzbeat.collector.util.JsonPathParser; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; @@ -55,7 +52,6 @@ import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.springframework.beans.factory.DisposableBean; @@ -68,10 +64,11 @@ public class RocketmqSingleCollectImpl extends AbstractCollect implements DisposableBean { private static final int WAIT_TIMEOUT = 10; + static final int QUEUE_CAPACITY = 5000; private static final Set SYSTEM_GROUP_SET = new HashSet<>(); - private final ExecutorService executorService; + private final ManagedExecutor executorService; static { // system consumer group @@ -86,24 +83,28 @@ public class RocketmqSingleCollectImpl extends AbstractCollect implements Dispos } public RocketmqSingleCollectImpl() { + this(createExecutor()); + } + + RocketmqSingleCollectImpl(ManagedExecutor executorService) { + this.executorService = executorService; + } + + private static ManagedExecutor createExecutor() { Runtime runtime = Runtime.getRuntime(); int corePoolSize = Math.max(8, runtime.availableProcessors()); int maximumPoolSize = Math.max(16, runtime.availableProcessors()); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("RocketMQCollectGroup has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("rocketMQ-collector-%d") - .build(); - this.executorService = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(5000), threadFactory, new ThreadPoolExecutor.DiscardOldestPolicy()); + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("RocketMQCollectGroup has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + return ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-collector", "rocketmq-collector-", + corePoolSize, maximumPoolSize, QUEUE_CAPACITY, handler); } @Override public void destroy() { - ThreadUtils.shutdownGracefully(this.executorService, 10L, TimeUnit.SECONDS); + this.executorService.close(); } /** @@ -270,7 +271,7 @@ private void collectConsumerData(DefaultMQAdminExt mqAdminExt, RocketmqCollectDa if (SYSTEM_GROUP_SET.contains(consumerGroup)) { continue; } - executorService.submit(() -> { + executeConsumerTask(() -> { RocketmqCollectData.ConsumerInfo consumerInfo = new RocketmqCollectData.ConsumerInfo(); consumerInfoList.add(consumerInfo); consumerInfo.setConsumerGroup(consumerGroup); @@ -369,4 +370,8 @@ private void fillBuilder(RocketmqCollectData rocketmqCollectData, CollectRep.Met builder.addValueRow(valueRowBuilder.build()); } } + + void executeConsumerTask(Runnable runnable) { + executorService.execute(runnable); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java index 9996e727276..b40ed2599d7 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java @@ -19,12 +19,20 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,6 +47,13 @@ public void setUp() throws Exception { collect = new RocketmqSingleCollectImpl(); } + @AfterEach + void tearDown() { + if (collect != null) { + collect.destroy(); + } + } + @Test void preCheck() { // metrics is null @@ -97,4 +112,51 @@ void collect() { void supportProtocol() { assertEquals(DispatchConstants.PROTOCOL_ROCKETMQ, collect.supportProtocol()); } + + @Test + void executeConsumerTaskRunsOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + collect.executeConsumerTask(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeConsumerTaskDiscardsOldestWhenQueueIsFull() throws InterruptedException { + ManagedExecutor executor = ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-test", + "rocketmq-test-", 1, 1, 1, (thread, throwable) -> { + }); + RocketmqSingleCollectImpl testCollect = new RocketmqSingleCollectImpl(executor); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch thirdStarted = new CountDownLatch(1); + AtomicBoolean secondExecuted = new AtomicBoolean(false); + try { + testCollect.executeConsumerTask(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + testCollect.executeConsumerTask(() -> secondExecuted.set(true)); + testCollect.executeConsumerTask(thirdStarted::countDown); + + releaseFirst.countDown(); + assertTrue(thirdStarted.await(5, TimeUnit.SECONDS)); + assertFalse(secondExecuted.get()); + } finally { + releaseFirst.countDown(); + testCollect.destroy(); + } + } } diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java new file mode 100644 index 00000000000..5f1a0e4fec2 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +/** + * Task admission mode for managed executors. + */ +public enum AdmissionMode { + + /** + * Start each task on its own virtual thread without an executor-level concurrency cap. + */ + UNBOUNDED_VT, + + /** + * Reject immediately when the configured concurrency cap has been reached. + */ + LIMIT_AND_REJECT, + + /** + * Block the submitter until a permit is available. + */ + LIMIT_AND_BLOCK +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java new file mode 100644 index 00000000000..c86d7a97d60 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.Executor; + +/** + * Executor with a stable logical name and close semantics. + */ +public interface ManagedExecutor extends Executor, AutoCloseable { + + /** + * Logical executor name for logging and metrics tags. + * + * @return executor name + */ + String name(); + + @Override + void close(); +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java new file mode 100644 index 00000000000..24226deadae --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.ArrayDeque; +import java.util.Objects; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Factory methods for managed executors. + */ +public final class ManagedExecutors { + + private ManagedExecutors() { + } + + /** + * Wrap an existing executor service. + * + * @param name executor name + * @param executorService delegate executor service + * @return managed executor wrapper + */ + public static ManagedExecutor wrap(String name, ExecutorService executorService) { + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a per-task virtual-thread executor with optional admission control. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param mode admission mode + * @param maxConcurrentTasks max concurrent tasks for limited modes + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newVirtualExecutor(String name, String threadNamePrefix, AdmissionMode mode, + int maxConcurrentTasks, Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, semaphore(mode, maxConcurrentTasks), mode); + } + + /** + * Create a platform-thread-per-task executor for a small number of long-running tasks. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newPlatformExecutor(String name, String threadNamePrefix, + Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a queued executor that preserves queue semantics while executing tasks on virtual threads. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity, {@code <= 0} means unbounded + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newQueuedVirtualExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for queued executors"); + } + return new QueuedVirtualManagedExecutor(name, threadNamePrefix, maxConcurrentTasks, queueCapacity, handler); + } + + /** + * Create a virtual-thread executor that preserves {@link java.util.concurrent.ThreadPoolExecutor} + * core/max/queue semantics with discard-oldest overflow handling. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param coreConcurrentTasks core concurrent tasks + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newDiscardOldestVirtualExecutor(String name, String threadNamePrefix, + int coreConcurrentTasks, int maxConcurrentTasks, + int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + if (coreConcurrentTasks <= 0) { + throw new IllegalArgumentException("coreConcurrentTasks must be greater than zero"); + } + if (maxConcurrentTasks < coreConcurrentTasks) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than or equal to coreConcurrentTasks"); + } + if (queueCapacity <= 0) { + throw new IllegalArgumentException("queueCapacity must be greater than zero"); + } + return new DiscardOldestVirtualManagedExecutor(name, threadNamePrefix, coreConcurrentTasks, + maxConcurrentTasks, queueCapacity, handler); + } + + private static Semaphore semaphore(AdmissionMode mode, int maxConcurrentTasks) { + if (mode == AdmissionMode.UNBOUNDED_VT) { + return null; + } + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for limited executors"); + } + return new Semaphore(maxConcurrentTasks); + } + + private static final class DefaultManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final Semaphore permits; + private final AdmissionMode admissionMode; + + private DefaultManagedExecutor(String name, ExecutorService delegate, Semaphore permits, + AdmissionMode admissionMode) { + this.name = Objects.requireNonNull(name, "name"); + this.delegate = Objects.requireNonNull(delegate, "delegate"); + this.permits = permits; + this.admissionMode = Objects.requireNonNull(admissionMode, "admissionMode"); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + acquirePermit(); + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + releasePermit(); + } + }); + submitted = true; + } finally { + if (!submitted) { + releasePermit(); + } + } + } + + @Override + public void close() { + delegate.shutdownNow(); + } + + private void acquirePermit() { + if (permits == null) { + return; + } + switch (admissionMode) { + case LIMIT_AND_REJECT: + if (!permits.tryAcquire()) { + throw new RejectedExecutionException(name + " rejected task because concurrency limit was reached"); + } + break; + case LIMIT_AND_BLOCK: + try { + permits.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException(name + " interrupted while waiting for an execution permit", e); + } + break; + case UNBOUNDED_VT: + break; + default: + throw new IllegalStateException("Unsupported admission mode: " + admissionMode); + } + } + + private void releasePermit() { + if (permits != null) { + permits.release(); + } + } + } + + private static final class QueuedVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final ExecutorService dispatcher; + private final BlockingDeque queue; + private final Semaphore permits; + private final Semaphore permitSignals; + private final AtomicBoolean closed; + + private QueuedVirtualManagedExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.queue = queueCapacity > 0 ? new LinkedBlockingDeque<>(queueCapacity) : new LinkedBlockingDeque<>(); + this.permits = new Semaphore(maxConcurrentTasks); + this.permitSignals = new Semaphore(0); + this.closed = new AtomicBoolean(false); + ThreadFactory dispatcherFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix + "dispatcher-", 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.dispatcher = Executors.newSingleThreadExecutor(dispatcherFactory); + this.dispatcher.execute(this::dispatchLoop); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + if (closed.get()) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (!queue.offerLast(command)) { + throw new RejectedExecutionException(name + " rejected task because queue capacity was reached"); + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + dispatcher.shutdownNow(); + delegate.shutdownNow(); + queue.clear(); + } + + private void dispatchLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + Runnable command = queue.takeFirst(); + if (!permits.tryAcquire()) { + queue.putFirst(command); + permitSignals.acquire(); + continue; + } + submit(command); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + permits.release(); + permitSignals.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + permits.release(); + permitSignals.release(); + } + } + } + } + + private static final class DiscardOldestVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final int coreConcurrentTasks; + private final int maxConcurrentTasks; + private final ArrayDeque queue; + private final int queueCapacity; + private final Object lock; + private boolean closed; + private int runningTasks; + + private DiscardOldestVirtualManagedExecutor(String name, String threadNamePrefix, int coreConcurrentTasks, + int maxConcurrentTasks, int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.coreConcurrentTasks = coreConcurrentTasks; + this.maxConcurrentTasks = maxConcurrentTasks; + this.queueCapacity = queueCapacity; + this.queue = new ArrayDeque<>(queueCapacity); + this.lock = new Object(); + this.closed = false; + this.runningTasks = 0; + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + Runnable taskToStart = null; + synchronized (lock) { + if (closed) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (runningTasks < coreConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else if (queue.size() < queueCapacity) { + queue.offerLast(command); + return; + } else if (runningTasks < maxConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else { + queue.pollFirst(); + queue.offerLast(command); + return; + } + } + submit(taskToStart); + } + + @Override + public void close() { + synchronized (lock) { + if (closed) { + return; + } + closed = true; + queue.clear(); + } + delegate.shutdownNow(); + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + onTaskComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + runningTasks--; + } + throw new RejectedExecutionException(name + " rejected task because delegate submission failed"); + } + } + } + + private void onTaskComplete() { + Runnable nextTask = null; + synchronized (lock) { + if (closed) { + runningTasks--; + return; + } + nextTask = queue.pollFirst(); + if (nextTask == null) { + runningTasks--; + return; + } + } + submit(nextTask); + } + } +} diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java new file mode 100644 index 00000000000..9f104b5e5fe --- /dev/null +++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ManagedExecutors}. + */ +class ManagedExecutorsTest { + + @Test + void shouldRunTaskOnVirtualThread() throws Exception { + ManagedExecutor executor = ManagedExecutors.newVirtualExecutor("test", "test-vt-", + AdmissionMode.UNBOUNDED_VT, 0, (thread, throwable) -> { + }); + try { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + executor.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } finally { + executor.close(); + } + } + + @Test + void shouldRejectTaskWhenAdmissionLimitReached() throws Exception { + ManagedExecutor executor = ManagedExecutors.newVirtualExecutor("limited", "limited-vt-", + AdmissionMode.LIMIT_AND_REJECT, 1, (thread, throwable) -> { + }); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + executor.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { + })); + } finally { + release.countDown(); + executor.close(); + } + } + + @Test + void shouldQueueTasksWhileKeepingVirtualThreadExecution() throws Exception { + ManagedExecutor executor = ManagedExecutors.newQueuedVirtualExecutor("queued", "queued-vt-", + 1, 0, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicBoolean firstVirtual = new AtomicBoolean(false); + AtomicBoolean secondVirtual = new AtomicBoolean(false); + try { + executor.execute(() -> { + firstVirtual.set(Thread.currentThread().isVirtual()); + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(() -> { + secondVirtual.set(Thread.currentThread().isVirtual()); + secondStarted.countDown(); + }); + + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertTrue(firstVirtual.get()); + assertTrue(secondVirtual.get()); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } + + @Test + void shouldRejectTaskWhenQueuedExecutorCapacityReached() throws Exception { + ManagedExecutor executor = ManagedExecutors.newQueuedVirtualExecutor("queued", "queued-vt-", + 1, 1, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + try { + executor.execute(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(secondStarted::countDown); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { + })); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } + + @Test + void shouldDiscardOldestTaskWhenDiscardOldestExecutorQueueIsFull() throws Exception { + ManagedExecutor executor = ManagedExecutors.newDiscardOldestVirtualExecutor("discard-oldest", + "discard-oldest-vt-", 1, 1, 1, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch thirdStarted = new CountDownLatch(1); + AtomicBoolean secondExecuted = new AtomicBoolean(false); + AtomicBoolean thirdVirtual = new AtomicBoolean(false); + try { + executor.execute(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(() -> secondExecuted.set(true)); + executor.execute(() -> { + thirdVirtual.set(Thread.currentThread().isVirtual()); + thirdStarted.countDown(); + }); + + releaseFirst.countDown(); + assertTrue(thirdStarted.await(5, TimeUnit.SECONDS)); + assertFalse(secondExecuted.get()); + assertTrue(thirdVirtual.get()); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java index 7eb3c680731..7adea1c05ac 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java @@ -31,6 +31,6 @@ @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG + SignConstants.DOT + ConfigConstants.FunctionModuleConstants.COMMON) -@EnableConfigurationProperties(CommonProperties.class) +@EnableConfigurationProperties({CommonProperties.class, VirtualThreadProperties.class}) public class CommonConfig { } diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java new file mode 100644 index 00000000000..f8590dee3ac --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.config; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Virtual-thread related configuration. + */ +@Getter +@Setter +@ConfigurationProperties(prefix = "hertzbeat.vthreads") +public class VirtualThreadProperties { + + private boolean enabled = true; + + private PoolProperties collector = PoolProperties.collectorDefaults(); + + private PoolProperties common = PoolProperties.commonDefaults(); + + private PoolProperties manager = PoolProperties.managerDefaults(); + + private AlerterProperties alerter = new AlerterProperties(); + + private PoolProperties warehouse = PoolProperties.warehouseDefaults(); + + private AsyncProperties async = new AsyncProperties(); + + /** + * Create a detached properties instance with runtime defaults. + * + * @return defaults instance + */ + public static VirtualThreadProperties defaults() { + return new VirtualThreadProperties(); + } + + /** + * Pool-level configuration. + */ + @Getter + @Setter + public static class PoolProperties { + + private AdmissionMode mode = AdmissionMode.UNBOUNDED_VT; + + private int maxConcurrentJobs; + + private static PoolProperties collectorDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(defaultCollectorConcurrency()); + return properties; + } + + private static PoolProperties warehouseDefaults() { + return new PoolProperties(); + } + + private static PoolProperties commonDefaults() { + return new PoolProperties(); + } + + private static PoolProperties managerDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(10); + return properties; + } + + private static PoolProperties alerterNotifyDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(64); + return properties; + } + + private static int defaultCollectorConcurrency() { + int historicalMax = Runtime.getRuntime().availableProcessors() * 16; + return Math.max(1, historicalMax - 1); + } + } + + /** + * Alerter-specific executor configuration. + */ + @Getter + @Setter + public static class AlerterProperties { + + private PoolProperties notify = PoolProperties.alerterNotifyDefaults(); + + private int periodicMaxConcurrentJobs = 10; + + private QueueProperties logWorker = QueueProperties.logWorkerDefaults(); + + private QueueProperties reduce = QueueProperties.reduceDefaults(); + + private QueueProperties windowEvaluator = QueueProperties.windowEvaluatorDefaults(); + + private int notifyMaxConcurrentPerChannel = 4; + } + + /** + * Queue-preserving executor configuration. + */ + @Getter + @Setter + public static class QueueProperties { + + private int maxConcurrentJobs; + + private int queueCapacity; + + private static QueueProperties reduceDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(2); + return properties; + } + + private static QueueProperties logWorkerDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(10); + properties.setQueueCapacity(1000); + return properties; + } + + private static QueueProperties windowEvaluatorDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(2); + return properties; + } + } + + /** + * Async executor configuration. + */ + @Getter + @Setter + public static class AsyncProperties { + + private boolean enabled = true; + + private int concurrencyLimit = 256; + + private boolean rejectWhenLimitReached = true; + + private long taskTerminationTimeout = 5000L; + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java index 4a231230f4f..a753dccb2d5 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java @@ -18,13 +18,18 @@ package org.apache.hertzbeat.common.support; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -34,22 +39,53 @@ @Slf4j public class CommonThreadPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public CommonThreadPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("common executor has uncaughtException."); + @Autowired + public CommonThreadPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("common executor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getCommon(); + return ManagedExecutors.newVirtualExecutor("common-worker", "common-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("common-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("common-long-running", "common-long-running-", + (thread, throwable) -> { + log.error("common longRunningExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("common-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(1, + return new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, @@ -67,10 +103,20 @@ public void execute(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + /** + * Run a long-lived task outside of the short-task execution lane. + * + * @param runnable task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + @Override public void destroy() throws Exception { - if (workerExecutor != null) { - workerExecutor.shutdownNow(); + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); } } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java index daa09493a88..063fdf73225 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java @@ -17,92 +17,89 @@ package org.apache.hertzbeat.common.support; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import java.lang.reflect.Field; + +import java.util.concurrent.CountDownLatch; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.BeforeEach; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** - * test for {@link CommonThreadPool} + * Test for {@link CommonThreadPool}. */ - class CommonThreadPoolTest { private CommonThreadPool commonThreadPool; - private ThreadPoolExecutor executorMock; - - @BeforeEach - public void setUp() throws Exception { - - commonThreadPool = new CommonThreadPool(); - - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - executorMock = mock(ThreadPoolExecutor.class); - workerExecutorField.set(commonThreadPool, executorMock); - } - - @Test - public void testExecuteTask() { - - Runnable task = mock(Runnable.class); - commonThreadPool.execute(task); - verify(executorMock).execute(task); + @AfterEach + void tearDown() throws Exception { + if (commonThreadPool != null) { + commonThreadPool.destroy(); + } } @Test - public void testExecuteTaskThrowsEx() { - - Runnable task = mock(Runnable.class); - doThrow(RejectedExecutionException.class).when(executorMock).execute(task); - - assertThrows( - RejectedExecutionException.class, - () -> commonThreadPool.execute(task) - ); - } + void testExecuteRunsOnVirtualThread() throws Exception { + commonThreadPool = new CommonThreadPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); - @Test - public void testDestroy() throws Exception { + commonThreadPool.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - commonThreadPool.destroy(); - verify(executorMock).shutdownNow(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); } @Test - public void testDestroyWithNull() throws Exception { + void testExecuteLongRunningRunsOnPlatformThread() throws Exception { + commonThreadPool = new CommonThreadPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(true); - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - workerExecutorField.set(commonThreadPool, null); + commonThreadPool.executeLongRunning(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - commonThreadPool.destroy(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertFalse(virtualThread.get()); } @Test - public void testInitialization() throws Exception { - CommonThreadPool pool = new CommonThreadPool(); - - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - ThreadPoolExecutor workerExecutor = (ThreadPoolExecutor) workerExecutorField.get(pool); - - assertNotNull(workerExecutor); - assertEquals(1, workerExecutor.getCorePoolSize()); - assertEquals(Integer.MAX_VALUE, workerExecutor.getMaximumPoolSize()); - assertEquals(10, workerExecutor.getKeepAliveTime(TimeUnit.SECONDS)); - assertTrue(workerExecutor.getQueue() instanceof SynchronousQueue); + void testExecuteRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties commonProperties = new VirtualThreadProperties.PoolProperties(); + commonProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + commonProperties.setMaxConcurrentJobs(1); + properties.setCommon(commonProperties); + commonThreadPool = new CommonThreadPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + commonThreadPool.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> commonThreadPool.execute(() -> { + })); + } finally { + release.countDown(); + } } - } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java index 31a537c6f2d..d45015a909d 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java @@ -61,11 +61,11 @@ public class LogSseManager { t.setDaemon(true); return t; }); - private final ExecutorService senderPool = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "sse-sender"); - t.setDaemon(true); - return t; - }); + private final ExecutorService senderPool = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("sse-sender-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("SSE sender has uncaughtException.", throwable)) + .factory()); private final AtomicLong queueSize = new AtomicLong(0); public LogSseManager() { diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java index 3f2c39ebeba..9ddc3b78477 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java +++ b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java @@ -26,7 +26,9 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; import java.io.IOException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -88,6 +91,24 @@ void shouldBroadcastLogWhenFilterMatches() throws IOException { ); } + @Test + void shouldSendBatchOnVirtualThread() throws IOException, InterruptedException { + SseEmitter mockEmitter = mock(SseEmitter.class); + AtomicBoolean virtualThread = new AtomicBoolean(false); + CountDownLatch latch = new CountDownLatch(1); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(mockEmitter).send(any(SseEmitter.SseEventBuilder.class)); + subscribeClient(CLIENT_ID, null, mockEmitter); + + logSseManager.broadcast(createLogEntry("INFO", "virtual-thread-send")); + + assertTrue(latch.await(1, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + @Test void shouldNotBroadcastLogWhenFilterDoesNotMatch() throws IOException, InterruptedException { // Given: A client with a filter for "ERROR" logs @@ -172,6 +193,16 @@ void shouldRemoveEmitterWhenBroadcastFails() throws IOException { }); } + @Test + void shouldDropLogsWhenQueueSizeLimitReached() { + for (int i = 0; i < 10_001; i++) { + logSseManager.broadcast(createLogEntry("INFO", "log-" + i)); + } + + assertEquals(10_000, logSseManager.getQueueSize()); + assertEquals(10_000, logSseManager.getLogQueue().size()); + } + /** * Helper method to create a subscriber and inject a mock emitter for testing */ @@ -190,4 +221,4 @@ private LogEntry createLogEntry(String severityText, String body) { .body(body) .build(); } -} \ No newline at end of file +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java index 3cbf7e22692..2ca8c48d05a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java @@ -78,7 +78,7 @@ public ServiceDiscoveryWorker(MonitorService monitorService, ParamDao paramDao, @Override public void afterPropertiesSet() { - workerPool.executeJob(new SdUpdateTask()); + workerPool.executeLongRunning(new SdUpdateTask()); } private class SdUpdateTask implements Runnable { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 94a0d7ca0cf..6639e5674c7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -30,11 +30,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.StatusPageComponent; @@ -45,6 +47,8 @@ import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; @@ -53,7 +57,7 @@ */ @Component @Slf4j -public class CalculateStatus { +public class CalculateStatus implements DisposableBean { private static final int DEFAULT_CALCULATE_INTERVAL_TIME = 300; @@ -67,180 +71,318 @@ public class CalculateStatus { private final int intervals; + private final ScheduledExecutorService calculateScheduler; + + private final ScheduledExecutorService combineHistoryScheduler; + + private final ExecutorService calculateExecutor; + + private final ExecutorService combineHistoryExecutor; + + private final ScheduledDispatchTask calculateTask; + + private final ScheduledDispatchTask combineHistoryTask; + public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, MonitorDao monitorDao) { + this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, + VirtualThreadProperties.defaults(), true); + } + + @Autowired + public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, + StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, + MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties) { + this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, + virtualThreadProperties, true); + } + + CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, + StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, + MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.statusPageOrgDao = statusPageOrgDao; this.monitorDao = monitorDao; this.statusPageComponentDao = statusPageComponentDao; this.statusPageHistoryDao = statusPageHistoryDao; intervals = statusProperties.getCalculate() == null ? DEFAULT_CALCULATE_INTERVAL_TIME : statusProperties.getCalculate().getInterval(); - startCalculate(); - startCombineHistory(); + this.calculateScheduler = createScheduler("status-page-calculate-%d", "Status calculate has uncaughtException."); + this.combineHistoryScheduler = createScheduler("status-page-history-%d", "History combine has uncaughtException."); + this.calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-", + "Status calculate worker has uncaughtException."); + this.combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-", + "History combine worker has uncaughtException."); + this.calculateTask = new ScheduledDispatchTask(calculateExecutor, this::runCalculate); + this.combineHistoryTask = new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory); + if (autoStart) { + startCalculate(); + startCombineHistory(); + } } private void startCalculate() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Status calculate has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("status-page-calculate-%d") - .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - scheduledExecutor.scheduleAtFixedRate(() -> { - log.info("start to calculate status page state"); - try { - // calculate component state from tag bind monitors status - List statusPageOrgList = statusPageOrgDao.findAll(); - for (StatusPageOrg statusPageOrg : statusPageOrgList) { - long orgId = statusPageOrg.getId(); - List pageComponentList = statusPageComponentDao.findByOrgId(orgId); - Set stateSet = new HashSet<>(8); - for (StatusPageComponent component : pageComponentList) { - byte state; - if (component.getMethod() == CommonConstants.STATUS_PAGE_CALCULATE_METHOD_MANUAL) { - state = component.getConfigState(); - } else { - Map labels = component.getLabels(); - if (labels == null || labels.isEmpty()) { - continue; - } - Specification specification = (root, query, criteriaBuilder) -> { - List predicates = new ArrayList<>(); - // create every label condition - labels.forEach((key, value) -> { - String pattern = String.format("%%\"%s\":\"%s\"%%", key, value); - predicates.add(criteriaBuilder.like(root.get("labels"), pattern)); - }); - - // use or connect them - return criteriaBuilder.or(predicates.toArray(new Predicate[0])); - }; - List monitorList = monitorDao.findAll(specification); - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN; - for (Monitor monitor : monitorList) { - if (monitor.getStatus() == CommonConstants.MONITOR_DOWN_CODE) { - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL; - break; - } else if (monitor.getStatus() == CommonConstants.MONITOR_UP_CODE) { - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL; - } + calculateScheduler.scheduleAtFixedRate(this::dispatchCalculate, 5, intervals, TimeUnit.SECONDS); + } + + private void startCombineHistory() { + // combine history every day at 1:00 AM + LocalDateTime now = LocalDateTime.now(); + LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); + if (now.isAfter(nextRun)) { + nextRun = nextRun.plusDays(1); + } + long delay = Duration.between(now, nextRun).toMillis(); + combineHistoryScheduler.scheduleAtFixedRate(this::dispatchCombineHistory, delay, + TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS); + } + + /** + * get calculate status intervals + * @return intervals + */ + public int getCalculateStatusIntervals() { + return intervals; + } + + void dispatchCalculate() { + calculateTask.dispatch(); + } + + void dispatchCombineHistory() { + combineHistoryTask.dispatch(); + } + + @Override + public void destroy() { + calculateScheduler.shutdownNow(); + combineHistoryScheduler.shutdownNow(); + if (calculateExecutor != null) { + calculateExecutor.shutdownNow(); + } + if (combineHistoryExecutor != null) { + combineHistoryExecutor.shutdownNow(); + } + } + + private void runCalculate() { + log.info("start to calculate status page state"); + try { + // calculate component state from tag bind monitors status + List statusPageOrgList = statusPageOrgDao.findAll(); + for (StatusPageOrg statusPageOrg : statusPageOrgList) { + long orgId = statusPageOrg.getId(); + List pageComponentList = statusPageComponentDao.findByOrgId(orgId); + Set stateSet = new HashSet<>(8); + for (StatusPageComponent component : pageComponentList) { + byte state; + if (component.getMethod() == CommonConstants.STATUS_PAGE_CALCULATE_METHOD_MANUAL) { + state = component.getConfigState(); + } else { + Map labels = component.getLabels(); + if (labels == null || labels.isEmpty()) { + continue; + } + Specification specification = (root, query, criteriaBuilder) -> { + List predicates = new ArrayList<>(); + // create every label condition + labels.forEach((key, value) -> { + String pattern = String.format("%%\"%s\":\"%s\"%%", key, value); + predicates.add(criteriaBuilder.like(root.get("labels"), pattern)); + }); + + // use or connect them + return criteriaBuilder.or(predicates.toArray(new Predicate[0])); + }; + List monitorList = monitorDao.findAll(specification); + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN; + for (Monitor monitor : monitorList) { + if (monitor.getStatus() == CommonConstants.MONITOR_DOWN_CODE) { + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL; + break; + } else if (monitor.getStatus() == CommonConstants.MONITOR_UP_CODE) { + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL; } } - stateSet.add(state); - component.setState(state); - statusPageComponentDao.save(component); - // insert component state history - StatusPageHistory statusPageHistory = StatusPageHistory.builder() - .componentId(component.getId()) - .state(state) - .timestamp(System.currentTimeMillis()) - .build(); - statusPageHistoryDao.save(statusPageHistory); } - stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); - if (stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL)) { - if (stateSet.contains(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL)) { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_SOME_ABNORMAL); - } else { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_ABNORMAL); - } + stateSet.add(state); + component.setState(state); + statusPageComponentDao.save(component); + // insert component state history + StatusPageHistory statusPageHistory = StatusPageHistory.builder() + .componentId(component.getId()) + .state(state) + .timestamp(System.currentTimeMillis()) + .build(); + statusPageHistoryDao.save(statusPageHistory); + } + stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + if (stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL)) { + if (stateSet.contains(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL)) { + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_SOME_ABNORMAL); } else { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_NORMAL); + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_ABNORMAL); } - statusPageOrg.setGmtUpdate(LocalDateTime.now()); - statusPageOrgDao.save(statusPageOrg); + } else { + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_NORMAL); } - } catch (Exception e) { - log.error("status page calculate component state error: {}", e.getMessage(), e); + statusPageOrg.setGmtUpdate(LocalDateTime.now()); + statusPageOrgDao.save(statusPageOrg); } - }, 5, intervals, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("status page calculate component state error: {}", e.getMessage(), e); + } } - private void startCombineHistory() { + private void runCombineHistory() { + try { + // combine pre day status history to one record + LocalDateTime nowTime = LocalDateTime.now(); + ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now()); + LocalDateTime midnight = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0); + LocalDateTime preNight = midnight.minusDays(1); + long midnightTimestamp = midnight.toInstant(zoneOffset).toEpochMilli(); + long preNightTimestamp = preNight.toInstant(zoneOffset).toEpochMilli(); + List statusPageHistoryList = statusPageHistoryDao + .findStatusPageHistoriesByTimestampBetween(preNightTimestamp, midnightTimestamp); + Map statusPageHistoryMap = new HashMap<>(8); + for (StatusPageHistory statusPageHistory : statusPageHistoryList) { + statusPageHistory.setNormal(0); + statusPageHistory.setAbnormal(0); + statusPageHistory.setUnknowing(0); + if (statusPageHistoryMap.containsKey(statusPageHistory.getComponentId())) { + StatusPageHistory history = statusPageHistoryMap.get(statusPageHistory.getComponentId()); + if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { + history.setAbnormal(history.getAbnormal() + intervals); + } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { + history.setUnknowing(history.getUnknowing() + intervals); + } else { + history.setNormal(history.getNormal() + intervals); + } + statusPageHistoryMap.put(statusPageHistory.getComponentId(), history); + } else { + if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { + statusPageHistory.setAbnormal(intervals); + } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { + statusPageHistory.setUnknowing(intervals); + } else { + statusPageHistory.setNormal(intervals); + } + statusPageHistoryMap.put(statusPageHistory.getComponentId(), statusPageHistory); + } + } + statusPageHistoryDao.deleteAll(statusPageHistoryList); + for (StatusPageHistory history : statusPageHistoryMap.values()) { + double total = history.getNormal() + history.getAbnormal() + history.getUnknowing(); + double uptime = 0; + if (total > 0) { + uptime = (double) history.getNormal() / total; + } + history.setUptime(uptime); + if (history.getAbnormal() > 0) { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL); + } else if (history.getNormal() > 0) { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL); + } else { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + } + statusPageHistoryDao.save(history); + } + } catch (Exception e) { + log.error("status page combine history error: {}", e.getMessage(), e); + } + } + + private ScheduledExecutorService createScheduler(String threadNameFormat, String errorMessage) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("History combine has uncaughtException."); + log.error(errorMessage); log.error(throwable.getMessage(), throwable); }) .setDaemon(true) - .setNameFormat("status-page-calculate-%d") + .setNameFormat(threadNameFormat) .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - // combine history every day at 1:00 AM - LocalDateTime now = LocalDateTime.now(); - LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); - if (now.isAfter(nextRun)) { - nextRun = nextRun.plusDays(1); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createVirtualExecutor(VirtualThreadProperties virtualThreadProperties, String threadPrefix, + String errorMessage) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; } - long delay = Duration.between(now, nextRun).toMillis(); - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - // combine pre day status history to one record - LocalDateTime nowTime = LocalDateTime.now(); - ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now()); - LocalDateTime midnight = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0); - LocalDateTime preNight = midnight.minusDays(1); - long midnightTimestamp = midnight.toInstant(zoneOffset).toEpochMilli(); - long preNightTimestamp = preNight.toInstant(zoneOffset).toEpochMilli(); - List statusPageHistoryList = statusPageHistoryDao - .findStatusPageHistoriesByTimestampBetween(preNightTimestamp, midnightTimestamp); - Map statusPageHistoryMap = new HashMap<>(8); - for (StatusPageHistory statusPageHistory : statusPageHistoryList) { - statusPageHistory.setNormal(0); - statusPageHistory.setAbnormal(0); - statusPageHistory.setUnknowing(0); - if (statusPageHistoryMap.containsKey(statusPageHistory.getComponentId())) { - StatusPageHistory history = statusPageHistoryMap.get(statusPageHistory.getComponentId()); - if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { - history.setAbnormal(history.getAbnormal() + intervals); - } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { - history.setUnknowing(history.getUnknowing() + intervals); - } else { - history.setNormal(history.getNormal() + intervals); - } - statusPageHistoryMap.put(statusPageHistory.getComponentId(), history); - } else { - if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { - statusPageHistory.setAbnormal(intervals); - } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { - statusPageHistory.setUnknowing(intervals); - } else { - statusPageHistory.setNormal(intervals); - } - statusPageHistoryMap.put(statusPageHistory.getComponentId(), statusPageHistory); - } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name(threadPrefix, 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error(errorMessage); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executorService; + private final Runnable task; + private final Object lock = new Object(); + private boolean running; + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { + this.executorService = executorService; + this.task = task; + } + + private void dispatch() { + if (executorService == null) { + task.run(); + return; + } + synchronized (lock) { + if (running) { + pendingRuns++; + return; } - statusPageHistoryDao.deleteAll(statusPageHistoryList); - for (StatusPageHistory history : statusPageHistoryMap.values()) { - double total = history.getNormal() + history.getAbnormal() + history.getUnknowing(); - double uptime = 0; - if (total > 0) { - uptime = (double) history.getNormal() / total; + running = true; + } + submit(); + } + + private void submit() { + boolean submitted = false; + try { + executorService.execute(() -> { + try { + task.run(); + } finally { + onComplete(); } - history.setUptime(uptime); - if (history.getAbnormal() > 0) { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL); - } else if (history.getNormal() > 0) { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL); - } else { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + running = false; + pendingRuns = 0; } - statusPageHistoryDao.save(history); } - } catch (Exception e) { - log.error("status page combine history error: {}", e.getMessage(), e); } - }, delay, TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS); - } + } - /** - * get calculate status intervals - * @return intervals - */ - public int getCalculateStatusIntervals() { - return intervals; + private void onComplete() { + boolean shouldRunAgain; + synchronized (lock) { + if (pendingRuns > 0) { + pendingRuns--; + shouldRunAgain = true; + } else { + running = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submit(); + } + } } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java index 1bc4a72b984..baefdcdbeb7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java @@ -18,12 +18,18 @@ package org.apache.hertzbeat.manager.scheduler; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,23 +37,54 @@ */ @Slf4j @Component -public class ManagerWorkerPool { - private ThreadPoolExecutor workerExecutor; +public class ManagerWorkerPool implements DisposableBean { + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public ManagerWorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("workerExecutor has uncaughtException."); + @Autowired + public ManagerWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("workerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getManager(); + return ManagedExecutors.newVirtualExecutor("manager-worker", "manager-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("manager-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("manager-long-running", "manager-long-running-", + (thread, throwable) -> { + log.error("manager longRunningExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("manager-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(6, + return new ThreadPoolExecutor(6, 10, 10, TimeUnit.SECONDS, @@ -59,4 +96,21 @@ private void initWorkExecutor() { public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } -} \ No newline at end of file + + /** + * Run a long-lived task outside of the short-task execution lane. + * + * @param runnable task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + + @Override + public void destroy() throws Exception { + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 352415b65f0..7fb26a974d6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -20,11 +20,13 @@ import io.netty.channel.Channel; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; @@ -39,6 +41,7 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; import org.apache.hertzbeat.remoting.netty.NettyServerConfig; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.Ordered; @@ -61,6 +64,14 @@ public class ManageServer implements CommandLineRunner { private ScheduledExecutorService channelSchedule; + private final ExecutorService channelCheckExecutor; + + private final Object channelCheckLock = new Object(); + + private boolean channelCheckRunning; + + private boolean channelCheckPending; + private RemotingServer remotingServer; private final Map clientChannelTable = new ConcurrentHashMap<>(16); @@ -69,9 +80,20 @@ public ManageServer(final SchedulerProperties schedulerProperties, final CollectorJobScheduler collectorJobScheduler, final CommonThreadPool threadPool, final CollectorAlertHandler collectorAlertHandler) { + this(schedulerProperties, collectorJobScheduler, threadPool, collectorAlertHandler, + VirtualThreadProperties.defaults()); + } + + @Autowired + public ManageServer(final SchedulerProperties schedulerProperties, + final CollectorJobScheduler collectorJobScheduler, + final CommonThreadPool threadPool, + final CollectorAlertHandler collectorAlertHandler, + final VirtualThreadProperties virtualThreadProperties) { this.collectorJobScheduler = collectorJobScheduler; this.collectorJobScheduler.setManageServer(this); this.collectorAlertHandler = collectorAlertHandler; + this.channelCheckExecutor = createChannelCheckExecutor(virtualThreadProperties); this.init(schedulerProperties, threadPool); } @@ -96,26 +118,18 @@ private void init(final SchedulerProperties schedulerProperties, final CommonThr public void start() { this.remotingServer.start(); - this.channelSchedule.scheduleAtFixedRate(() -> { - try { - this.clientChannelTable.forEach((collector, channel) -> { - if (!channel.isActive()) { - channel.closeFuture(); - this.clientChannelTable.remove(collector); - this.collectorJobScheduler.collectorGoOffline(collector); - this.collectorAlertHandler.offline(collector); - } - }); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - }, 10, 3, TimeUnit.SECONDS); + this.channelSchedule.scheduleAtFixedRate(this::dispatchChannelHealthCheck, 10, 3, TimeUnit.SECONDS); } public void shutdown() { this.remotingServer.shutdown(); - this.channelSchedule.shutdownNow(); + if (this.channelSchedule != null) { + this.channelSchedule.shutdownNow(); + } + if (this.channelCheckExecutor != null) { + this.channelCheckExecutor.shutdownNow(); + } } public CollectorJobScheduler getCollectorAndJobScheduler() { @@ -173,6 +187,21 @@ public ClusterMsg.Message sendMsgSync(final String identityId, final ClusterMsg. return null; } + void dispatchChannelHealthCheck() { + if (channelCheckExecutor == null) { + runChannelHealthCheck(); + return; + } + synchronized (channelCheckLock) { + if (channelCheckRunning) { + channelCheckPending = true; + return; + } + channelCheckRunning = true; + } + submitChannelHealthCheck(); + } + @Override public void run(String... args) throws Exception { this.start(); @@ -200,4 +229,68 @@ public void onChannelIdle(Channel channel) { } } } + + private ExecutorService createChannelCheckExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("manager-channel-check-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> log.error("Channel checker has uncaughtException.", throwable)) + .factory()); + } + + private void submitChannelHealthCheck() { + boolean submitted = false; + try { + channelCheckExecutor.execute(() -> { + try { + runChannelHealthCheck(); + } finally { + onChannelHealthCheckComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (channelCheckLock) { + channelCheckRunning = false; + channelCheckPending = false; + } + } + } + } + + private void onChannelHealthCheckComplete() { + boolean shouldRunAgain; + synchronized (channelCheckLock) { + if (channelCheckPending) { + channelCheckPending = false; + shouldRunAgain = true; + } else { + channelCheckRunning = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submitChannelHealthCheck(); + } + } + + private void runChannelHealthCheck() { + try { + this.clientChannelTable.forEach((collector, channel) -> { + if (!channel.isActive()) { + channel.closeFuture(); + this.clientChannelTable.remove(collector); + this.collectorJobScheduler.collectorGoOffline(collector); + this.collectorAlertHandler.offline(collector); + } + }); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java new file mode 100644 index 00000000000..0ec7833213f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.component.status; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.manager.config.StatusProperties; +import org.apache.hertzbeat.manager.dao.MonitorDao; +import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; +import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; +import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Test case for {@link CalculateStatus}. + */ +@ExtendWith(MockitoExtension.class) +class CalculateStatusTest { + + @Mock + private StatusPageOrgDao statusPageOrgDao; + + @Mock + private StatusPageComponentDao statusPageComponentDao; + + @Mock + private StatusPageHistoryDao statusPageHistoryDao; + + @Mock + private MonitorDao monitorDao; + + private CalculateStatus calculateStatus; + + @BeforeEach + void setUp() { + calculateStatus = new CalculateStatus(statusPageOrgDao, statusPageComponentDao, statusProperties(), + statusPageHistoryDao, monitorDao, new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (calculateStatus != null) { + calculateStatus.destroy(); + } + } + + @Test + void dispatchCalculateRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCombineHistoryRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return Collections.emptyList(); + }).when(statusPageHistoryDao).findStatusPageHistoriesByTimestampBetween(anyLong(), anyLong()); + + calculateStatus.dispatchCombineHistory(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCalculateDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + calculateStatus.dispatchCalculate(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private StatusProperties statusProperties() { + StatusProperties statusProperties = new StatusProperties(); + StatusProperties.CalculateProperties calculateProperties = new StatusProperties.CalculateProperties(); + calculateProperties.setInterval(300); + statusProperties.setCalculate(calculateProperties); + return statusProperties; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java new file mode 100644 index 00000000000..35757bf5afc --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.scheduler; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Test for {@link ManagerWorkerPool}. + */ +class ManagerWorkerPoolTest { + + private ManagerWorkerPool managerWorkerPool; + + @AfterEach + void tearDown() throws Exception { + if (managerWorkerPool != null) { + managerWorkerPool.destroy(); + } + } + + @Test + void testExecuteJobRunsOnVirtualThread() throws Exception { + managerWorkerPool = new ManagerWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + managerWorkerPool.executeJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties managerProperties = new VirtualThreadProperties.PoolProperties(); + managerProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + managerProperties.setMaxConcurrentJobs(1); + properties.setManager(managerProperties); + managerWorkerPool = new ManagerWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + managerWorkerPool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> managerWorkerPool.executeJob(() -> { + })); + } finally { + release.countDown(); + } + } + + @Test + void testExecuteLongRunningRunsOnPlatformThread() throws Exception { + managerWorkerPool = new ManagerWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(true); + + managerWorkerPool.executeLongRunning(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertFalse(virtualThread.get()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java new file mode 100644 index 00000000000..eef43410c0a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.scheduler.netty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.support.CommonThreadPool; +import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; +import org.apache.hertzbeat.manager.scheduler.SchedulerProperties; +import org.apache.hertzbeat.remoting.RemotingServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +/** + * Test case for {@link ManageServer}. + */ +@ExtendWith(MockitoExtension.class) +class ManageServerTest { + + @Mock + private CollectorJobScheduler collectorJobScheduler; + + @Mock + private CommonThreadPool commonThreadPool; + + @Mock + private CollectorAlertHandler collectorAlertHandler; + + private ManageServer manageServer; + + @BeforeEach + void setUp() { + manageServer = new ManageServer(schedulerProperties(), collectorJobScheduler, commonThreadPool, + collectorAlertHandler, new VirtualThreadProperties()); + ReflectionTestUtils.setField(manageServer, "remotingServer", mock(RemotingServer.class)); + } + + @AfterEach + void tearDown() { + if (manageServer != null) { + manageServer.shutdown(); + } + } + + @Test + void dispatchChannelHealthCheckRunsOnVirtualThread() throws Exception { + Channel channel = mock(Channel.class); + when(channel.isActive()).thenReturn(false); + when(channel.closeFuture()).thenReturn(mock(ChannelFuture.class)); + clientChannelTable().put("collector-1", channel); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(collectorJobScheduler).collectorGoOffline(anyString()); + + manageServer.dispatchChannelHealthCheck(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchChannelHealthCheckDoesNotRunConcurrently() throws Exception { + Channel channel = mock(Channel.class); + clientChannelTable().put("collector-1", channel); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return true; + }).when(channel).isActive(); + + manageServer.dispatchChannelHealthCheck(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + manageServer.dispatchChannelHealthCheck(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @SuppressWarnings("unchecked") + private Map clientChannelTable() { + return (Map) ReflectionTestUtils.getField(manageServer, "clientChannelTable"); + } + + private SchedulerProperties schedulerProperties() { + SchedulerProperties schedulerProperties = new SchedulerProperties(); + SchedulerProperties.ServerProperties serverProperties = new SchedulerProperties.ServerProperties(); + serverProperties.setPort(1158); + serverProperties.setIdleStateEventTriggerTime(100); + schedulerProperties.setServer(serverProperties); + return schedulerProperties; + } +} diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java index a01c7fab804..7863e830e31 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java @@ -72,7 +72,7 @@ public NettyRemotingClient(final NettyClientConfig nettyClientConfig, @Override public void start() { - this.threadPool.execute(() -> { + this.threadPool.executeLongRunning(() -> { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("NettyClientWorker has uncaughtException."); diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java index 48062fadb9b..169f38220f3 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java @@ -77,7 +77,7 @@ public NettyRemotingServer(final NettyServerConfig nettyServerConfig, @Override public void start() { - this.threadPool.execute(() -> { + this.threadPool.executeLongRunning(() -> { int port = this.nettyServerConfig.getPort(); ThreadFactory bossThreadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java new file mode 100644 index 00000000000..28b0074c1d4 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SimpleAsyncTaskExecutor; + +/** + * Dedicated {@code @Async} executor configuration. + */ +@Configuration(proxyBeanMethods = false) +public class AsyncConfig { + + @Bean(name = "taskExecutor", destroyMethod = "close") + public SimpleAsyncTaskExecutor taskExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.getAsync(); + SimpleAsyncTaskExecutor executor = new SimpleAsyncTaskExecutor("async-worker-"); + executor.setVirtualThreads(virtualThreadProperties.isEnabled() && asyncProperties.isEnabled()); + executor.setConcurrencyLimit(asyncProperties.getConcurrencyLimit()); + executor.setRejectTasksWhenLimitReached(asyncProperties.isRejectWhenLimitReached()); + executor.setTaskTerminationTimeout(asyncProperties.getTaskTerminationTimeout()); + return executor; + } +} diff --git a/hertzbeat-startup/src/main/resources/application-test.yml b/hertzbeat-startup/src/main/resources/application-test.yml index 5a869dae1c2..eaf3e299d24 100644 --- a/hertzbeat-startup/src/main/resources/application-test.yml +++ b/hertzbeat-startup/src/main/resources/application-test.yml @@ -190,3 +190,34 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index 1203a679b16..ba90e282afb 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -334,3 +334,34 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java new file mode 100644 index 00000000000..00d6e2d1052 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.Test; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.core.task.TaskRejectedException; + +/** + * Tests for {@link AsyncConfig}. + */ +class AsyncConfigTest { + + private final AsyncConfig asyncConfig = new AsyncConfig(); + + @Test + void taskExecutorRunsAsyncTasksOnVirtualThreads() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + executor.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + } + + @Test + void taskExecutorRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + properties.getAsync().setConcurrencyLimit(1); + properties.getAsync().setRejectWhenLimitReached(true); + + try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + + executor.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(TaskRejectedException.class, () -> executor.execute(() -> { + })); + } finally { + release.countDown(); + } + } + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java index 14b77f9ba03..af4eb5172ee 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java @@ -18,12 +18,18 @@ package org.apache.hertzbeat.warehouse; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,24 +37,55 @@ */ @Component @Slf4j -public class WarehouseWorkerPool { +public class WarehouseWorkerPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public WarehouseWorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); + } + + @Autowired + public WarehouseWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Warehouse workerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getWarehouse(); + return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-", + (thread, throwable) -> { + log.error("Warehouse longRunningExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }); } - private void initWorkExecutor() { - // Thread factory + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("workerExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("warehouse-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(2, + return new ThreadPoolExecutor(2, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, @@ -65,4 +102,21 @@ private void initWorkExecutor() { public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + + /** + * Run a long-lived warehouse consumer outside of the per-task executor. + * + * @param runnable consumer task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + + @Override + public void destroy() throws Exception { + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); + } + } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java index 177b337344f..aee2a33ac2a 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java @@ -100,7 +100,7 @@ protected void startPersistentDataStorage() { } } }; - workerPool.executeJob(runnable); + workerPool.executeLongRunning(runnable); } protected void startLogDataStorage() { @@ -132,7 +132,7 @@ protected void startLogDataStorage() { } } }; - workerPool.executeJob(runnable); + workerPool.executeLongRunning(runnable); } protected void calculateMonitorStatus(CollectRep.MetricsData metricsData) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java index 22bcd468272..900c1fdc7a7 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java @@ -334,7 +334,7 @@ private void initStreamLoadWriter() { */ private void startFlushThread() { try { - warehouseWorkerPool.executeJob(() -> { + warehouseWorkerPool.executeLongRunning(() -> { flushTaskStarted = true; try { while (flushThreadRunning || !metricsBufferQueue.isEmpty()) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java index d98c2ee7fb9..17ec3b60bfd 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java @@ -21,6 +21,7 @@ import com.zaxxer.hikari.HikariDataSource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; @@ -29,6 +30,7 @@ import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -46,6 +48,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -71,13 +74,35 @@ public class DuckdbDatabaseDataStorage extends AbstractHistoryDataStorage { private final String expireTimeStr; private final String dbPath; + private final ScheduledExecutorService cleanerScheduler; + private final ExecutorService cleanerExecutor; + private final ScheduledDispatchTask cleanerTask; private HikariDataSource dataSource; public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties) { + this(duckdbProperties, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + VirtualThreadProperties virtualThreadProperties) { + this(duckdbProperties, virtualThreadProperties, true); + } + + DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + VirtualThreadProperties virtualThreadProperties, + boolean autoStartCleaner) { this.dbPath = duckdbProperties.storePath(); this.expireTimeStr = duckdbProperties.expireTime(); + this.cleanerScheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "duckdb-cleaner"); + thread.setDaemon(true); + return thread; + }); + this.cleanerExecutor = createCleanerExecutor(virtualThreadProperties); + this.cleanerTask = new ScheduledDispatchTask(cleanerExecutor, this::runExpiredDataCleaner); this.serverAvailable = initDuckDb(); - if (this.serverAvailable) { + if (this.serverAvailable && autoStartCleaner) { startExpiredDataCleaner(); } } @@ -131,60 +156,8 @@ CREATE TABLE IF NOT EXISTS hzb_history ( } private void startExpiredDataCleaner() { - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(r -> { - Thread thread = new Thread(r, "duckdb-cleaner"); - thread.setDaemon(true); - return thread; - }); // Run every 1 hour - scheduledExecutor.scheduleAtFixedRate(() -> { - log.info("[duckdb] start data cleaner and checkpoint..."); - long expireTime; - try { - // Ensure no whitespace issues - String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim(); - Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr); - - if (NumberUtils.isParsable(cleanExpireStr)) { - expireTime = NumberUtils.toLong(cleanExpireStr); - expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L; - } else if (dayMatcher.matches()) { - // Strictly matched "90d" or "90D" format - long days = Long.parseLong(dayMatcher.group(1)); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days)); - expireTime = dateTime.toEpochSecond() * 1000L; - } else { - // Fallback to existing utility for other units (h, m, s, etc.) - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - expireTime = dateTime.toEpochSecond() * 1000L; - } - } catch (Exception e) { - log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30)); - expireTime = dateTime.toEpochSecond() * 1000L; - } - - try (Connection connection = this.dataSource.getConnection()) { - // 1. Delete expired data - try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) { - statement.setLong(1, expireTime); - int rows = statement.executeUpdate(); - if (rows > 0) { - log.info("[duckdb] delete {} expired records.", rows); - } - } - - // 2. Force Checkpoint to compress data and flush WAL - // This is crucial for keeping file size small and moving data from WAL to column store - try (Statement statement = connection.createStatement()) { - statement.execute("CHECKPOINT"); - } - - } catch (Exception e) { - log.error("[duckdb] clean expired data error: {}", e.getMessage(), e); - } - }, 5, 60, TimeUnit.MINUTES); + cleanerScheduler.scheduleAtFixedRate(this::dispatchExpiredDataCleaner, 5, 60, TimeUnit.MINUTES); } @Override @@ -444,8 +417,151 @@ private String formatValue(int type, ResultSet resultSet) throws SQLException { @Override public void destroy() throws Exception { + cleanerScheduler.shutdownNow(); + if (cleanerExecutor != null) { + cleanerExecutor.shutdownNow(); + } if (this.dataSource != null) { this.dataSource.close(); } } + + void dispatchExpiredDataCleaner() { + cleanerTask.dispatch(); + } + + void beforeExpiredDataCleanerRun() { + // Test hook. + } + + private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("duckdb-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("[duckdb] cleaner worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runExpiredDataCleaner() { + log.info("[duckdb] start data cleaner and checkpoint..."); + beforeExpiredDataCleanerRun(); + long expireTime; + try { + // Ensure no whitespace issues + String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim(); + Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr); + + if (NumberUtils.isParsable(cleanExpireStr)) { + expireTime = NumberUtils.toLong(cleanExpireStr); + expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L; + } else if (dayMatcher.matches()) { + // Strictly matched "90d" or "90D" format + long days = Long.parseLong(dayMatcher.group(1)); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days)); + expireTime = dateTime.toEpochSecond() * 1000L; + } else { + // Fallback to existing utility for other units (h, m, s, etc.) + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000L; + } + } catch (Exception e) { + log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30)); + expireTime = dateTime.toEpochSecond() * 1000L; + } + + try (Connection connection = this.dataSource.getConnection()) { + // 1. Delete expired data + try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) { + statement.setLong(1, expireTime); + int rows = statement.executeUpdate(); + if (rows > 0) { + log.info("[duckdb] delete {} expired records.", rows); + } + } + + // 2. Force Checkpoint to compress data and flush WAL + // This is crucial for keeping file size small and moving data from WAL to column store + try (Statement statement = connection.createStatement()) { + statement.execute("CHECKPOINT"); + } + + } catch (Exception e) { + log.error("[duckdb] clean expired data error: {}", e.getMessage(), e); + } + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executorService; + private final Runnable task; + private final Object lock = new Object(); + private boolean running; + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { + this.executorService = executorService; + this.task = task; + } + + private void dispatch() { + if (executorService == null) { + task.run(); + return; + } + synchronized (lock) { + if (running) { + pendingRuns++; + return; + } + running = true; + } + submit(); + } + + private void submit() { + boolean submitted = false; + try { + executorService.execute(() -> { + try { + task.run(); + } finally { + onComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + running = false; + pendingRuns = 0; + } + } + } + } + + private void onComplete() { + boolean shouldRunAgain; + synchronized (lock) { + if (pendingRuns > 0) { + pendingRuns--; + shouldRunAgain = true; + } else { + running = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submit(); + } + } + } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java index 028ae722cb1..c014ae24c57 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java @@ -18,9 +18,16 @@ package org.apache.hertzbeat.warehouse; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.BeforeEach; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -30,18 +37,19 @@ class WarehouseWorkerPoolTest { private static final int NUMBER_OF_THREADS = 10; private WarehouseWorkerPool pool; - private AtomicInteger counter; - private CountDownLatch latch; - @BeforeEach - void setUp() { - pool = new WarehouseWorkerPool(); - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); + @AfterEach + void tearDown() throws Exception { + if (pool != null) { + pool.destroy(); + } } @Test void executeJob() throws InterruptedException { + pool = new WarehouseWorkerPool(); + AtomicInteger counter = new AtomicInteger(); + CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); for (int i = 0; i < NUMBER_OF_THREADS; i++) { pool.executeJob(() -> { counter.incrementAndGet(); @@ -53,4 +61,47 @@ void executeJob() throws InterruptedException { assertEquals(NUMBER_OF_THREADS, counter.get()); } + @Test + void executeJobRunsOnVirtualThread() throws InterruptedException { + pool = new WarehouseWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties warehouseProperties = new VirtualThreadProperties.PoolProperties(); + warehouseProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + warehouseProperties.setMaxConcurrentJobs(1); + properties.setWarehouse(warehouseProperties); + pool = new WarehouseWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeJob(() -> { + })); + } finally { + release.countDown(); + } + } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java new file mode 100644 index 00000000000..4a796ec99ed --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Test case for {@link DuckdbDatabaseDataStorage}. + */ +class DuckdbDatabaseDataStorageTest { + + private TestDuckdbDatabaseDataStorage dataStorage; + + @TempDir + Path tempDir; + + @AfterEach + void tearDown() throws Exception { + if (dataStorage != null) { + dataStorage.destroy(); + } + } + + @Test + void dispatchExpiredDataCleanerRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + dataStorage = new TestDuckdbDatabaseDataStorage(properties(), latch, virtualThread, + null, null, null, null, null, null); + + dataStorage.dispatchExpiredDataCleaner(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchExpiredDataCleanerDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + dataStorage = new TestDuckdbDatabaseDataStorage(properties(), null, null, + firstStarted, releaseFirst, secondStarted, concurrent, maxConcurrent, invocations); + + dataStorage.dispatchExpiredDataCleaner(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + dataStorage.dispatchExpiredDataCleaner(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private DuckdbProperties properties() { + return new DuckdbProperties(true, "1d", tempDir.resolve("history.duckdb").toString()); + } + + private static final class TestDuckdbDatabaseDataStorage extends DuckdbDatabaseDataStorage { + + private final CountDownLatch virtualThreadLatch; + private final AtomicBoolean virtualThread; + private final CountDownLatch firstStarted; + private final CountDownLatch releaseFirst; + private final CountDownLatch secondStarted; + private final AtomicInteger concurrent; + private final AtomicInteger maxConcurrent; + private final AtomicInteger invocations; + + private TestDuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + CountDownLatch virtualThreadLatch, + AtomicBoolean virtualThread, + CountDownLatch firstStarted, + CountDownLatch releaseFirst, + CountDownLatch secondStarted, + AtomicInteger concurrent, + AtomicInteger maxConcurrent, + AtomicInteger invocations) { + super(duckdbProperties, new VirtualThreadProperties(), false); + this.virtualThreadLatch = virtualThreadLatch; + this.virtualThread = virtualThread; + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.concurrent = concurrent; + this.maxConcurrent = maxConcurrent; + this.invocations = invocations; + } + + @Override + void beforeExpiredDataCleanerRun() { + if (virtualThreadLatch != null && virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + virtualThreadLatch.countDown(); + } + if (firstStarted == null || releaseFirst == null || secondStarted == null + || concurrent == null || maxConcurrent == null || invocations == null) { + return; + } + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } +} From 6f389d012ea03b49c1be64d05f012e3a294adc4a Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:32:06 +0800 Subject: [PATCH 02/14] chore: document virtual thread configuration defaults --- .../src/main/resources/application.yml | 1 + .../config/VirtualThreadPropertiesTest.java | 59 +++++++++ .../src/main/resources/application.yml | 1 + home/docs/start/custom-config.md | 6 + home/docs/start/virtual-thread.md | 115 ++++++++++++++++++ .../current/start/custom-config.md | 6 + .../current/start/virtual-thread.md | 115 ++++++++++++++++++ script/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 31 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ 13 files changed, 494 insertions(+) create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java create mode 100644 home/docs/start/virtual-thread.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 43057c241bf..9615e547b94 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -77,6 +77,7 @@ common: type: netty hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true common: diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java new file mode 100644 index 00000000000..7433ae3944c --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.junit.jupiter.api.Test; + +class VirtualThreadPropertiesTest { + + @Test + void defaultsRemainSafeWithoutExternalConfiguration() { + VirtualThreadProperties properties = VirtualThreadProperties.defaults(); + + assertTrue(properties.isEnabled()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); + assertTrue(properties.getCollector().getMaxConcurrentJobs() >= 1); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); + assertEquals(10, properties.getManager().getMaxConcurrentJobs()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getAlerter().getNotify().getMode()); + assertEquals(64, properties.getAlerter().getNotify().getMaxConcurrentJobs()); + assertEquals(10, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + assertEquals(10, properties.getAlerter().getLogWorker().getMaxConcurrentJobs()); + assertEquals(1000, properties.getAlerter().getLogWorker().getQueueCapacity()); + assertEquals(2, properties.getAlerter().getReduce().getMaxConcurrentJobs()); + assertEquals(0, properties.getAlerter().getReduce().getQueueCapacity()); + assertEquals(2, properties.getAlerter().getWindowEvaluator().getMaxConcurrentJobs()); + assertEquals(0, properties.getAlerter().getWindowEvaluator().getQueueCapacity()); + assertEquals(4, properties.getAlerter().getNotifyMaxConcurrentPerChannel()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getWarehouse().getMode()); + + assertTrue(properties.getAsync().isEnabled()); + assertEquals(256, properties.getAsync().getConcurrencyLimit()); + assertTrue(properties.getAsync().isRejectWhenLimitReached()); + assertEquals(5000L, properties.getAsync().getTaskTerminationTimeout()); + } +} diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index ba90e282afb..ecebc66681f 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -336,6 +336,7 @@ grafana: password: admin hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true common: diff --git a/home/docs/start/custom-config.md b/home/docs/start/custom-config.md index 59df329d6aa..4d6953a0972 100644 --- a/home/docs/start/custom-config.md +++ b/home/docs/start/custom-config.md @@ -14,6 +14,12 @@ Configuring the HertzBeat configuration file: - **Docker Deployment:** ⚠️ When using a Docker container, the `application.yml` file must be mounted to the host machine - **Installation Package Deployment:** Extract the package and modify the configuration file located at `hertzbeat/config/application.yml` +## 0. Virtual Thread Configuration + +Virtual-thread related defaults, tuning guidance, rollback switches, and Docker/package config locations are documented on a dedicated page: + +- [Virtual Thread Configuration](./virtual-thread) + ## 1. Configuring Custom Alert Parameters ```yaml diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md new file mode 100644 index 00000000000..d28da0f3ca0 --- /dev/null +++ b/home/docs/start/virtual-thread.md @@ -0,0 +1,115 @@ +--- +id: virtual-thread +title: Virtual Thread Configuration +sidebar_label: Virtual Threads +description: Configure HertzBeat virtual-thread executors, defaults, rollback switches, and tuning guidance. +--- + +HertzBeat runs on JDK 21 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. + +## 1. Where to Configure It + +Choose the config file that matches your deployment mode: + +- Package deployment: `hertzbeat/config/application.yml` +- Docker single-node deployment: mount your local `application.yml` to `/opt/hertzbeat/config/application.yml` +- Docker Compose deployment: edit `script/docker-compose/*/conf/application.yml` +- Standalone collector deployment: edit `hertzbeat-collector/config/application.yml` + +## 2. No Configuration Required + +You can leave out the entire `hertzbeat.vthreads` block. + +```yaml +# No virtual-thread override is required. +``` + +HertzBeat will apply runtime defaults automatically. + +## 3. Full Optional Configuration Template + +Use this only when you want to override the defaults: + +```yaml +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 +``` + +## 4. Built-In Defaults + +| Key | Default | Notes | +| --- | --- | --- | +| `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | +| `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | +| `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | Computed at runtime; no YAML update required | +| `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | +| `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | +| `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | +| `hertzbeat.vthreads.alerter.notify.max-concurrent-jobs` | `64` | Global notify concurrency | +| `hertzbeat.vthreads.alerter.notify-max-concurrent-per-channel` | `4` | Per notification channel/type | +| `hertzbeat.vthreads.alerter.periodic-max-concurrent-jobs` | `10` | Global periodic alert concurrency | +| `hertzbeat.vthreads.alerter.log-worker.max-concurrent-jobs` | `10` | Log alert short-task concurrency | +| `hertzbeat.vthreads.alerter.log-worker.queue-capacity` | `1000` | Bounded queue to preserve backlog semantics | +| `hertzbeat.vthreads.alerter.reduce.max-concurrent-jobs` | `2` | Alarm reduce concurrency | +| `hertzbeat.vthreads.alerter.reduce.queue-capacity` | unbounded | Leave unset to keep the legacy unbounded queue behavior | +| `hertzbeat.vthreads.alerter.window-evaluator.max-concurrent-jobs` | `2` | Window evaluator concurrency | +| `hertzbeat.vthreads.alerter.window-evaluator.queue-capacity` | unbounded | Leave unset to keep the legacy unbounded queue behavior | +| `hertzbeat.vthreads.warehouse.mode` | `UNBOUNDED_VT` | Storage short tasks; downstream pools still limit real resources | +| `hertzbeat.vthreads.async.enabled` | `true` | Dedicated `@Async` executor switch | +| `hertzbeat.vthreads.async.concurrency-limit` | `256` | `@Async` concurrency guard | +| `hertzbeat.vthreads.async.reject-when-limit-reached` | `true` | Reject extra `@Async` tasks at the limit | +| `hertzbeat.vthreads.async.task-termination-timeout` | `5000` | Milliseconds | + +## 5. Tuning Guidance + +- Start with the defaults unless you already know a downstream dependency is weak. +- Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. +- Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. +- Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. +- `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. + +## 6. Rollback + +Disable HertzBeat virtual-thread executors with: + +```yaml +hertzbeat: + vthreads: + enabled: false +``` + +This rolls the affected executors back to their legacy platform-thread implementations. + +## 7. Notes + +- You do not need to add `spring.threads.virtual.enabled` for this feature set. HertzBeat uses a dedicated `@Async` executor configuration. +- Shipping YAML files now include the optional `hertzbeat.vthreads` block as an example, but the block is still optional. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md index 03c9538218a..ed18b3af3f3 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md @@ -14,6 +14,12 @@ sidebar_label: 常见参数配置 - **Docker部署：** ⚠️docker容器方式需要将 `application.yml` 文件挂载到主机本地 - **安装包方式：** 解压修改位于 `hertzbeat/config/application.yml` 的配置文件即可 +## 0. 虚拟线程配置 + +虚拟线程的默认值、调优建议、回滚方式，以及 Docker/安装包对应的配置文件位置，已经单独整理到文档页： + +- [虚拟线程配置说明](./virtual-thread) + ## 1. 配置告警自定义参数 ```yaml diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md new file mode 100644 index 00000000000..a8c4c1ba4da --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -0,0 +1,115 @@ +--- +id: virtual-thread +title: 虚拟线程配置说明 +sidebar_label: 虚拟线程 +description: 说明 HertzBeat 虚拟线程执行器的默认值、回滚开关和调优方式。 +--- + +HertzBeat 基于 JDK 21 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 + +## 1. 到哪里配置 + +根据你的部署方式修改对应配置文件： + +- 安装包部署：`hertzbeat/config/application.yml` +- Docker 单机部署：把本地 `application.yml` 挂载到容器内 `/opt/hertzbeat/config/application.yml` +- Docker Compose 部署：修改 `script/docker-compose/*/conf/application.yml` +- 独立 collector 部署：修改 `hertzbeat-collector/config/application.yml` + +## 2. 不配置也可以 + +你可以完全不写 `hertzbeat.vthreads` 这一段： + +```yaml +# 虚拟线程配置可以整体省略。 +``` + +HertzBeat 会自动使用运行时默认值。 + +## 3. 完整可选配置模板 + +只有在你需要覆盖默认值时，才需要显式写出下面这段： + +```yaml +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 +``` + +## 4. 内置默认值 + +| 配置项 | 默认值 | 说明 | +| --- | --- | --- | +| `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | +| `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | +| `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | 运行时动态计算，不依赖 YAML | +| `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | +| `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | +| `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | +| `hertzbeat.vthreads.alerter.notify.max-concurrent-jobs` | `64` | 通知全局并发 | +| `hertzbeat.vthreads.alerter.notify-max-concurrent-per-channel` | `4` | 单通知通道/类型并发 | +| `hertzbeat.vthreads.alerter.periodic-max-concurrent-jobs` | `10` | 周期告警全局并发 | +| `hertzbeat.vthreads.alerter.log-worker.max-concurrent-jobs` | `10` | 日志告警短任务并发 | +| `hertzbeat.vthreads.alerter.log-worker.queue-capacity` | `1000` | 有界队列，保留 backlog 语义 | +| `hertzbeat.vthreads.alerter.reduce.max-concurrent-jobs` | `2` | 告警 reduce 并发 | +| `hertzbeat.vthreads.alerter.reduce.queue-capacity` | 无界 | 默认不填，保持旧版无界队列语义 | +| `hertzbeat.vthreads.alerter.window-evaluator.max-concurrent-jobs` | `2` | 窗口 evaluator 并发 | +| `hertzbeat.vthreads.alerter.window-evaluator.queue-capacity` | 无界 | 默认不填，保持旧版无界队列语义 | +| `hertzbeat.vthreads.warehouse.mode` | `UNBOUNDED_VT` | 仓储短任务执行；真实资源仍由下游连接池限制 | +| `hertzbeat.vthreads.async.enabled` | `true` | 专用 `@Async` 执行器开关 | +| `hertzbeat.vthreads.async.concurrency-limit` | `256` | `@Async` 并发保护阈值 | +| `hertzbeat.vthreads.async.reject-when-limit-reached` | `true` | 达到上限后拒绝额外 `@Async` 任务 | +| `hertzbeat.vthreads.async.task-termination-timeout` | `5000` | 单位毫秒 | + +## 5. 调优建议 + +- 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 +- 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 +- 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 +- `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 +- `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 + +## 6. 回滚方式 + +如需关闭 HertzBeat 的虚拟线程执行器，可配置： + +```yaml +hertzbeat: + vthreads: + enabled: false +``` + +这样相关执行器会回退到原来的平台线程实现。 + +## 7. 补充说明 + +- 这套改造不要求额外配置 `spring.threads.virtual.enabled`。HertzBeat 当前使用的是单独的 `@Async` 执行器配置。 +- 交付给用户的 YAML 示例文件现在已经带上了可选的 `hertzbeat.vthreads` 配置块，但这段配置依然可以整体删除。 diff --git a/script/application.yml b/script/application.yml index 1203a679b16..ecebc66681f 100644 --- a/script/application.yml +++ b/script/application.yml @@ -334,3 +334,35 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index 2ee71362555..c4c9d80797e 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -234,3 +234,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index b4c195c22fd..8a6dfe1528f 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -232,3 +232,34 @@ grafana: username: admin password: admin +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index c81a69107f0..d9118c6c560 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -234,3 +234,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index a479216209c..0a1feb682dc 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -231,3 +231,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index d071e395be4..ed4c59bcbb4 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -233,3 +233,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 From b9a60e444f24af2a059320906c0e929bee05f738 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:34:57 +0800 Subject: [PATCH 03/14] docs: rewrite virtual thread notes for users --- home/docs/start/virtual-thread.md | 4 ++-- .../current/start/virtual-thread.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index d28da0f3ca0..89fb0094585 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -111,5 +111,5 @@ This rolls the affected executors back to their legacy platform-thread implement ## 7. Notes -- You do not need to add `spring.threads.virtual.enabled` for this feature set. HertzBeat uses a dedicated `@Async` executor configuration. -- Shipping YAML files now include the optional `hertzbeat.vthreads` block as an example, but the block is still optional. +- If your current deployment is stable, you can keep your existing `application.yml` unchanged. +- Add the `hertzbeat.vthreads` block only when you want to tune concurrency limits or explicitly disable the feature. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index a8c4c1ba4da..bcdc21d55c9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -111,5 +111,5 @@ hertzbeat: ## 7. 补充说明 -- 这套改造不要求额外配置 `spring.threads.virtual.enabled`。HertzBeat 当前使用的是单独的 `@Async` 执行器配置。 -- 交付给用户的 YAML 示例文件现在已经带上了可选的 `hertzbeat.vthreads` 配置块，但这段配置依然可以整体删除。 +- 如果你当前的部署运行稳定，可以继续保持现有 `application.yml` 不变。 +- 只有在你希望调节并发阈值，或者显式关闭该能力时，才需要增加 `hertzbeat.vthreads` 配置块。 From 0404c2bff6b9e07fccea0e975cf9a4473f1e22f1 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:41:22 +0800 Subject: [PATCH 04/14] feat: raise collector virtual thread concurrency default --- .../common/config/VirtualThreadProperties.java | 5 +++-- .../common/config/VirtualThreadPropertiesTest.java | 2 +- home/docs/help/issue.md | 11 +++++------ home/docs/start/virtual-thread.md | 4 +++- .../current/help/issue.md | 11 +++++------ .../current/start/virtual-thread.md | 4 +++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index f8590dee3ac..fde25f1e9fe 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -30,6 +30,8 @@ @ConfigurationProperties(prefix = "hertzbeat.vthreads") public class VirtualThreadProperties { + private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; + private boolean enabled = true; private PoolProperties collector = PoolProperties.collectorDefaults(); @@ -94,8 +96,7 @@ private static PoolProperties alerterNotifyDefaults() { } private static int defaultCollectorConcurrency() { - int historicalMax = Runtime.getRuntime().availableProcessors() * 16; - return Math.max(1, historicalMax - 1); + return DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS; } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index 7433ae3944c..a6b32e41da9 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -32,7 +32,7 @@ void defaultsRemainSafeWithoutExternalConfiguration() { assertTrue(properties.isEnabled()); assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); - assertTrue(properties.getCollector().getMaxConcurrentJobs() >= 1); + assertEquals(512, properties.getCollector().getMaxConcurrentJobs()); assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); diff --git a/home/docs/help/issue.md b/home/docs/help/issue.md index 0eae3fca8df..5c6713e23de 100644 --- a/home/docs/help/issue.md +++ b/home/docs/help/issue.md @@ -32,12 +32,11 @@ sidebar_label: Common issues 6. What is the task limit for a single collector? - > Specific limit parameters: - Core thread count: Math.max(2, Runtime.getRuntime().availableProcessors()) – at least 2 threads, or equal to the number of CPU cores. - Maximum thread count: Runtime.getRuntime().availableProcessors() * 16 – 16 times the number of CPU cores. - > The limit depends entirely on the server's CPU core count. For example, on an 8-core CPU server, a maximum of 8 × 16 = 128 collection tasks can be processed simultaneously. Exceeding this number triggers the error message. This is a dynamic configuration that adjusts automatically based on the hardware specifications of the runtime environment. - > If the runtime exceeds the maximum thread count, an error will appear: "the worker pool is full, reject this metrics task, put in queue again". - > In such cases, it is recommended to configure a new collector in public mode. HertzBeat will automatically distribute tasks to other collectors, avoiding errors due to the task limit of a single collector. + > In current versions, the default collector concurrency limit is `512` concurrent collection tasks when virtual threads are enabled. + > This default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. + > If the runtime exceeds the configured collector limit, an error will appear: "the worker pool is full, reject this metrics task, put in queue again". + > You can tune this limit through `hertzbeat.vthreads.collector.max-concurrent-jobs` in `application.yml`. + > If a single node still cannot absorb the workload, configure additional collectors in public mode so HertzBeat can distribute tasks across them. ### Docker Deployment common issues diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index 89fb0094585..6de95e1dbcb 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | Computed at runtime; no YAML update required | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Default single-node collector concurrency target | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | @@ -92,7 +92,9 @@ hertzbeat: ## 5. Tuning Guidance - Start with the defaults unless you already know a downstream dependency is weak. +- The collector default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. - Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. +- Raise `collector.max-concurrent-jobs` above `512` on dedicated collector nodes when downstream services, network bandwidth, and timeouts are already understood and controlled. - Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. - Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. - `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md index 792264ed0f6..c92f3279808 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md @@ -32,12 +32,11 @@ sidebar_label: 常见问题 6. 单个采集器的任务上限是多少? - > 具体上限参数 - 核心线程数: Math.max(2, Runtime.getRuntime().availableProcessors()) - 至少2个线程，或等于CPU核心数。 - 最大线程数: Runtime.getRuntime().availableProcessors() * 16 - CPU核心数的16倍。 - > 上限完全取决于服务器的CPU核心数。例如，在8核CPU的服务器上，最大可同时处理 8 × 16 = 128 个采集任务。当超过这个数量时就会触发该错误消息。这是一个动态配置，会根据运行环境的硬件规格自动调整。 - > 当运行时超出最大线程数会报错提示"the worker pool is full, reject this metrics task，put in queue again"。 - > 此时建议配置新的采集器,并设置为public模式，hertzbeat会自动将任务分配给其他采集器，不会因为单个采集器任务上限而报错。 + > 在当前版本中，启用虚拟线程后，单个 collector 默认可并发执行 `512` 个采集任务。 + > 这个默认值刻意高于旧版按 CPU 推导出来的线程池上限，目的是让单独部署的 HertzBeat 节点先尽量承载更多阻塞型采集任务，再决定是否扩容额外 collector。 + > 当运行时超出已配置的 collector 并发上限时，会报错提示 "the worker pool is full, reject this metrics task，put in queue again"。 + > 你可以在 `application.yml` 里通过 `hertzbeat.vthreads.collector.max-concurrent-jobs` 调整这个限制。 + > 如果单机仍然无法承载当前任务量，再建议增加新的 collector，并设置为 public 模式，让 HertzBeat 自动做任务分发。 ### Docker部署常见问题 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index bcdc21d55c9..7731c105740 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | 运行时动态计算，不依赖 YAML | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 单机默认采集并发目标值 | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | @@ -92,7 +92,9 @@ hertzbeat: ## 5. 调优建议 - 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 +- collector 默认值刻意高于旧版按 CPU 推导的线程池上限，这样单独部署 HertzBeat 主程序时可以承载更多阻塞型采集任务，减少对额外 collector 的依赖。 - 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 +- 对于专门部署的 collector 节点，如果你已经清楚下游容量、网络带宽和超时设置，也可以把 `collector.max-concurrent-jobs` 提高到 `512` 以上。 - 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 - `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 - `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 From 61c9b00aea35fe26e40e5a4538d3d960a837937e Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 15:23:03 +0800 Subject: [PATCH 05/14] docs: add virtual thread tuning guidance --- home/docs/start/virtual-thread.md | 9 +++++++-- .../current/start/virtual-thread.md | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index 6de95e1dbcb..f9d36190316 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Default single-node collector concurrency target | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Balanced default for mixed HTTP and JDBC collection workloads on a single node | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | @@ -93,11 +93,16 @@ hertzbeat: - Start with the defaults unless you already know a downstream dependency is weak. - The collector default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. +- `512` is the default because it is a good mixed-workload starting point. In local verification, HTTP-heavy collection continued scaling beyond `512`, while JDBC-style collection peaked around `512` and dropped when concurrency was pushed higher. +- Virtual threads remove platform-thread pressure, but they do not remove database limits, HTTP connection limits, network bandwidth limits, file descriptor limits, or downstream rate limits. Raising concurrency too far just moves the bottleneck. +- If most of your workload is HTTP collection across many different targets, try `768` first and then `1024` if timeouts, error rates, and connection usage remain stable. +- If most of your workload is JDBC or other database-backed collection, keep `collector.max-concurrent-jobs` around `256` to `512`. In this type of workload, raising concurrency above `512` can reduce total throughput instead of improving it. +- If you are not sure about the workload mix, keep `512` as the starting point. It is a safer default than `768+` for mixed environments. - Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. -- Raise `collector.max-concurrent-jobs` above `512` on dedicated collector nodes when downstream services, network bandwidth, and timeouts are already understood and controlled. - Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. - Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. - `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. +- Change concurrency in steps and observe timeout rate, downstream `429` or `5xx`, database pool wait time, and memory or file descriptor usage before raising it again. ## 6. Rollback diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index 7731c105740..ae69b63e0a9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 单机默认采集并发目标值 | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 面向单机混合 HTTP/JDBC 采集场景的折中默认值 | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | @@ -93,11 +93,16 @@ hertzbeat: - 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 - collector 默认值刻意高于旧版按 CPU 推导的线程池上限，这样单独部署 HertzBeat 主程序时可以承载更多阻塞型采集任务，减少对额外 collector 的依赖。 +- 默认值之所以设为 `512`，是因为它更适合作为混合负载起点。我们本地验证时，HTTP 型采集在 `512` 以上还能继续扩展，而 JDBC 型采集在接近 `512` 时已经接近甜点，继续提高并发反而会掉总吞吐。 +- 虚拟线程解决的是平台线程成本问题，不会消除数据库连接上限、HTTP 连接上限、网络带宽、文件描述符或下游服务限流。并发调得过高，只是把瓶颈转移到这些资源上。 +- 如果你的采集任务大多是 HTTP，而且目标分散在很多不同主机上，可以先尝试把 `collector.max-concurrent-jobs` 提高到 `768`，稳定后再考虑 `1024`。 +- 如果你的采集任务大多是 JDBC 或其他数据库型采集，建议把 `collector.max-concurrent-jobs` 控制在 `256` 到 `512`。这类场景下，并发超过 `512` 后不一定更快。 +- 如果你暂时不确定负载结构，就先保持 `512`。对于混合场景，它通常比直接上 `768+` 更稳。 - 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 -- 对于专门部署的 collector 节点，如果你已经清楚下游容量、网络带宽和超时设置，也可以把 `collector.max-concurrent-jobs` 提高到 `512` 以上。 - 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 - `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 - `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 +- 每次调高并发时都建议按小步递增，并同时观察超时率、下游 `429/5xx`、数据库连接池等待时间、内存和文件描述符使用情况。 ## 6. 回滚方式 From 9a62f596ab94ede38f5ab223945214a5cf820173 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 15:43:32 +0800 Subject: [PATCH 06/14] refactor: convert virtual thread properties to records --- .../hertzbeat/alert/AlerterWorkerPool.java | 16 +- .../periodic/PeriodicAlertRuleScheduler.java | 4 +- .../realtime/window/AlarmEvaluator.java | 6 +- .../alert/reduce/AlarmCommonReduce.java | 6 +- .../alert/reduce/AlarmGroupReduce.java | 2 +- .../alert/reduce/AlarmInhibitReduce.java | 2 +- .../alert/AlerterWorkerPoolTest.java | 65 ++++--- .../PeriodicAlertRuleSchedulerTest.java | 17 +- .../realtime/window/AlarmEvaluatorTest.java | 18 +- .../alert/reduce/AlarmCommonReduceTest.java | 18 +- .../collector/dispatch/WorkerPoolTest.java | 13 +- .../collector/dispatch/WorkerPool.java | 8 +- .../dispatch/entrance/CollectServer.java | 2 +- .../config/VirtualThreadProperties.java | 167 +++++++++--------- .../common/support/CommonThreadPool.java | 8 +- .../config/VirtualThreadPropertiesTest.java | 52 +++--- .../common/support/CommonThreadPoolTest.java | 13 +- .../component/status/CalculateStatus.java | 2 +- .../manager/scheduler/ManagerWorkerPool.java | 8 +- .../manager/scheduler/netty/ManageServer.java | 2 +- .../scheduler/ManagerWorkerPoolTest.java | 13 +- .../apache/hertzbeat/startup/AsyncConfig.java | 10 +- .../hertzbeat/startup/AsyncConfigTest.java | 11 +- .../warehouse/WarehouseWorkerPool.java | 8 +- .../duckdb/DuckdbDatabaseDataStorage.java | 2 +- .../warehouse/WarehouseWorkerPoolTest.java | 13 +- 26 files changed, 279 insertions(+), 207 deletions(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java index b6b743b8a2e..b8abee9edf7 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java @@ -83,13 +83,13 @@ private void initNotifyExecutor(VirtualThreadProperties properties) { log.error("Alerter notifyExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.AlerterProperties alerterProperties = properties.getAlerter(); - VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.getNotify(); - notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.getNotifyMaxConcurrentPerChannel()); + if (properties.enabled()) { + VirtualThreadProperties.AlerterProperties alerterProperties = properties.alerter(); + VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.notifyPool(); + notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.notifyMaxConcurrentPerChannel()); notifyChannelPermits = new ConcurrentHashMap<>(8); notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-", - notifyProperties.getMode(), notifyProperties.getMaxConcurrentJobs(), handler); + notifyProperties.mode(), notifyProperties.maxConcurrentJobs(), handler); return; } notifyMaxConcurrentPerChannel = 0; @@ -117,10 +117,10 @@ private void initLogWorkerExecutor(VirtualThreadProperties properties) { log.error("Alerter logWorkerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties logWorkerProperties = properties.getAlerter().getLogWorker(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties logWorkerProperties = properties.alerter().logWorker(); logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-", - logWorkerProperties.getMaxConcurrentJobs(), logWorkerProperties.getQueueCapacity(), handler); + logWorkerProperties.maxConcurrentJobs(), logWorkerProperties.queueCapacity(), handler); return; } logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler)); diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 9ad94c4d4dd..49e9004e77d 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -85,8 +85,8 @@ public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculat this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - this.virtualThreadsEnabled = properties.isEnabled(); - int maxConcurrentPeriodicTasks = Math.max(1, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + this.virtualThreadsEnabled = properties.enabled(); + int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs()); this.periodicExecutor = virtualThreadsEnabled ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() .name("periodic-alert-task-", 0) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java index 1d210ac47c8..e85b145355f 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java @@ -74,10 +74,10 @@ public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) { log.error("alerter-reduce-worker has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getWindowEvaluator(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().windowEvaluator(); return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-", - queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); } return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2, 10, diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 4d6b2edb0e5..75f18f9ee37 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -58,10 +58,10 @@ private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { log.error("alerter-reduce-worker has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getReduce(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().reduce(); return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-", - queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); } return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2, 2, diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index e43b2e43bcb..597ef9a58a6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -159,7 +159,7 @@ private ScheduledExecutorService createScheduler() { } private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index c8a51392692..4ebfdd22bf6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -151,7 +151,7 @@ private ScheduledExecutorService createCleanupScheduler() { } private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java index 434a999eb65..ba83880959d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java @@ -82,14 +82,20 @@ void executeNotifyRunsOnVirtualThread() throws Exception { @Test void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); - notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - notifyProperties.setMaxConcurrentJobs(1); - alerterProperties.setNotify(notifyProperties); - alerterProperties.setNotifyMaxConcurrentPerChannel(8); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 8), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); @@ -114,14 +120,20 @@ void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { @Test void executeNotifyRejectsWhenChannelLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); - notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - notifyProperties.setMaxConcurrentJobs(8); - alerterProperties.setNotify(notifyProperties); - alerterProperties.setNotifyMaxConcurrentPerChannel(1); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 8), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 1), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); @@ -160,13 +172,20 @@ void executeLogJob() throws InterruptedException { @Test void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.QueueProperties logWorkerProperties = new VirtualThreadProperties.QueueProperties(); - logWorkerProperties.setMaxConcurrentJobs(1); - logWorkerProperties.setQueueCapacity(1); - alerterProperties.setLogWorker(logWorkerProperties); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + new VirtualThreadProperties.QueueProperties(1, 1), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch firstStarted = new CountDownLatch(1); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java index a21f66fc2a8..a18b8113e7d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -195,8 +195,19 @@ private AlertDefine metricRule(Long id) { } private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { - VirtualThreadProperties properties = VirtualThreadProperties.defaults(); - properties.getAlerter().setPeriodicMaxConcurrentJobs(maxConcurrentJobs); - return properties; + return new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + maxConcurrentJobs, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java index 394b9b77d02..d520821bec7 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java @@ -378,10 +378,20 @@ void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception { @Test void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); - queueProperties.setMaxConcurrentJobs(1); - properties.getAlerter().setWindowEvaluator(queueProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); alarmEvaluator.destroy(); alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index 8dae8999511..02081885293 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -87,10 +87,20 @@ void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception { @Test void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); - queueProperties.setMaxConcurrentJobs(1); - properties.getAlerter().setReduce(queueProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); alarmCommonReduce.destroy(); alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties); diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java index e4d2aac786d..2348dc1cf34 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java @@ -59,11 +59,14 @@ void testExecuteJobRunsOnVirtualThread() throws Exception { @Test void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties collectorProperties = new VirtualThreadProperties.PoolProperties(); - collectorProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - collectorProperties.setMaxConcurrentJobs(1); - properties.setCollector(collectorProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); workerPool = new WorkerPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java index 7ba179ce40d..a42cc9a4283 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getCollector(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.collector(); return ManagedExecutors.newVirtualExecutor("collector-worker", "collect-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("collector-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("collector-long-running", "collect-long-running-", diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 112209c8b76..1e77da0e211 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -217,7 +217,7 @@ public void onChannelIdle(Channel channel) { private ExecutorService createHeartbeatExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index fde25f1e9fe..8fdbc9c51d1 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -17,34 +17,40 @@ package org.apache.hertzbeat.common.config; -import lombok.Getter; -import lombok.Setter; import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; +import org.springframework.boot.context.properties.bind.Name; /** * Virtual-thread related configuration. */ -@Getter -@Setter @ConfigurationProperties(prefix = "hertzbeat.vthreads") -public class VirtualThreadProperties { +public record VirtualThreadProperties( + @DefaultValue("true") boolean enabled, + @DefaultValue PoolProperties collector, + @DefaultValue PoolProperties common, + @DefaultValue PoolProperties manager, + @DefaultValue AlerterProperties alerter, + @DefaultValue PoolProperties warehouse, + @DefaultValue AsyncProperties async) { private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; - private boolean enabled = true; - - private PoolProperties collector = PoolProperties.collectorDefaults(); - - private PoolProperties common = PoolProperties.commonDefaults(); - - private PoolProperties manager = PoolProperties.managerDefaults(); - - private AlerterProperties alerter = new AlerterProperties(); - - private PoolProperties warehouse = PoolProperties.warehouseDefaults(); + public VirtualThreadProperties { + collector = collector == null ? PoolProperties.collectorDefaults() : collector; + common = common == null ? PoolProperties.commonDefaults() : common; + manager = manager == null ? PoolProperties.managerDefaults() : manager; + alerter = alerter == null ? AlerterProperties.defaults() : alerter; + warehouse = warehouse == null ? PoolProperties.warehouseDefaults() : warehouse; + async = async == null ? AsyncProperties.defaults() : async; + } - private AsyncProperties async = new AsyncProperties(); + public VirtualThreadProperties() { + this(true, PoolProperties.collectorDefaults(), PoolProperties.commonDefaults(), + PoolProperties.managerDefaults(), AlerterProperties.defaults(), + PoolProperties.warehouseDefaults(), AsyncProperties.defaults()); + } /** * Create a detached properties instance with runtime defaults. @@ -58,41 +64,36 @@ public static VirtualThreadProperties defaults() { /** * Pool-level configuration. */ - @Getter - @Setter - public static class PoolProperties { + public record PoolProperties( + @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, + @DefaultValue("0") int maxConcurrentJobs) { - private AdmissionMode mode = AdmissionMode.UNBOUNDED_VT; + public PoolProperties { + mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; + } - private int maxConcurrentJobs; + public PoolProperties() { + this(AdmissionMode.UNBOUNDED_VT, 0); + } - private static PoolProperties collectorDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(defaultCollectorConcurrency()); - return properties; + public static PoolProperties collectorDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, defaultCollectorConcurrency()); } - private static PoolProperties warehouseDefaults() { + public static PoolProperties warehouseDefaults() { return new PoolProperties(); } - private static PoolProperties commonDefaults() { + public static PoolProperties commonDefaults() { return new PoolProperties(); } - private static PoolProperties managerDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(10); - return properties; + public static PoolProperties managerDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 10); } - private static PoolProperties alerterNotifyDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(64); - return properties; + public static PoolProperties alerterNotifyDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 64); } private static int defaultCollectorConcurrency() { @@ -103,67 +104,71 @@ private static int defaultCollectorConcurrency() { /** * Alerter-specific executor configuration. */ - @Getter - @Setter - public static class AlerterProperties { - - private PoolProperties notify = PoolProperties.alerterNotifyDefaults(); - - private int periodicMaxConcurrentJobs = 10; - - private QueueProperties logWorker = QueueProperties.logWorkerDefaults(); - - private QueueProperties reduce = QueueProperties.reduceDefaults(); + public record AlerterProperties( + @Name("notify") @DefaultValue PoolProperties notifyPool, + @DefaultValue("10") int periodicMaxConcurrentJobs, + @DefaultValue QueueProperties logWorker, + @DefaultValue QueueProperties reduce, + @DefaultValue QueueProperties windowEvaluator, + @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + + public AlerterProperties { + notifyPool = notifyPool == null ? PoolProperties.alerterNotifyDefaults() : notifyPool; + logWorker = logWorker == null ? QueueProperties.logWorkerDefaults() : logWorker; + reduce = reduce == null ? QueueProperties.reduceDefaults() : reduce; + windowEvaluator = windowEvaluator == null ? QueueProperties.windowEvaluatorDefaults() : windowEvaluator; + } - private QueueProperties windowEvaluator = QueueProperties.windowEvaluatorDefaults(); + public AlerterProperties() { + this(PoolProperties.alerterNotifyDefaults(), 10, + QueueProperties.logWorkerDefaults(), QueueProperties.reduceDefaults(), + QueueProperties.windowEvaluatorDefaults(), 4); + } - private int notifyMaxConcurrentPerChannel = 4; + public static AlerterProperties defaults() { + return new AlerterProperties(); + } } /** * Queue-preserving executor configuration. */ - @Getter - @Setter - public static class QueueProperties { - - private int maxConcurrentJobs; + public record QueueProperties( + @DefaultValue("0") int maxConcurrentJobs, + @DefaultValue("0") int queueCapacity) { - private int queueCapacity; + public QueueProperties() { + this(0, 0); + } - private static QueueProperties reduceDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(2); - return properties; + public static QueueProperties reduceDefaults() { + return new QueueProperties(2, 0); } - private static QueueProperties logWorkerDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(10); - properties.setQueueCapacity(1000); - return properties; + public static QueueProperties logWorkerDefaults() { + return new QueueProperties(10, 1000); } - private static QueueProperties windowEvaluatorDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(2); - return properties; + public static QueueProperties windowEvaluatorDefaults() { + return new QueueProperties(2, 0); } } /** * Async executor configuration. */ - @Getter - @Setter - public static class AsyncProperties { - - private boolean enabled = true; - - private int concurrencyLimit = 256; - - private boolean rejectWhenLimitReached = true; + public record AsyncProperties( + @DefaultValue("true") boolean enabled, + @DefaultValue("256") int concurrencyLimit, + @DefaultValue("true") boolean rejectWhenLimitReached, + @DefaultValue("5000") long taskTerminationTimeout) { + + public AsyncProperties() { + this(true, 256, true, 5000L); + } - private long taskTerminationTimeout = 5000L; + public static AsyncProperties defaults() { + return new AsyncProperties(); + } } } diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java index a753dccb2d5..5ae508740d1 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("common executor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getCommon(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.common(); return ManagedExecutors.newVirtualExecutor("common-worker", "common-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("common-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("common-long-running", "common-long-running-", diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index a6b32e41da9..e468bfbd192 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -29,31 +29,31 @@ class VirtualThreadPropertiesTest { void defaultsRemainSafeWithoutExternalConfiguration() { VirtualThreadProperties properties = VirtualThreadProperties.defaults(); - assertTrue(properties.isEnabled()); - - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); - assertEquals(512, properties.getCollector().getMaxConcurrentJobs()); - - assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); - assertEquals(10, properties.getManager().getMaxConcurrentJobs()); - - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getAlerter().getNotify().getMode()); - assertEquals(64, properties.getAlerter().getNotify().getMaxConcurrentJobs()); - assertEquals(10, properties.getAlerter().getPeriodicMaxConcurrentJobs()); - assertEquals(10, properties.getAlerter().getLogWorker().getMaxConcurrentJobs()); - assertEquals(1000, properties.getAlerter().getLogWorker().getQueueCapacity()); - assertEquals(2, properties.getAlerter().getReduce().getMaxConcurrentJobs()); - assertEquals(0, properties.getAlerter().getReduce().getQueueCapacity()); - assertEquals(2, properties.getAlerter().getWindowEvaluator().getMaxConcurrentJobs()); - assertEquals(0, properties.getAlerter().getWindowEvaluator().getQueueCapacity()); - assertEquals(4, properties.getAlerter().getNotifyMaxConcurrentPerChannel()); - - assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getWarehouse().getMode()); - - assertTrue(properties.getAsync().isEnabled()); - assertEquals(256, properties.getAsync().getConcurrencyLimit()); - assertTrue(properties.getAsync().isRejectWhenLimitReached()); - assertEquals(5000L, properties.getAsync().getTaskTerminationTimeout()); + assertTrue(properties.enabled()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.common().mode()); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.manager().mode()); + assertEquals(10, properties.manager().maxConcurrentJobs()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.alerter().notifyPool().mode()); + assertEquals(64, properties.alerter().notifyPool().maxConcurrentJobs()); + assertEquals(10, properties.alerter().periodicMaxConcurrentJobs()); + assertEquals(10, properties.alerter().logWorker().maxConcurrentJobs()); + assertEquals(1000, properties.alerter().logWorker().queueCapacity()); + assertEquals(2, properties.alerter().reduce().maxConcurrentJobs()); + assertEquals(0, properties.alerter().reduce().queueCapacity()); + assertEquals(2, properties.alerter().windowEvaluator().maxConcurrentJobs()); + assertEquals(0, properties.alerter().windowEvaluator().queueCapacity()); + assertEquals(4, properties.alerter().notifyMaxConcurrentPerChannel()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.warehouse().mode()); + + assertTrue(properties.async().enabled()); + assertEquals(256, properties.async().concurrencyLimit()); + assertTrue(properties.async().rejectWhenLimitReached()); + assertEquals(5000L, properties.async().taskTerminationTimeout()); } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java index 063fdf73225..160eab430d5 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java @@ -76,11 +76,14 @@ void testExecuteLongRunningRunsOnPlatformThread() throws Exception { @Test void testExecuteRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties commonProperties = new VirtualThreadProperties.PoolProperties(); - commonProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - commonProperties.setMaxConcurrentJobs(1); - properties.setCommon(commonProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); commonThreadPool = new CommonThreadPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 6639e5674c7..80590944272 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -308,7 +308,7 @@ private ExecutorService createVirtualExecutor(VirtualThreadProperties virtualThr String errorMessage) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java index baefdcdbeb7..0c516377089 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java @@ -59,16 +59,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("workerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getManager(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.manager(); return ManagedExecutors.newVirtualExecutor("manager-worker", "manager-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("manager-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("manager-long-running", "manager-long-running-", diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 7fb26a974d6..275eb4ebf1d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -233,7 +233,7 @@ public void onChannelIdle(Channel channel) { private ExecutorService createChannelCheckExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java index 35757bf5afc..b15d192414c 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java @@ -61,11 +61,14 @@ void testExecuteJobRunsOnVirtualThread() throws Exception { @Test void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties managerProperties = new VirtualThreadProperties.PoolProperties(); - managerProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - managerProperties.setMaxConcurrentJobs(1); - properties.setManager(managerProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); managerWorkerPool = new ManagerWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java index 28b0074c1d4..5fdfedbb132 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java @@ -30,12 +30,12 @@ public class AsyncConfig { @Bean(name = "taskExecutor", destroyMethod = "close") public SimpleAsyncTaskExecutor taskExecutor(VirtualThreadProperties virtualThreadProperties) { - VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.getAsync(); + VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.async(); SimpleAsyncTaskExecutor executor = new SimpleAsyncTaskExecutor("async-worker-"); - executor.setVirtualThreads(virtualThreadProperties.isEnabled() && asyncProperties.isEnabled()); - executor.setConcurrencyLimit(asyncProperties.getConcurrencyLimit()); - executor.setRejectTasksWhenLimitReached(asyncProperties.isRejectWhenLimitReached()); - executor.setTaskTerminationTimeout(asyncProperties.getTaskTerminationTimeout()); + executor.setVirtualThreads(virtualThreadProperties.enabled() && asyncProperties.enabled()); + executor.setConcurrencyLimit(asyncProperties.concurrencyLimit()); + executor.setRejectTasksWhenLimitReached(asyncProperties.rejectWhenLimitReached()); + executor.setTaskTerminationTimeout(asyncProperties.taskTerminationTimeout()); return executor; } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java index 00d6e2d1052..da0eaae1105 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java @@ -54,9 +54,14 @@ void taskExecutorRunsAsyncTasksOnVirtualThreads() throws Exception { @Test void taskExecutorRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - properties.getAsync().setConcurrencyLimit(1); - properties.getAsync().setRejectWhenLimitReached(true); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + new VirtualThreadProperties.AsyncProperties(true, 1, true, 5000L)); try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java index af4eb5172ee..43c6fbe2a74 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("Warehouse workerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getWarehouse(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.warehouse(); return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-", diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java index 17ec3b60bfd..01cb7c04291 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java @@ -437,7 +437,7 @@ void beforeExpiredDataCleanerRun() { private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java index c014ae24c57..ddcb449bdfe 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java @@ -78,11 +78,14 @@ void executeJobRunsOnVirtualThread() throws InterruptedException { @Test void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties warehouseProperties = new VirtualThreadProperties.PoolProperties(); - warehouseProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - warehouseProperties.setMaxConcurrentJobs(1); - properties.setWarehouse(warehouseProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new WarehouseWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); From f7d0bb3d3d74fca5828ca7d1e043224d7b2cdef1 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 17:16:10 +0800 Subject: [PATCH 07/14] fix: restore record-based virtual thread binding --- .../config/VirtualThreadProperties.java | 83 ++++++++++++++----- .../config/VirtualThreadPropertiesTest.java | 49 +++++++++++ 2 files changed, 112 insertions(+), 20 deletions(-) diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index 8fdbc9c51d1..0962e608ad3 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -19,6 +19,7 @@ import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.ConstructorBinding; import org.springframework.boot.context.properties.bind.DefaultValue; import org.springframework.boot.context.properties.bind.Name; @@ -28,19 +29,24 @@ @ConfigurationProperties(prefix = "hertzbeat.vthreads") public record VirtualThreadProperties( @DefaultValue("true") boolean enabled, - @DefaultValue PoolProperties collector, - @DefaultValue PoolProperties common, - @DefaultValue PoolProperties manager, - @DefaultValue AlerterProperties alerter, - @DefaultValue PoolProperties warehouse, - @DefaultValue AsyncProperties async) { + PoolProperties collector, + PoolProperties common, + PoolProperties manager, + AlerterProperties alerter, + PoolProperties warehouse, + AsyncProperties async) { private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; + private static final int DEFAULT_MANAGER_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS = 64; + private static final int DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL = 4; + @ConstructorBinding public VirtualThreadProperties { - collector = collector == null ? PoolProperties.collectorDefaults() : collector; + collector = normalizePool(collector, PoolProperties.collectorDefaults()); common = common == null ? PoolProperties.commonDefaults() : common; - manager = manager == null ? PoolProperties.managerDefaults() : manager; + manager = normalizePool(manager, PoolProperties.managerDefaults()); alerter = alerter == null ? AlerterProperties.defaults() : alerter; warehouse = warehouse == null ? PoolProperties.warehouseDefaults() : warehouse; async = async == null ? AsyncProperties.defaults() : async; @@ -68,6 +74,7 @@ public record PoolProperties( @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, @DefaultValue("0") int maxConcurrentJobs) { + @ConstructorBinding public PoolProperties { mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; } @@ -89,11 +96,11 @@ public static PoolProperties commonDefaults() { } public static PoolProperties managerDefaults() { - return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 10); + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_MANAGER_MAX_CONCURRENT_JOBS); } public static PoolProperties alerterNotifyDefaults() { - return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 64); + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS); } private static int defaultCollectorConcurrency() { @@ -105,24 +112,29 @@ private static int defaultCollectorConcurrency() { * Alerter-specific executor configuration. */ public record AlerterProperties( - @Name("notify") @DefaultValue PoolProperties notifyPool, + @Name("notify") PoolProperties notifyPool, @DefaultValue("10") int periodicMaxConcurrentJobs, - @DefaultValue QueueProperties logWorker, - @DefaultValue QueueProperties reduce, - @DefaultValue QueueProperties windowEvaluator, + QueueProperties logWorker, + QueueProperties reduce, + QueueProperties windowEvaluator, @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + @ConstructorBinding public AlerterProperties { - notifyPool = notifyPool == null ? PoolProperties.alerterNotifyDefaults() : notifyPool; - logWorker = logWorker == null ? QueueProperties.logWorkerDefaults() : logWorker; - reduce = reduce == null ? QueueProperties.reduceDefaults() : reduce; - windowEvaluator = windowEvaluator == null ? QueueProperties.windowEvaluatorDefaults() : windowEvaluator; + notifyPool = normalizePool(notifyPool, PoolProperties.alerterNotifyDefaults()); + periodicMaxConcurrentJobs = periodicMaxConcurrentJobs <= 0 + ? DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS : periodicMaxConcurrentJobs; + logWorker = normalizeQueue(logWorker, QueueProperties.logWorkerDefaults()); + reduce = normalizeQueue(reduce, QueueProperties.reduceDefaults()); + windowEvaluator = normalizeQueue(windowEvaluator, QueueProperties.windowEvaluatorDefaults()); + notifyMaxConcurrentPerChannel = notifyMaxConcurrentPerChannel <= 0 + ? DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL : notifyMaxConcurrentPerChannel; } public AlerterProperties() { - this(PoolProperties.alerterNotifyDefaults(), 10, + this(PoolProperties.alerterNotifyDefaults(), DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS, QueueProperties.logWorkerDefaults(), QueueProperties.reduceDefaults(), - QueueProperties.windowEvaluatorDefaults(), 4); + QueueProperties.windowEvaluatorDefaults(), DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL); } public static AlerterProperties defaults() { @@ -137,6 +149,10 @@ public record QueueProperties( @DefaultValue("0") int maxConcurrentJobs, @DefaultValue("0") int queueCapacity) { + @ConstructorBinding + public QueueProperties { + } + public QueueProperties() { this(0, 0); } @@ -163,6 +179,10 @@ public record AsyncProperties( @DefaultValue("true") boolean rejectWhenLimitReached, @DefaultValue("5000") long taskTerminationTimeout) { + @ConstructorBinding + public AsyncProperties { + } + public AsyncProperties() { this(true, 256, true, 5000L); } @@ -171,4 +191,27 @@ public static AsyncProperties defaults() { return new AsyncProperties(); } } + + private static PoolProperties normalizePool(PoolProperties configured, PoolProperties defaults) { + if (configured == null) { + return defaults; + } + if (configured.mode() != AdmissionMode.UNBOUNDED_VT && configured.maxConcurrentJobs() <= 0) { + return new PoolProperties(configured.mode(), defaults.maxConcurrentJobs()); + } + return configured; + } + + private static QueueProperties normalizeQueue(QueueProperties configured, QueueProperties defaults) { + if (configured == null) { + return defaults; + } + int maxConcurrentJobs = configured.maxConcurrentJobs() <= 0 + ? defaults.maxConcurrentJobs() : configured.maxConcurrentJobs(); + int queueCapacity = configured.queueCapacity(); + if (queueCapacity <= 0 && defaults.queueCapacity() > 0) { + queueCapacity = defaults.queueCapacity(); + } + return new QueueProperties(maxConcurrentJobs, queueCapacity); + } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index e468bfbd192..3ead4ddca16 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -22,9 +22,14 @@ import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; class VirtualThreadPropertiesTest { + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(BindingConfig.class); + @Test void defaultsRemainSafeWithoutExternalConfiguration() { VirtualThreadProperties properties = VirtualThreadProperties.defaults(); @@ -56,4 +61,48 @@ void defaultsRemainSafeWithoutExternalConfiguration() { assertTrue(properties.async().rejectWhenLimitReached()); assertEquals(5000L, properties.async().taskTerminationTimeout()); } + + @Test + void collectorModeOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.collector.mode=LIMIT_AND_REJECT") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + }); + } + + @Test + void collectorModeOverrideUsesConfiguredMode() { + contextRunner.withPropertyValues("hertzbeat.vthreads.collector.mode=LIMIT_AND_BLOCK") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_BLOCK, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + }); + } + + @Test + void notifyModeOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.alerter.notify.mode=LIMIT_AND_BLOCK") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_BLOCK, properties.alerter().notifyPool().mode()); + assertEquals(64, properties.alerter().notifyPool().maxConcurrentJobs()); + }); + } + + @Test + void logWorkerQueueOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.alerter.log-worker.queue-capacity=32") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(10, properties.alerter().logWorker().maxConcurrentJobs()); + assertEquals(32, properties.alerter().logWorker().queueCapacity()); + }); + } + + @EnableConfigurationProperties(VirtualThreadProperties.class) + static class BindingConfig { + } } From d0306d8b4888728bb962d7b39fc5e256a47f96a0 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 22:08:02 +0800 Subject: [PATCH 08/14] refactor: update parameter handling and imports across multiple files --- .../ai/tools/impl/MonitorToolsImpl.java | 10 +- .../dispatch/entrance/CollectServerTest.java | 4 +- .../hertzbeat-collector-common/pom.xml | 4 - .../dispatch/entrance/CollectServer.java | 45 ++++- .../entrance/processor/GoCloseProcessor.java | 17 +- .../processor/GoOfflineProcessor.java | 12 +- .../entrance/processor/GoOnlineProcessor.java | 10 +- .../collector/timer/TimerDispatcher.java | 29 ++- .../collector/timer/WheelTimerTask.java | 14 +- .../processor/GoOnlineProcessorTest.java | 43 ++--- .../collect/kafka/KafkaCollectImpl.java | 19 +- .../mongodb/MongodbSingleCollectImpl.java | 19 +- .../collect/nebulagraph/NgqlCollectImpl.java | 25 +-- .../rocketmq/RocketmqSingleCollectImpl.java | 21 ++- hertzbeat-common-core/pom.xml | 1 + .../concurrent/BackgroundTaskExecutor.java | 48 +++++ .../common/config/BaseKafkaProperties.java | 0 .../config/VirtualThreadProperties.java | 42 ++--- .../common/entity/dto/sms/SmsConfig.java | 6 +- .../hertzbeat/common/entity/job/Job.java | 63 +++---- .../hertzbeat/common/entity/job/Metrics.java | 0 .../common/entity/job/RuntimeParamDefine.java | 60 ++++-- .../common/entity/job/SshTunnel.java | 4 +- .../common/entity/plugin/PluginConfig.java | 9 +- .../common/entity/push/PushMetricsDto.java | 0 .../support/ResourceBundleUtf8Control.java | 0 .../exception/AlertExpressionException.java | 2 +- .../CommonDataQueueUnknownException.java | 2 +- .../support/exception/CommonException.java | 0 .../exception/ExpressionVisitorException.java | 2 +- .../support/exception/IgnoreException.java | 0 .../exception/SendMessageException.java | 0 .../support/valid/SqlSecurityException.java | 0 .../common/util/ResourceBundleUtil.java | 0 hertzbeat-common-spring/pom.xml | 1 + .../hertzbeat/common/config/CommonConfig.java | 8 +- .../common/config/SmsConfigBinding.java | 28 +++ .../VirtualThreadPropertiesBinding.java | 177 ++++++++++++++++++ .../common/entity/job/SshTunnel.java | 87 --------- .../common/support/CommonThreadPool.java | 3 +- .../hertzbeat/common/util/IpDomainUtil.java | 169 ----------------- .../common/config/SmsConfigBindingTest.java | 50 +++++ .../config/VirtualThreadPropertiesTest.java | 8 +- .../component/validator/ParamValidator.java | 6 +- .../validator/ParamValidatorManager.java | 6 +- .../validator/impl/ArrayParamValidator.java | 6 +- .../validator/impl/BooleanParamValidator.java | 6 +- .../impl/HostParamValidatorAdapter.java | 6 +- .../validator/impl/JsonParamValidator.java | 6 +- .../validator/impl/NumberParamValidator.java | 6 +- .../validator/impl/OptionParamValidator.java | 10 +- .../impl/PasswordParamValidator.java | 6 +- .../validator/impl/TextParamValidator.java | 6 +- .../manager/controller/AppController.java | 4 +- .../controller/CollectorController.java | 2 +- .../controller/MonitorsController.java | 14 +- .../manager/controller/PluginController.java | 2 +- .../controller/StatusPageController.java | 41 ++-- .../StatusPagePublicController.java | 12 +- .../manager/pojo/dto/CollectorInfo.java | 82 ++++++++ .../manager/pojo}/dto/CollectorSummary.java | 13 +- .../manager/pojo/dto/ComponentStatus.java | 63 +++++-- .../manager/pojo/dto/MonitorDto.java | 93 +++++++-- .../manager/pojo/dto/MonitorInfo.java | 134 +++++++++++++ .../manager/pojo/dto/MonitorParam.java | 81 ++++++++ .../manager/pojo/dto/ParamDefineDto.java | 3 +- .../manager/pojo/dto/ParamDefineInfo.java | 144 ++++++++++++++ .../manager/pojo/dto/PluginParametersVO.java | 5 +- .../manager/pojo}/dto/PluginUpload.java | 5 +- .../pojo/dto/StatusPageComponentInfo.java | 95 ++++++++++ .../pojo/dto/StatusPageHistoryInfo.java | 94 ++++++++++ .../dto/StatusPageIncidentContentInfo.java | 84 +++++++++ .../pojo/dto/StatusPageIncidentInfo.java | 115 ++++++++++++ .../manager/pojo/dto/StatusPageOrgInfo.java | 99 ++++++++++ .../scheduler/CollectorJobScheduler.java | 4 +- .../manager/scheduler/SchedulerInit.java | 4 +- .../manager/scheduler/netty/ManageServer.java | 24 ++- .../CollectCyclicDataResponseProcessor.java | 9 +- ...ServiceDiscoveryDataResponseProcessor.java | 9 +- .../hertzbeat/manager/service/AppService.java | 4 +- .../manager/service/CollectorService.java | 2 +- .../manager/service/PluginService.java | 2 +- .../manager/service/StatusPageService.java | 26 +-- .../manager/service/impl/AppServiceImpl.java | 47 +++-- .../service/impl/CollectorServiceImpl.java | 6 +- .../service/impl/MonitorServiceImpl.java | 26 +-- .../service/impl/PluginServiceImpl.java | 6 +- .../service/impl/StatusPageServiceImpl.java | 66 ++++--- .../validator/ParamValidatorManagerTest.java | 12 +- .../impl/ArrayParamValidatorTest.java | 16 +- .../impl/BooleanParamValidatorTest.java | 20 +- .../impl/HostParamValidatorAdapterTest.java | 24 +-- .../impl/JsonParamValidatorTest.java | 12 +- .../impl/NumberParamValidatorTest.java | 20 +- .../impl/OptionParamValidatorTest.java | 20 +- .../impl/PasswordParamValidatorTest.java | 12 +- .../impl/TextParamValidatorTest.java | 12 +- .../manager/controller/AppControllerTest.java | 6 +- .../controller/MonitorsControllerTest.java | 25 ++- .../controller/StatusPageControllerTest.java | 45 ++--- .../StatusPagePublicControllerTest.java | 4 +- .../scheduler/netty/ManageServerTest.java | 10 +- .../manager/service/AppServiceTest.java | 11 +- .../manager/service/MonitorServiceTest.java | 101 ++++------ .../manager/service/PluginServiceTest.java | 2 +- .../service/StatusPageServiceTest.java | 29 +-- hertzbeat-push/pom.xml | 5 + hertzbeat-remoting/pom.xml | 2 +- .../remoting/netty/NettyRemotingClient.java | 8 +- .../remoting/netty/NettyRemotingServer.java | 6 +- .../remoting/RemotingServiceTest.java | 23 ++- 111 files changed, 2102 insertions(+), 849 deletions(-) create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java (84%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java (89%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java (93%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java (100%) rename hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java => hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java (56%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/plugin/PluginConfig.java (84%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/push/PushMetricsDto.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/ResourceBundleUtf8Control.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/AlertExpressionException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/CommonDataQueueUnknownException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/CommonException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/ExpressionVisitorException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/IgnoreException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/SendMessageException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/valid/SqlSecurityException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/util/ResourceBundleUtil.java (100%) create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/SmsConfigBinding.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesBinding.java delete mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/SshTunnel.java delete mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/util/IpDomainUtil.java create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/SmsConfigBindingTest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/CollectorInfo.java rename {hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity => hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo}/dto/CollectorSummary.java (88%) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/MonitorInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/MonitorParam.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ParamDefineInfo.java rename {hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity => hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo}/dto/PluginUpload.java (93%) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageComponentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageHistoryInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageIncidentContentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageIncidentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageOrgInfo.java diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java index 7495cf83eb9..41080c79851 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.ai.config.McpContextHolder; import org.apache.hertzbeat.manager.pojo.dto.MonitorDto; +import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo; import org.apache.hertzbeat.manager.service.MonitorService; import org.apache.hertzbeat.manager.service.AppService; import org.apache.hertzbeat.ai.utils.UtilityClass; @@ -32,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.Param; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import java.util.ArrayList; import java.util.List; @@ -280,7 +280,9 @@ public String addMonitor( // Validate that all required parameters for this monitor type are provided try { - MonitorDto monitorDto = MonitorDto.builder().monitor(monitor).params(paramList).build(); + MonitorDto monitorDto = new MonitorDto(); + monitorDto.setMonitor(monitor); + monitorDto.setParams(paramList); monitorService.validate(monitorDto, false); } catch (IllegalArgumentException argumentException) { if (argumentException.getMessage().contains("required")) { @@ -456,7 +458,7 @@ public String getMonitorParams( } // Get parameter definitions from app service - List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); + List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); if (paramDefines == null || paramDefines.isEmpty()) { return String.format("No parameter definitions found for monitor type '%s'. " @@ -468,7 +470,7 @@ public String getMonitorParams( response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n", app, paramDefines.size())); - for (ParamDefine paramDefine : paramDefines) { + for (ParamDefineInfo paramDefine : paramDefines) { response.append("• Field: ").append(paramDefine.getField()).append("\n"); // Add display name if available diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java index c56d53e948f..03a6977da63 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java @@ -36,9 +36,9 @@ import org.apache.hertzbeat.collector.dispatch.DispatchProperties; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -70,7 +70,7 @@ class CollectServerTest { private DispatchProperties.EntranceProperties.NettyProperties nettyProperties; @Mock - private CommonThreadPool threadPool; + private BackgroundTaskExecutor threadPool; @Mock private CollectorInfoProperties infoProperties; diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index b9a80815d3a..d4cffe23fb7 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -51,10 +51,6 @@ org.apache.hertzbeat hertzbeat-common-core - - org.apache.hertzbeat - hertzbeat-common-spring - org.apache.sshd diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 1e77da0e211..a5bf494750f 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -32,16 +32,18 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProcessor; import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.RemotingClient; import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; @@ -81,20 +83,33 @@ public class CollectServer implements CommandLineRunner { private boolean heartbeatPending; + private final Runnable closeApplicationAction; + public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, - final CommonThreadPool threadPool, + final BackgroundTaskExecutor threadPool, final CollectorInfoProperties infoProperties) { - this(collectJobService, timerDispatch, properties, threadPool, infoProperties, VirtualThreadProperties.defaults()); + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, + VirtualThreadProperties.defaults()); + } + + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final BackgroundTaskExecutor threadPool, + final CollectorInfoProperties infoProperties, + final VirtualThreadProperties virtualThreadProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, virtualThreadProperties); } @Autowired public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, - final CommonThreadPool threadPool, + final BackgroundTaskExecutor threadPool, final CollectorInfoProperties infoProperties, + final ConfigurableApplicationContext applicationContext, final VirtualThreadProperties virtualThreadProperties) { if (properties == null || properties.getEntrance() == null || properties.getEntrance().getNetty() == null) { log.error("init error, please config dispatch entrance netty props in application.yml"); @@ -109,10 +124,11 @@ public CollectServer(final CollectJobService collectJobService, this.collectJobService.setCollectServer(this); this.infoProperties = infoProperties; this.heartbeatExecutor = createHeartbeatExecutor(virtualThreadProperties); + this.closeApplicationAction = createCloseApplicationAction(applicationContext); this.init(properties, threadPool); } - private void init(final DispatchProperties properties, final CommonThreadPool threadPool) { + private void init(final DispatchProperties properties, final BackgroundTaskExecutor threadPool) { NettyClientConfig nettyClientConfig = new NettyClientConfig(); DispatchProperties.EntranceProperties.NettyProperties nettyProperties = properties.getEntrance().getNetty(); nettyClientConfig.setServerHost(nettyProperties.getManagerHost()); @@ -123,9 +139,10 @@ private void init(final DispatchProperties properties, final CommonThreadPool th this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this)); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, + new GoCloseProcessor(this, timerDispatch, closeApplicationAction)); } public void shutdown() { @@ -279,4 +296,16 @@ private void sendHeartbeat(String identity) { log.error("schedule send heartbeat to server error.{}", e.getMessage()); } } + + private Runnable createCloseApplicationAction(ConfigurableApplicationContext applicationContext) { + if (applicationContext == null) { + return () -> {}; + } + return () -> { + SpringApplication.exit(applicationContext, () -> 0); + if (applicationContext.isActive()) { + applicationContext.close(); + } + }; + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java index 4b068f1e45f..b259af80313 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java @@ -23,9 +23,7 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; -import org.springframework.boot.SpringApplication; /** * handle collector close message @@ -34,24 +32,25 @@ @Slf4j public class GoCloseProcessor implements NettyRemotingProcessor { private final CollectServer collectServer; - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + private final Runnable closeApplicationAction; - public GoCloseProcessor(final CollectServer collectServer) { + public GoCloseProcessor(final CollectServer collectServer, + final TimerDispatch timerDispatch, + final Runnable closeApplicationAction) { this.collectServer = collectServer; + this.timerDispatch = timerDispatch; + this.closeApplicationAction = closeApplicationAction; } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg()); } this.timerDispatch.goOffline(); this.collectServer.shutdown(); - SpringApplication.exit(SpringContextHolder.getApplicationContext(), () -> 0); - SpringContextHolder.shutdown(); + closeApplicationAction.run(); log.info("receive offline message and close success"); return null; } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java index aad85194ee4..ccacf943e53 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java @@ -23,7 +23,6 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; /** @@ -32,14 +31,15 @@ */ @Slf4j public class GoOfflineProcessor implements NettyRemotingProcessor { - - private TimerDispatch timerDispatch; + + private final TimerDispatch timerDispatch; + + public GoOfflineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } timerDispatch.goOffline(); log.info("receive offline message and handle success"); if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java index acebb42e3ad..e35c66a151b 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java @@ -24,7 +24,6 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.dto.ServerInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.AesUtil; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; @@ -36,13 +35,14 @@ @Slf4j public class GoOnlineProcessor implements NettyRemotingProcessor { - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + + public GoOnlineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().isEmpty()) { log.warn("The message that server response to collector is empty, please upgrade server"); } else { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java index 4aa6e4c0250..a8d6a600021 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java @@ -25,9 +25,11 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.constants.ScheduleTypeEnum; +import org.apache.hertzbeat.collector.dispatch.MetricsTaskDispatch; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectResponseEventListener; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; @@ -35,6 +37,8 @@ import org.apache.hertzbeat.common.timer.HashedWheelTimer; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.Timer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.DisposableBean; import org.springframework.scheduling.support.CronExpression; import org.springframework.stereotype.Component; @@ -69,7 +73,19 @@ public class TimerDispatcher implements TimerDispatch, DisposableBean { */ private final AtomicBoolean started; + private final Supplier metricsTaskDispatchSupplier; + public TimerDispatcher() { + this(() -> timeout -> { + }); + } + + @Autowired + public TimerDispatcher(ObjectProvider metricsTaskDispatchProvider) { + this(resolveMetricsTaskDispatchSupplier(metricsTaskDispatchProvider)); + } + + private TimerDispatcher(Supplier metricsTaskDispatchSupplier) { this.wheelTimer = new HashedWheelTimer(r -> { Thread ret = new Thread(r, "wheelTimer"); ret.setDaemon(true); @@ -79,6 +95,16 @@ public TimerDispatcher() { this.currentTempTaskMap = new ConcurrentHashMap<>(8); this.eventListeners = new ConcurrentHashMap<>(8); this.started = new AtomicBoolean(true); + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; + } + + private static Supplier resolveMetricsTaskDispatchSupplier( + ObjectProvider metricsTaskDispatchProvider) { + if (metricsTaskDispatchProvider == null) { + return () -> timeout -> { + }; + } + return metricsTaskDispatchProvider::getObject; } @Override @@ -87,7 +113,8 @@ public void addJob(Job addJob, CollectResponseEventListener eventListener) { log.warn("Collector is offline, can not dispatch collect jobs."); return; } - WheelTimerTask timerJob = new WheelTimerTask(addJob); + // Delay dispatcher lookup to avoid a startup cycle with CommonDispatcher. + WheelTimerTask timerJob = new WheelTimerTask(addJob, metricsTaskDispatchSupplier); if (addJob.isCyclic()) { Long nextExecutionTime = getNextExecutionInterval(addJob); Timeout timeout = wheelTimer.newTimeout(timerJob, nextExecutionTime, TimeUnit.SECONDS); diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java index 5dfe1922a57..30d97ba2e11 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java @@ -27,7 +27,6 @@ import org.apache.hertzbeat.common.entity.job.Configmap; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.TimerTask; import org.apache.hertzbeat.common.util.AesUtil; @@ -35,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -44,11 +44,15 @@ public class WheelTimerTask implements TimerTask { private final Job job; - private final MetricsTaskDispatch metricsTaskDispatch; + private final Supplier metricsTaskDispatchSupplier; private static final Gson GSON = new Gson(); - public WheelTimerTask(Job job) { - this.metricsTaskDispatch = SpringContextHolder.getBean(MetricsTaskDispatch.class); + public WheelTimerTask(Job job, MetricsTaskDispatch metricsTaskDispatch) { + this(job, () -> metricsTaskDispatch); + } + + public WheelTimerTask(Job job, Supplier metricsTaskDispatchSupplier) { + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; this.job = job; // The initialization job will monitor the actual parameter value and replace the collection field initJobMetrics(job); @@ -93,7 +97,7 @@ private void initJobMetrics(Job job) { @Override public void run(Timeout timeout) throws Exception { job.setDispatchTime(System.currentTimeMillis()); - metricsTaskDispatch.dispatchMetricsTask(timeout); + metricsTaskDispatchSupplier.get().dispatchMetricsTask(timeout); } public Job getJob() { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java index d43de3e97d2..b544c8c39bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java @@ -20,19 +20,15 @@ import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import io.netty.channel.ChannelHandlerContext; -import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.collector.timer.TimerDispatcher; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.JsonUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.lang.reflect.Field; @@ -51,32 +47,27 @@ class GoOnlineProcessorTest { @Mock private ChannelHandlerContext channelHandlerContext; - private MockedStatic springContextHolderMockedStatic; - @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - goOnlineProcessor = new GoOnlineProcessor(); timerDispatcher = new TimerDispatcher(); - springContextHolderMockedStatic = Mockito.mockStatic(SpringContextHolder.class); - springContextHolderMockedStatic.when(() -> SpringContextHolder.getBean(TimerDispatch.class)).thenReturn(timerDispatcher); + goOnlineProcessor = new GoOnlineProcessor(timerDispatcher); } @AfterEach void tearDown() throws Exception { - springContextHolderMockedStatic.close(); timerDispatcher.destroy(); } @Test void verifyTaskMapPreservation() throws Exception { Job job = Job.builder() - .app("test") - .id(12345L) - .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) - .configmap(Lists.newArrayList()) - .isCyclic(true) - .build(); + .app("test") + .id(12345L) + .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) + .configmap(Lists.newArrayList()) + .isCyclic(true) + .build(); timerDispatcher.addJob(job, null); Field cyclicTaskMapField = TimerDispatcher.class.getDeclaredField("currentCyclicTaskMap"); @@ -85,20 +76,20 @@ void verifyTaskMapPreservation() throws Exception { assertEquals(1, currentCyclicTaskMap.size(), "Task map should have 1 job initially"); ClusterMsg.Message responseMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.RESPONSE) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.RESPONSE) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, responseMsg); assertEquals(1, currentCyclicTaskMap.size(), "Task map should still have 1 job after receiving RESPONSE"); ClusterMsg.Message requestMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.REQUEST) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.REQUEST) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, requestMsg); assertEquals(0, currentCyclicTaskMap.size(), "Task map should be empty after receiving REQUEST"); } diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java index fabdf973628..b4f91304f16 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java @@ -46,7 +46,6 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; -import org.springframework.util.Assert; import java.util.Collection; import java.util.Collections; @@ -223,14 +222,14 @@ private static boolean filterInternalTopics(String topic, Boolean monitorInterna @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null, "Metrics cannot be null"); + require(metrics != null, "Metrics cannot be null"); KafkaProtocol kafkaProtocol = metrics.getKclient(); // Ensure that metrics and kafkaProtocol are not null - Assert.isTrue(metrics != null && kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); + require(kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); // Ensure that host and port are not empty - Assert.hasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); - Assert.hasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); + requireHasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); + requireHasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); } @Override @@ -384,4 +383,14 @@ private long getLatestOffset(AdminClient adminClient, TopicPartition topicPartit public String supportProtocol() { return DispatchConstants.PROTOCOL_KAFKA; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java index b99304e60e4..0cf3107c580 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java @@ -43,7 +43,6 @@ import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.CommonUtil; import org.bson.Document; -import org.springframework.util.Assert; /** * Mongodb single collect @@ -87,11 +86,11 @@ public class MongodbSingleCollectImpl extends AbstractCollect { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException{ - Assert.isTrue(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); + require(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); MongodbProtocol mongodbProtocol = metrics.getMongodb(); - Assert.hasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); - Assert.hasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); - Assert.hasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); + requireHasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); + requireHasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); + requireHasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); } @Override @@ -226,4 +225,14 @@ private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) { connectionCommonCache.addCache(identifier, mongodbConnect, 3600 * 1000L); return mongoClient; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java index 33a361aa0bf..4034c1e15bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.collector.collect.AbstractCollect; @@ -33,8 +34,6 @@ import org.apache.hertzbeat.common.entity.job.protocol.NgqlProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.entity.message.CollectRep.MetricsData.Builder; -import org.springframework.util.Assert; -import org.springframework.util.StopWatch; /** * connect nebulaGraph and collect metrics use NGQL @@ -53,18 +52,17 @@ public class NgqlCollectImpl extends AbstractCollect { @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { NgqlProtocol ngql = metrics.getNgql(); - Assert.hasText(ngql.getHost(), "NGQL protocol host is required"); - Assert.hasText(ngql.getPort(), "Port protocol host is required"); - Assert.hasText(ngql.getParseType(), "NGQL protocol parseType is required"); - Assert.hasText(ngql.getUsername(), "NGQL protocol username is required"); - Assert.hasText(ngql.getPassword(), "NGQL protocol password is required"); + requireHasText(ngql.getHost(), "NGQL protocol host is required"); + requireHasText(ngql.getPort(), "Port protocol host is required"); + requireHasText(ngql.getParseType(), "NGQL protocol parseType is required"); + requireHasText(ngql.getUsername(), "NGQL protocol username is required"); + requireHasText(ngql.getPassword(), "NGQL protocol password is required"); } @Override public void collect(Builder builder, Metrics metrics) { NgqlProtocol ngql = metrics.getNgql(); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); + long startTimeNanos = System.nanoTime(); NebulaTemplate nebulaTemplate = new NebulaTemplate(); try { boolean initSuccess = nebulaTemplate.initSession(ngql); @@ -79,8 +77,7 @@ public void collect(Builder builder, Metrics metrics) { return; } - stopWatch.stop(); - long responseTime = stopWatch.getTotalTimeMillis(); + long responseTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); try { switch (ngql.getParseType()) { case PARSE_TYPE_FILTER_COUNT -> filterCount(nebulaTemplate, ngql, metrics.getAliasFields(), builder, responseTime); @@ -247,4 +244,10 @@ private Map showJobs(NebulaTemplate template, String protocolSpa result.put("running_jobs", String.valueOf(jobs.stream().filter(job -> Objects.equals(job.get("Status"), STATUS_RUNNING)).count())); return result; } + + private static void requireHasText(String value, String message) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException(message); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java index a8077ca6091..a98ff1ada44 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java @@ -54,14 +54,12 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.util.Assert; /** * rocketmq collect */ @Slf4j -public class RocketmqSingleCollectImpl extends AbstractCollect implements DisposableBean { +public class RocketmqSingleCollectImpl extends AbstractCollect { private static final int WAIT_TIMEOUT = 10; static final int QUEUE_CAPACITY = 5000; @@ -102,7 +100,6 @@ private static ManagedExecutor createExecutor() { corePoolSize, maximumPoolSize, QUEUE_CAPACITY, handler); } - @Override public void destroy() { this.executorService.close(); } @@ -113,10 +110,10 @@ public void destroy() { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); + require(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); RocketmqProtocol rocketmq = metrics.getRocketmq(); - Assert.hasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); - Assert.hasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); + requireHasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); + requireHasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); } @Override @@ -374,4 +371,14 @@ private void fillBuilder(RocketmqCollectData rocketmqCollectData, CollectRep.Met void executeConsumerTask(Runnable runnable) { executorService.execute(runnable); } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-common-core/pom.xml b/hertzbeat-common-core/pom.xml index 8cccffe102a..949cf19ed53 100644 --- a/hertzbeat-common-core/pom.xml +++ b/hertzbeat-common-core/pom.xml @@ -28,6 +28,7 @@ hertzbeat-common-core ${project.artifactId} + Framework-agnostic shared runtime models, utilities, and protocol support. diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java new file mode 100644 index 00000000000..030800faff5 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.RejectedExecutionException; + +/** + * Generic background task executor abstraction for runtime components. + */ +public interface BackgroundTaskExecutor { + + /** + * Execute a short-lived task. + * + * @param runnable task + * @throws RejectedExecutionException when execution is rejected + */ + void execute(Runnable runnable) throws RejectedExecutionException; + + /** + * Execute a long-lived background task. + * + * @param runnable task + */ + void executeLongRunning(Runnable runnable); + + /** + * Release executor resources. + * + * @throws Exception close exception + */ + void destroy() throws Exception; +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java similarity index 84% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index 0962e608ad3..956b8a54140 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -18,17 +18,12 @@ package org.apache.hertzbeat.common.config; import org.apache.hertzbeat.common.concurrent.AdmissionMode; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.bind.ConstructorBinding; -import org.springframework.boot.context.properties.bind.DefaultValue; -import org.springframework.boot.context.properties.bind.Name; /** - * Virtual-thread related configuration. + * Framework-agnostic virtual-thread runtime configuration. */ -@ConfigurationProperties(prefix = "hertzbeat.vthreads") public record VirtualThreadProperties( - @DefaultValue("true") boolean enabled, + boolean enabled, PoolProperties collector, PoolProperties common, PoolProperties manager, @@ -42,7 +37,6 @@ public record VirtualThreadProperties( private static final int DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS = 10; private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL = 4; - @ConstructorBinding public VirtualThreadProperties { collector = normalizePool(collector, PoolProperties.collectorDefaults()); common = common == null ? PoolProperties.commonDefaults() : common; @@ -71,10 +65,9 @@ public static VirtualThreadProperties defaults() { * Pool-level configuration. */ public record PoolProperties( - @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, - @DefaultValue("0") int maxConcurrentJobs) { + AdmissionMode mode, + int maxConcurrentJobs) { - @ConstructorBinding public PoolProperties { mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; } @@ -112,14 +105,13 @@ private static int defaultCollectorConcurrency() { * Alerter-specific executor configuration. */ public record AlerterProperties( - @Name("notify") PoolProperties notifyPool, - @DefaultValue("10") int periodicMaxConcurrentJobs, + PoolProperties notifyPool, + int periodicMaxConcurrentJobs, QueueProperties logWorker, QueueProperties reduce, QueueProperties windowEvaluator, - @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + int notifyMaxConcurrentPerChannel) { - @ConstructorBinding public AlerterProperties { notifyPool = normalizePool(notifyPool, PoolProperties.alerterNotifyDefaults()); periodicMaxConcurrentJobs = periodicMaxConcurrentJobs <= 0 @@ -146,12 +138,8 @@ public static AlerterProperties defaults() { * Queue-preserving executor configuration. */ public record QueueProperties( - @DefaultValue("0") int maxConcurrentJobs, - @DefaultValue("0") int queueCapacity) { - - @ConstructorBinding - public QueueProperties { - } + int maxConcurrentJobs, + int queueCapacity) { public QueueProperties() { this(0, 0); @@ -174,14 +162,10 @@ public static QueueProperties windowEvaluatorDefaults() { * Async executor configuration. */ public record AsyncProperties( - @DefaultValue("true") boolean enabled, - @DefaultValue("256") int concurrencyLimit, - @DefaultValue("true") boolean rejectWhenLimitReached, - @DefaultValue("5000") long taskTerminationTimeout) { - - @ConstructorBinding - public AsyncProperties { - } + boolean enabled, + int concurrencyLimit, + boolean rejectWhenLimitReached, + long taskTerminationTimeout) { public AsyncProperties() { this(true, 256, true, 5000L); diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java similarity index 89% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java index 2b2c3a5f6ae..c689b913949 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java @@ -21,17 +21,13 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; /** - * SMS configuration + * Framework-agnostic SMS runtime configuration. */ @Data @AllArgsConstructor @NoArgsConstructor -@Component -@ConfigurationProperties(prefix = "alerter.sms") public class SmsConfig { /** diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java similarity index 93% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java index 4b4e1d53265..e7fb6e52c1f 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.common.entity.job; - import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Collections; import java.util.Comparator; @@ -35,13 +34,11 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; -import org.springframework.util.CollectionUtils; /** - * Collect task details + * Collect task details. */ @Data @AllArgsConstructor @@ -51,15 +48,15 @@ public class Job { /** - * Task Job id + * Task Job id. */ private long id; /** - * Tenant id + * Tenant id. */ private long tenantId = 0; /** - * Monitoring Task ID + * Monitoring Task ID. */ private long monitorId; /** @@ -68,11 +65,11 @@ public class Job { */ private Map metadata; /** - * bind labels + * bind labels. */ private Map labels; /** - * bind annotations + * bind annotations. */ private Map annotations; /** @@ -86,12 +83,12 @@ public class Job { */ private String category; /** - * Type of monitoring eg: linux | mysql | jvm + * Type of monitoring eg: linux | mysql | jvm. */ private String app; /** * The internationalized name of the monitoring type - * PING CONNECT + * PING CONNECT. */ private Map name; /** @@ -101,55 +98,54 @@ public class Job { */ private Map help; /** - * The monitor help link + * The monitor help link. */ private Map helpLink; /** - * Task dispatch start timestamp + * Task dispatch start timestamp. */ private long timestamp; /** - * Default task collection time interval (unit: second) eg: 30,60,600 + * Default task collection time interval (unit: second) eg: 30,60,600. */ private long defaultInterval = 600L; /** - * Refresh time list for one cycle of the job + * Refresh time list for one cycle of the job. */ private ConcurrentLinkedDeque intervals; /** - * Whether it is a recurring periodic task true is yes, false is no + * Whether it is a recurring periodic task true is yes, false is no. */ private boolean isCyclic = false; /** - * monitor input need params + * monitor input need params. */ - private List params; + private List params; /** - * Metrics configuration eg: cpu memory - * eg: cpu memory + * Metrics configuration eg: cpu memory. */ private List metrics; /** - * Monitoring configuration parameter properties and values eg: username password timeout host + * Monitoring configuration parameter properties and values eg: username password timeout host. */ private List configmap; /** - * Whether it is a service discovery job, true is yes, false is no + * Whether it is a service discovery job, true is yes, false is no. */ private boolean isSd = false; /** - * Whether to use the Prometheus proxy + * Whether to use the Prometheus proxy. */ private boolean prometheusProxyMode = false; /** - * Scheduling type: interval or cron + * Scheduling type: interval or cron. */ private String scheduleType = "interval"; /** - * Cron expression for scheduling, used when scheduleType is "cron" + * Cron expression for scheduling, used when scheduleType is "cron". */ private String cronExpression = null; @@ -160,7 +156,7 @@ public class Job { private Map envConfigmaps; /** - * collector use - timestamp when the task was scheduled by the time wheel + * collector use - timestamp when the task was scheduled by the time wheel. */ @JsonIgnore private transient long dispatchTime; @@ -179,13 +175,13 @@ public class Job { private transient LinkedList> priorMetrics; /** - * collector use - Temporarily store one-time task metrics response data + * collector use - Temporarily store one-time task metrics response data. */ @JsonIgnore private transient List responseDataTemp; /** - * collector use - construct to initialize metrics execution view + * collector use - construct to initialize metrics execution view. */ public synchronized void constructPriorMetrics() { long now = System.currentTimeMillis(); @@ -232,7 +228,7 @@ public synchronized void constructPriorMetrics() { } /** - * collector use - to get the next set of priority metric group tasks + * collector use - to get the next set of priority metric group tasks. * * @param metrics Current Metrics * @param first Is it the first time to get @@ -314,7 +310,7 @@ public void initIntervals() { } /** - * The greatest common divisor + * The greatest common divisor. */ public static long gcd(long a, long b) { while (b != 0) { @@ -326,7 +322,7 @@ public static long gcd(long a, long b) { } /** - * The least common multiple + * The least common multiple. */ public static long lcm(List array) { if (array != null && !array.isEmpty()) { @@ -340,9 +336,8 @@ public static long lcm(List array) { } /** - * * @param metricsIntervals A unique list composed of intervals for all metrics - * Generate a list of refresh intervals for metric collection + * Generate a list of refresh intervals for metric collection. */ public synchronized void generateMetricsIntervals(List metricsIntervals) { // 1. To find the least common multiple (LCM) of all metric refresh intervals @@ -368,7 +363,7 @@ public synchronized void generateMetricsIntervals(List metricsIntervals) { } public synchronized long getInterval() { - if (!CollectionUtils.isEmpty(this.intervals)) { + if (this.intervals != null && !this.intervals.isEmpty()) { Long interval = this.intervals.removeFirst(); if (interval != null) { this.intervals.addLast(interval); diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java similarity index 56% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java index f93093157d2..eae3e24e286 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java @@ -17,38 +17,60 @@ package org.apache.hertzbeat.common.entity.job; -import java.io.Serializable; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; /** - * Monitoring configuration parameter properties and values - * During the process, you need to replace the content with the identifier ^_^key^_^ - * in the protocol configuration parameter with the real value in the configuration parameter + * Framework-agnostic parameter definition used by runtime templates and jobs. */ @Data +@Builder @AllArgsConstructor @NoArgsConstructor -@Builder -public class Configmap implements Serializable { +public class RuntimeParamDefine { - /** - * Parameter key, replace the content with the identifier ^^_key_^^ in the protocol - * configuration parameter with the real value in the configuration parameter - */ - private String key; + private String app; - /** - * parameter value - */ - private Object value; + private Map name; + + private String field; + + private String type; + + private boolean required = false; + + private String defaultValue; + + private String placeholder; + + private String range; + + private Short limit; + + private List + + + org.apache.hertzbeat + hertzbeat-common-spring + org.springframework.boot diff --git a/hertzbeat-remoting/pom.xml b/hertzbeat-remoting/pom.xml index 1fddaf8853d..c9d4f53d37e 100644 --- a/hertzbeat-remoting/pom.xml +++ b/hertzbeat-remoting/pom.xml @@ -35,7 +35,7 @@ org.apache.hertzbeat - hertzbeat-common-spring + hertzbeat-common-core diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java index 7863e830e31..2001fd912fd 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java @@ -36,9 +36,9 @@ import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; import java.util.concurrent.ThreadFactory; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; import org.apache.hertzbeat.remoting.event.NettyEventListener; @@ -51,10 +51,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { private static final int DEFAULT_WORKER_THREAD_NUM = Math.min(4, Runtime.getRuntime().availableProcessors()); - + private final NettyClientConfig nettyClientConfig; - private final CommonThreadPool threadPool; + private final BackgroundTaskExecutor threadPool; private final Bootstrap bootstrap = new Bootstrap(); @@ -64,7 +64,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti public NettyRemotingClient(final NettyClientConfig nettyClientConfig, final NettyEventListener nettyEventListener, - final CommonThreadPool threadPool) { + final BackgroundTaskExecutor threadPool) { super(nettyEventListener); this.nettyClientConfig = nettyClientConfig; this.threadPool = threadPool; diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java index 169f38220f3..f6e743bda5e 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java @@ -43,9 +43,9 @@ import io.netty.handler.timeout.IdleStateHandler; import java.util.List; import java.util.concurrent.ThreadFactory; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingServer; import org.apache.hertzbeat.remoting.event.NettyEventListener; @@ -59,7 +59,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti private final NettyServerConfig nettyServerConfig; - private final CommonThreadPool threadPool; + private final BackgroundTaskExecutor threadPool; private EventLoopGroup bossGroup; @@ -69,7 +69,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti public NettyRemotingServer(final NettyServerConfig nettyServerConfig, final NettyEventListener nettyEventListener, - final CommonThreadPool threadPool) { + final BackgroundTaskExecutor threadPool) { super(nettyEventListener); this.nettyServerConfig = nettyServerConfig; this.threadPool = threadPool; diff --git a/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java b/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java index 4a852d99d8f..5d98432572e 100644 --- a/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java +++ b/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java @@ -18,8 +18,10 @@ package org.apache.hertzbeat.remoting; import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; @@ -35,7 +37,24 @@ */ public class RemotingServiceTest { - private final CommonThreadPool threadPool = new CommonThreadPool(); + private final BackgroundTaskExecutor threadPool = new BackgroundTaskExecutor() { + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Override + public void execute(Runnable runnable) { + executor.execute(runnable); + } + + @Override + public void executeLongRunning(Runnable runnable) { + executor.execute(runnable); + } + + @Override + public void destroy() { + executor.shutdownNow(); + } + }; private RemotingServer remotingServer; From 494094886c7f7624b156359ec62a57d64e6cc310 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 22:13:26 +0800 Subject: [PATCH 09/14] refactor: replace MappingJackson2HttpMessageConverter with JacksonJsonHttpMessageConverter in test classes --- .../controller/MetricsFavoriteControllerTest.java | 4 ++-- .../manager/controller/MonitorsControllerTest.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java index 3be38dcb15e..f926a477d8a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java @@ -29,7 +29,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.http.MediaType; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.http.converter.json.JacksonJsonHttpMessageConverter; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -73,7 +73,7 @@ class MetricsFavoriteControllerTest { void setUp() { this.mockMvc = MockMvcBuilders.standaloneSetup(metricsFavoriteController) .setControllerAdvice(new GlobalExceptionHandler()) - .setMessageConverters(new MappingJackson2HttpMessageConverter()) + .setMessageConverters(new JacksonJsonHttpMessageConverter()) .build(); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java index 7cd4b68040f..f5026a2d928 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java @@ -39,13 +39,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.data.web.config.EnableSpringDataWebSupport; import org.springframework.http.MediaType; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.http.converter.json.JacksonJsonHttpMessageConverter; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; -import org.springframework.data.web.config.SpringDataJacksonConfiguration; +import org.springframework.data.web.config.SpringDataJackson3Configuration; import org.springframework.data.web.config.SpringDataWebSettings; +import tools.jackson.databind.json.JsonMapper; /** * Test case for {@link MonitorsController} @@ -63,9 +63,9 @@ class MonitorsControllerTest { @BeforeEach void setUp() { - MappingJackson2HttpMessageConverter messageConverter = new MappingJackson2HttpMessageConverter( - Jackson2ObjectMapperBuilder.json() - .modules(new SpringDataJacksonConfiguration.PageModule( + JacksonJsonHttpMessageConverter messageConverter = new JacksonJsonHttpMessageConverter( + JsonMapper.builder() + .addModule(new SpringDataJackson3Configuration.PageModule( new SpringDataWebSettings(EnableSpringDataWebSupport.PageSerializationMode.DIRECT))) .build()); this.mockMvc = MockMvcBuilders.standaloneSetup(monitorsController) From 7a79d18356852b2d923b7d970a8e95b64e9343bf Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 22:22:31 +0800 Subject: [PATCH 10/14] Add native docs and config notes --- .github/workflows/collector-native-build.yml | 99 +++++++++ CONTRIBUTING.md | 4 +- README.md | 9 +- README_CN.md | 9 +- README_JP.md | 21 +- hertzbeat-ai/pom.xml | 2 +- .../hertzbeat-collector-basic/pom.xml | 6 +- .../hertzbeat-collector-collector/pom.xml | 205 +++++++++++++++++- .../apache/hertzbeat/collector/Collector.java | 12 +- .../CollectorRuntimeHintsRegistrar.java | 111 ++++++++++ .../nativex/NativeCollectorDefaults.java | 58 +++++ .../main/resources/META-INF/spring.factories | 2 + .../reflect-config.json | 54 +++++ ...ertzbeat.collector.collect.AbstractCollect | 37 ++++ .../nativex/NativeCollectorDefaultsTest.java | 39 ++++ .../hertzbeat-collector-common/pom.xml | 4 +- .../collect/common/ssh/SshTunnelHelper.java | 63 +++--- .../hertzbeat-collector-kafka/pom.xml | 6 +- .../hertzbeat-collector-mongodb/pom.xml | 6 +- .../hertzbeat-collector-nebulagraph/pom.xml | 6 +- hertzbeat-collector/pom.xml | 2 +- .../hertzbeat-collector-basic-e2e/pom.xml | 4 +- .../hertzbeat-collector-common-e2e/pom.xml | 6 +- .../hertzbeat-collector-kafka-e2e/pom.xml | 4 +- hertzbeat-e2e/hertzbeat-log-e2e/pom.xml | 4 +- hertzbeat-e2e/pom.xml | 4 +- home/docs/community/contribution.md | 2 +- home/docs/community/development.md | 17 +- home/docs/community/how-to-release.md | 15 +- home/docs/download.md | 17 +- home/docs/help/db2.md | 7 + home/docs/help/mysql.md | 7 + home/docs/help/oceanbase.md | 7 + home/docs/help/oracle.md | 7 + home/docs/help/risc-v.md | 8 +- home/docs/start/native-collector.md | 75 +++++++ home/docs/start/package-deploy.md | 55 ++++- home/docs/start/quickstart.md | 17 +- home/docs/start/virtual-thread.md | 2 +- .../current/community/contribution.md | 2 +- .../current/community/development.md | 17 +- .../current/community/how-to-release.md | 15 +- .../current/download.md | 17 +- .../current/help/db2.md | 7 + .../current/help/mysql.md | 7 + .../current/help/oceanbase.md | 7 + .../current/help/oracle.md | 7 + .../current/help/risc-v.md | 8 +- .../current/start/native-collector.md | 75 +++++++ .../current/start/package-deploy.md | 60 +++-- .../current/start/quickstart.md | 17 +- .../current/start/virtual-thread.md | 2 +- home/sidebars.json | 1 + home/src/components/StructuredData.js | 2 +- home/src/pages/faq.js | 2 +- home/src/pages/zh-cn/faq.js | 2 +- home/static/llms-zh.txt | 2 +- home/static/llms.txt | 2 +- pom.xml | 6 +- script/assembly/collector/assembly-native.xml | 82 +++++++ script/assembly/collector/assembly.xml | 5 +- .../collector/bin-native-win/restart.bat | 25 +++ .../collector/bin-native-win/shutdown.bat | 49 +++++ .../collector/bin-native-win/startup.bat | 102 +++++++++ .../assembly/collector/bin-native/restart.sh | 27 +++ .../assembly/collector/bin-native/shutdown.sh | 62 ++++++ .../assembly/collector/bin-native/startup.sh | 98 +++++++++ script/assembly/collector/bin/startup.sh | 2 +- script/assembly/server/bin/startup.sh | 2 +- .../ci/github-actions/setup-deps/action.yml | 6 +- script/docker/collector/Dockerfile | 4 +- script/docker/collector/build.sh | 11 +- script/docker/server/Dockerfile | 2 +- script/ext-lib/README | 8 +- 74 files changed, 1573 insertions(+), 183 deletions(-) create mode 100644 .github/workflows/collector-native-build.yml create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java create mode 100644 home/docs/start/native-collector.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md create mode 100644 script/assembly/collector/assembly-native.xml create mode 100644 script/assembly/collector/bin-native-win/restart.bat create mode 100644 script/assembly/collector/bin-native-win/shutdown.bat create mode 100644 script/assembly/collector/bin-native-win/startup.bat create mode 100644 script/assembly/collector/bin-native/restart.sh create mode 100644 script/assembly/collector/bin-native/shutdown.sh create mode 100644 script/assembly/collector/bin-native/startup.sh diff --git a/.github/workflows/collector-native-build.yml b/.github/workflows/collector-native-build.yml new file mode 100644 index 00000000000..97a14b3e8fd --- /dev/null +++ b/.github/workflows/collector-native-build.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Collector Native CI + +on: + workflow_dispatch: + push: + branches: [ action* ] + paths: + - '.github/workflows/collector-native-build.yml' + - 'pom.xml' + - 'hertzbeat-collector/**' + - 'script/assembly/collector/**' + pull_request: + branches: [ master, dev ] + paths: + - '.github/workflows/collector-native-build.yml' + - 'pom.xml' + - 'hertzbeat-collector/**' + - 'script/assembly/collector/**' + +jobs: + build-native-collector: + name: Native collector (${{ matrix.platform }}) + permissions: + contents: read + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: ubuntu-latest + archive_ext: tar.gz + - platform: linux-arm64 + runner: ubuntu-24.04-arm + archive_ext: tar.gz + - platform: macos-amd64 + runner: macos-13 + archive_ext: tar.gz + - platform: macos-arm64 + runner: macos-14 + archive_ext: tar.gz + - platform: windows-amd64 + runner: windows-latest + archive_ext: zip + + steps: + - uses: actions/checkout@v4 + + - name: Set up GraalVM JDK 25 + uses: graalvm/setup-graalvm@v1 + with: + distribution: graalvm-community + java-version: '25' + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: maven + native-image-job-reports: 'true' + + - name: Verify toolchain + shell: pwsh + run: | + java -version + native-image --version + mvn -version + + - name: Build native collector package + run: mvn -B -pl hertzbeat-collector/hertzbeat-collector-collector -am -Pnative -DskipTests package + + - name: Locate native collector package + id: package + shell: pwsh + run: | + $package = Get-ChildItem -Path "dist/apache-hertzbeat-collector-native-*-${{ matrix.platform }}-bin.${{ matrix.archive_ext }}" | Select-Object -First 1 + if (-not $package) { + throw "Native collector package not found for ${{ matrix.platform }}" + } + "archive=$($package.FullName)" >> $env:GITHUB_OUTPUT + + - name: Upload native collector package + uses: actions/upload-artifact@v4 + with: + name: apache-hertzbeat-collector-native-${{ matrix.platform }} + path: ${{ steps.package.outputs.archive }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a1738267c9..9f329883578 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` @@ -172,7 +172,7 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/README.md b/README.md index 0d8cff96a67..e6b5b7dbf7a 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,12 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh ` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters (Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download](https://hertzbeat.apache.org/docs/download) + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml collector: @@ -148,7 +148,8 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh ` or `bin/startup.bat` + - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access `http://localhost:1157` and you will see the registered new collector in dashboard Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -156,7 +157,7 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` diff --git a/README_CN.md b/README_CN.md index 7393ace7010..fda8d7ca640 100644 --- a/README_CN.md +++ b/README_CN.md @@ -127,12 +127,12 @@ ##### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包 `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml` (可选) 3. 部署启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群（可选） - - 下载采集器安装包 `hertzbeat-collector-xx.tar.gz` 到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) + - 下载采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或与你目标平台匹配的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主 HertzBeat 服务的对外 IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml collector: @@ -145,7 +145,8 @@ manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` + - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包。 + - JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -153,7 +154,7 @@ ##### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程 `hertzbeat-startup` 和前端工程 `web-app` -2. 后端：需要 `maven3+`, `java21` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 +2. 后端：需要 `maven3+`, `java25` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 3. 前端：需要 `nodejs npm angular-cli`环境，待本地后端启动后，在 `web-app` 目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` diff --git a/README_JP.md b/README_JP.md index 13c49de6a4f..f8993253319 100644 --- a/README_JP.md +++ b/README_JP.md @@ -108,7 +108,7 @@ docker run -d -p 1157:1157 -p 1158:1158 --name hertzbeat apache/hertzbeat ``` -2. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 +2. スタート：`http://localhost:1157`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 3. コレクタークラスタのデプロイメント（オプション） @@ -126,13 +126,13 @@ ##### 方式２：インストールパッケージ -1. リリースパッケージをダウンロード `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) -2. HertzBeatのymlファイルを設定 `hertzbeat/config/application.yml` （オプション） -3. コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行 -4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat` +1. リリースパッケージ `apache-hertzbeat-xx-bin.tar.gz` をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) +2. HertzBeat の設定ファイル `hertzbeat/config/application.yml` を編集します（任意） +3. コマンド `$ ./bin/startup.sh` または `bin/startup.bat` を実行します +4. ブラウザで `http://localhost:1157` にアクセスします。デフォルトのアカウントとパスワードは `admin/hertzbeat` です 5. コレクタークラスタのデプロイメント（オプション） - - コレクターパッケージを別のホストにダウンロード `hertzbeat-collector-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) - - コレクターのymlファイルを設定 `hertzbeat-collector/config/application.yml` + - 別ホストにコレクターのインストールパッケージ `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM コレクター）または対象プラットフォーム向けの Native コレクターパッケージ（例: `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`）をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) + - コレクターの設定ファイル `hertzbeat-collector/config/application.yml` を編集します ```yaml collector: dispatch: @@ -148,15 +148,16 @@ - `mode: ${MODE:public}`：実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。 - `manager-host: ${MANAGER_HOST:127.0.0.1}`：メインhertzbeatサーバーのIP。 - `manager-port: ${MANAGER_PORT:1158}`：メインhertzbeatサーバポート。 - - コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行。 - - `http://localhost:1157`にアクセスし、登録された新しいコレクターを見ることがでます。 + - `ext-lib` で MySQL、OceanBase、Oracle、DB2 などの外部 JDBC ドライバーを読み込む必要がある場合は、JVM コレクターのインストールパッケージを使用してください。 + - JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。 + - メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。 詳細ステップ [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) ##### 方式３：ローカルの実行 1. ローカルの実行には、バックエンドのプロジェクト`hertzbeat-startup`とフロントエンドのプロジェクト`web-app`を起動する必要があります。 -2. バックエンド：`maven3+`、`Java21`と`lombok`の環境は必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し、`hertzbeat-startup` を起動します。 +2. バックエンド：`maven3+`、`Java25`、`lombok` の環境が必要です。`YML` 設定を修正し、Java 仮想マシンパラメータに `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加して `hertzbeat-startup` を起動します。 3. フロントエンド：`nodejs npm angular-cli`の環境は必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `ng serve --open` というコマンドを実行します。 4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 diff --git a/hertzbeat-ai/pom.xml b/hertzbeat-ai/pom.xml index 2b9298b6b5e..235b2481dce 100644 --- a/hertzbeat-ai/pom.xml +++ b/hertzbeat-ai/pom.xml @@ -27,7 +27,7 @@ ${hertzbeat.version} 1.1.1 - 21 + 25 diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index c43972cda0a..de107950e41 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -30,8 +30,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 1.2.5 @@ -189,4 +189,4 @@ ${zookeeper.version} - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index 8a108c99de0..d40bdf60abb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -29,12 +29,18 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 + + org.apache.hertzbeat + hertzbeat-common-spring + ${hertzbeat.version} + + org.apache.hertzbeat @@ -142,8 +148,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -216,8 +222,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -344,5 +350,192 @@ + + native + + hertzbeat-collector-collector + + ../../script/assembly/collector/bin-native + tar.gz + unsupported + ${native.target.platform}-bin + apache-hertzbeat-collector-native-${hzb.version}-${native.target.platform}-bin + target/${native.image.name}${native.binary.extension} + ${project.build.finalName}${native.binary.extension} + + + apache-hertzbeat-collector-native-${hzb.version} + + + src/main/resources + + META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + full + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-supported-native-platform + validate + + enforce + + + + + native.target.platform + ^(linux-amd64|linux-arm64|macos-amd64|macos-arm64|windows-amd64)$ + Unsupported native collector platform. Build the native collector on a supported GitHub runner or matching local host: linux-amd64, linux-arm64, macos-amd64, macos-arm64, windows-amd64. + + + + + + + + org.graalvm.buildtools + native-maven-plugin + 0.11.4 + + ${native.image.name} + + -H:+UnlockExperimentalVMOptions + -H:-AddAllFileSystemProviders + -H:ServiceLoaderFeatureExcludeServiceProviders=org.apache.sshd.common.file.root.RootedFileSystemProvider,org.apache.sshd.sftp.client.fs.SftpFileSystemProvider + --initialize-at-build-time=net.i2p.crypto.eddsa.EdDSASecurityProvider,org.apache.arrow.memory.util.MemoryUtil + -J--add-opens=java.base/java.nio=ALL-UNNAMED + -J-Dorg.apache.sshd.security.registrars=org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + native-bin + package + + single + + + + ../../script/assembly/collector/assembly-native.xml + + ../../dist + + + + + + + + + native-platform-linux-amd64 + + + Linux + amd64 + + + + linux-amd64 + + + + native-platform-linux-x86_64 + + + Linux + x86_64 + + + + linux-amd64 + + + + native-platform-linux-arm64 + + + Linux + aarch64 + + + + linux-arm64 + + + + native-platform-macos-amd64 + + + mac + x86_64 + + + + macos-amd64 + + + + native-platform-macos-arm64 + + + mac + aarch64 + + + + macos-arm64 + + + + native-platform-windows-amd64 + + + windows + amd64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + + + native-platform-windows-x86_64 + + + windows + x86_64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java index 3e41f228010..0eca3ce9e35 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java @@ -18,10 +18,13 @@ package org.apache.hertzbeat.collector; import jakarta.annotation.PostConstruct; +import org.apache.hertzbeat.collector.nativex.CollectorRuntimeHintsRegistrar; +import org.apache.hertzbeat.collector.nativex.NativeCollectorDefaults; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ImportRuntimeHints; /** * collector startup @@ -29,13 +32,20 @@ @ComponentScan(basePackages = {"org.apache.hertzbeat"}) @ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"}) @SpringBootApplication +@ImportRuntimeHints(CollectorRuntimeHintsRegistrar.class) public class Collector { public static void main(String[] args) { - SpringApplication.run(Collector.class, args); + SpringApplication application = new SpringApplication(Collector.class); + NativeCollectorDefaults.applyTo(application); + application.run(args); } @PostConstruct public void init() { System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory"); + if (System.getProperty("arrow.allocation.manager.type") == null + && System.getenv("ARROW_ALLOCATION_MANAGER_TYPE") == null) { + System.setProperty("arrow.allocation.manager.type", "Netty"); + } } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java new file mode 100644 index 00000000000..3464caa1b1b --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.LinkedHashSet; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.arrow.memory.netty.NettyAllocationManager; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.IntervalUnit; +import org.apache.arrow.vector.types.MetadataVersion; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.UnionMode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.hertzbeat.common.entity.dto.ServerInfo; +import org.springframework.aot.hint.BindingReflectionHintsRegistrar; +import org.springframework.aot.hint.MemberCategory; +import org.springframework.aot.hint.RuntimeHints; +import org.springframework.aot.hint.RuntimeHintsRegistrar; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.lang.NonNull; +import org.springframework.util.ClassUtils; + +/** + * Registers native binding hints for collector-side message payloads. + */ +@Slf4j +public class CollectorRuntimeHintsRegistrar implements RuntimeHintsRegistrar { + + private static final String JOB_PACKAGE = "org.apache.hertzbeat.common.entity.job"; + private static final String JOB_PROTOCOL_PACKAGE = "org.apache.hertzbeat.common.entity.job.protocol"; + + @Override + public void registerHints(@NonNull RuntimeHints hints, ClassLoader classLoader) { + BindingReflectionHintsRegistrar bindingRegistrar = new BindingReflectionHintsRegistrar(); + registerType(bindingRegistrar, hints, ServerInfo.class); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PACKAGE); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PROTOCOL_PACKAGE); + hints.reflection().registerType(NettyAllocationManager.class, MemberCategory.DECLARED_FIELDS); + registerType(bindingRegistrar, hints, Schema.class); + registerType(bindingRegistrar, hints, Field.class); + registerType(bindingRegistrar, hints, FieldType.class); + registerType(bindingRegistrar, hints, DictionaryEncoding.class); + registerType(bindingRegistrar, hints, DateUnit.class); + registerType(bindingRegistrar, hints, FloatingPointPrecision.class); + registerType(bindingRegistrar, hints, IntervalUnit.class); + registerType(bindingRegistrar, hints, MetadataVersion.class); + registerType(bindingRegistrar, hints, TimeUnit.class); + registerType(bindingRegistrar, hints, UnionMode.class); + for (Class nestedClass : ArrowType.class.getDeclaredClasses()) { + if (!nestedClass.isAnnotation() && !nestedClass.isInterface()) { + registerType(bindingRegistrar, hints, nestedClass); + } + } + } + + private void scanBindingPackage(ClassLoader classLoader, BindingReflectionHintsRegistrar bindingRegistrar, + RuntimeHints hints, String basePackage) { + for (Class clazz : findBindingTypes(basePackage, classLoader)) { + registerType(bindingRegistrar, hints, clazz); + } + } + + private void registerType(BindingReflectionHintsRegistrar bindingRegistrar, RuntimeHints hints, Class clazz) { + bindingRegistrar.registerReflectionHints(hints.reflection(), clazz); + } + + private Set> findBindingTypes(String basePackage, ClassLoader classLoader) { + Set> bindingTypes = new LinkedHashSet<>(); + ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false); + TypeFilter includeAll = (metadataReader, metadataReaderFactory) -> true; + scanner.addIncludeFilter(includeAll); + for (BeanDefinition candidate : scanner.findCandidateComponents(basePackage)) { + String className = candidate.getBeanClassName(); + if (className == null) { + continue; + } + try { + Class clazz = ClassUtils.forName(className, classLoader); + if (!clazz.isAnnotation() && !clazz.isInterface()) { + bindingTypes.add(clazz); + } + } catch (Throwable ex) { + log.debug("Skip native binding hint registration for {}", className, ex); + } + } + return bindingTypes; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java new file mode 100644 index 00000000000..5f97c36ecbf --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.Map; +import org.springframework.boot.SpringApplication; +import org.springframework.core.NativeDetector; + +/** + * Applies the native-only collector defaults without forking {@code application.yml}. + */ +public final class NativeCollectorDefaults { + + static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude"; + static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + "org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration", + "org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration", + "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration", + "org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.health.DataSourceHealthContributorAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.metrics.DataSourcePoolMetricsAutoConfiguration", + "org.springframework.boot.tomcat.autoconfigure.metrics.TomcatMetricsAutoConfiguration"); + + private NativeCollectorDefaults() { + } + + public static void applyTo(SpringApplication application) { + Map defaultProperties = defaultProperties(NativeDetector.inNativeImage()); + if (!defaultProperties.isEmpty()) { + application.setDefaultProperties(defaultProperties); + } + } + + static Map defaultProperties(boolean nativeImage) { + if (!nativeImage) { + return Map.of(); + } + return Map.of(AUTOCONFIGURE_EXCLUDE_PROPERTY, NATIVE_AUTOCONFIGURE_EXCLUDES); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..16c40f8c6f3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +# Intentionally left blank. +# Native collector defaults are applied from Collector.main. diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json new file mode 100644 index 00000000000..227a978fe57 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json @@ -0,0 +1,54 @@ +[ + { + "name": "org.apache.hertzbeat.collector.Collector__ApplicationContextInitializer", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.collector.Collector__BeanFactoryRegistrations", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.common.entity.dto.ServerInfo", + "allDeclaredConstructors": true, + "allDeclaredFields": true, + "allDeclaredMethods": true + }, + { + "name": "io.netty.channel.kqueue.KQueueDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.nio.NioEventLoopGroup", + "allPublicConstructors": true + } +] diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect new file mode 100644 index 00000000000..81ed2990010 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect @@ -0,0 +1,37 @@ +org.apache.hertzbeat.collector.collect.http.HttpCollectImpl +org.apache.hertzbeat.collector.collect.http.SslCertificateCollectImpl +org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect +org.apache.hertzbeat.collector.collect.icmp.IcmpCollectImpl +org.apache.hertzbeat.collector.collect.jmx.JmxCollectImpl +org.apache.hertzbeat.collector.collect.redis.RedisCommonCollectImpl +org.apache.hertzbeat.collector.collect.mongodb.MongodbSingleCollectImpl +org.apache.hertzbeat.collector.collect.rocketmq.RocketmqSingleCollectImpl +org.apache.hertzbeat.collector.collect.snmp.SnmpCollectImpl +org.apache.hertzbeat.collector.collect.ssh.SshCollectImpl +org.apache.hertzbeat.collector.collect.telnet.TelnetCollectImpl +org.apache.hertzbeat.collector.collect.smtp.SmtpCollectImpl +org.apache.hertzbeat.collector.collect.ntp.NtpCollectImpl +org.apache.hertzbeat.collector.collect.websocket.WebsocketCollectImpl +org.apache.hertzbeat.collector.collect.ftp.FtpCollectImpl +org.apache.hertzbeat.collector.collect.udp.UdpCollectImpl +org.apache.hertzbeat.collector.collect.push.PushCollectImpl +org.apache.hertzbeat.collector.collect.dns.DnsCollectImpl +org.apache.hertzbeat.collector.collect.nginx.NginxCollectImpl +org.apache.hertzbeat.collector.collect.memcached.MemcachedCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl +org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl +org.apache.hertzbeat.collector.collect.registry.RegistryImpl +org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl +org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl +org.apache.hertzbeat.collector.collect.script.ScriptCollectImpl +org.apache.hertzbeat.collector.collect.mqtt.MqttCollectImpl +org.apache.hertzbeat.collector.collect.ipmi2.IpmiCollectImpl +org.apache.hertzbeat.collector.collect.kafka.KafkaCollectImpl +org.apache.hertzbeat.collector.collect.sd.HttpSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.NacosSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.DnsSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.EurekaSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.ConsulSdCollectImpl +org.apache.hertzbeat.collector.collect.modbus.ModbusCollectImpl +org.apache.hertzbeat.collector.collect.s7.S7CollectImpl diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java new file mode 100644 index 00000000000..2ffa4beeebc --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +class NativeCollectorDefaultsTest { + + @Test + void shouldProvideNativeAutoconfigureExcludesWhenNativeImage() { + Map properties = NativeCollectorDefaults.defaultProperties(true); + assertEquals(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void shouldNotProvideNativeSpecificPropertiesForJvmCollector() { + assertTrue(NativeCollectorDefaults.defaultProperties(false).isEmpty()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index d4cffe23fb7..7e0de3eda8a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java index 16b3087e895..de65033abef 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java @@ -17,10 +17,6 @@ package org.apache.hertzbeat.collector.collect.common.ssh; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.Scheduler; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -35,7 +31,6 @@ import java.io.IOException; import java.net.ServerSocket; import java.security.GeneralSecurityException; -import java.time.Duration; import java.util.Comparator; import java.util.ArrayList; import java.util.Iterator; @@ -53,34 +48,7 @@ public class SshTunnelHelper { private static final long DEFAULT_CACHE_TIMEOUT = 500 * 1000; - private static final Cache TRACKER_CACHE = - Caffeine.newBuilder() - .initialCapacity(1) - .maximumSize(1000) - .expireAfterAccess(Duration.ofMillis(DEFAULT_CACHE_TIMEOUT)) - .scheduler(Scheduler.systemScheduler()) - .removalListener((key, value, cause) -> { - if (cause == RemovalCause.REPLACED) { - return; - } - if (key != null && value != null) { - // 1. try close tunnel - SshClientSessionWrapper clientSessionWrapper = (SshClientSessionWrapper) key; - LocalPortForwardingWrapper wrapper = (LocalPortForwardingWrapper) value; - wrapper.remove(clientSessionWrapper.getClientSession()); - - // 2. try close session - if (!clientSessionWrapper.isShareConnection()) { - try { - clientSessionWrapper.close(); - log.info("[SSH Tunnel] close unshared ssh connection, {}", clientSessionWrapper); - } catch (IOException e) { - log.error("[SSH Tunnel] close unshared ssh connection error", e); - } - } - } - }) - .build(); + private static final Map TRACKER_CACHE = new ConcurrentHashMap<>(); /** @@ -122,15 +90,13 @@ public static int localPortForward(SshTunnel sshTunnel, String remoteHost, Strin // 2. get tunnel LocalPortForwardingWrapper forwardingWrapper = selectWrapper( - TRACKER_CACHE.getIfPresent(sessionWrapper), sessionWrapper, remoteHost, remotePort); + TRACKER_CACHE.get(sessionWrapper), sessionWrapper, remoteHost, remotePort); int localPort; if (forwardingWrapper == null) { localPort = getRandomPort(); LocalPortForwardingWrapper newForwardingWrapper = sessionWrapper .createLocalPortForwardingTracker(localPort, remoteHost, Integer.parseInt(remotePort)); - if (TRACKER_CACHE.getIfPresent(sessionWrapper) == null) { - TRACKER_CACHE.put(sessionWrapper, newForwardingWrapper); - } + TRACKER_CACHE.putIfAbsent(sessionWrapper, newForwardingWrapper); log.info("[SSH Tunnel] created ssh forwarding tracker ssh:{}, remote:{}, localPort:{}", sshTunnel.getHost() + ":" + sshTunnel.getPort(), remoteHost + ":" + remotePort, localPort); } else { @@ -184,6 +150,23 @@ private static int getRandomPort() throws IOException { } } + private static void removeSessionCacheEntry(ClientSession session) { + TRACKER_CACHE.entrySet().removeIf(entry -> { + if (!Objects.equals(entry.getKey().getClientSession(), session)) { + return false; + } + if (!entry.getKey().isShareConnection()) { + try { + entry.getKey().close(); + log.info("[SSH Tunnel] close unshared ssh connection, {}", entry.getKey()); + } catch (IOException e) { + log.error("[SSH Tunnel] close unshared ssh connection error", e); + } + } + return true; + }); + } + @Getter @Setter @EqualsAndHashCode @@ -267,6 +250,10 @@ public List select(ClientSession session, Predicate< list.add(wrapper); } } + if (trackerList.isEmpty()) { + map.remove(session); + removeSessionCacheEntry(session); + } return list; } @@ -290,6 +277,8 @@ public void remove(ClientSession session) { log.error("[SSH Tunnel] Remove ssh session local port forwarding error", e); } } + map.remove(session); + removeSessionCacheEntry(session); } public void close() throws IOException { diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml index 9acd5c98d94..7175050e818 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -47,4 +47,4 @@ kafka-clients - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml index 1cdd2070f38..472dc55c3b9 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -46,4 +46,4 @@ mongodb-driver-sync - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml index b21d4554633..9ea2e9d914e 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -57,4 +57,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index 48daddead48..bbb03e841d5 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -28,7 +28,7 @@ ${project.artifactId} pom - 21 + 25 ${java.version} ${java.version} diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml index af0d937df12..316661344be 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-basic-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 42.5.5 diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml index 323e3c57b7e..1e2952bdf28 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-common-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -67,4 +67,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml index d56825f6def..3c1824951d1 100644 --- a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-kafka-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml index c725c7bd35f..a5638e23a20 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-log-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 3.3.1 3.6.1 diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index dbcbad9c5df..445a8a50776 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -36,8 +36,8 @@ true - 17 - 17 + ${java.version} + ${java.version} UTF-8 2.0.3 4.13.2 diff --git a/home/docs/community/contribution.md b/home/docs/community/contribution.md index 72b823e716d..e1651205688 100644 --- a/home/docs/community/contribution.md +++ b/home/docs/community/contribution.md @@ -52,7 +52,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` diff --git a/home/docs/community/development.md b/home/docs/community/development.md index fd2c83327cf..c52e5c075e1 100644 --- a/home/docs/community/development.md +++ b/home/docs/community/development.md @@ -12,7 +12,7 @@ sidebar_label: Development ### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` @@ -34,7 +34,7 @@ sidebar_label: Development ## Build HertzBeat binary package -> Requires `maven3+`, `java21`, `node` and `pnpm` environments. +> Requires `maven3+`, `java25`, `node` and `pnpm` environments. ### Frontend build @@ -50,20 +50,23 @@ sidebar_label: Development ### Backend build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean package -Prelease` -The HertzBeat install package will at `dist/hertzbeat-{version}.tar.gz` +The HertzBeat install package will be generated at `dist/apache-hertzbeat-{version}-bin.tar.gz` ### Collector build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean install` 3. Cd to the `hertzbeat-collector` directory: `cd hertzbeat-collector` -4. Execute under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +4. Build the JVM collector package under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +5. Build the native collector package under `hertzbeat-collector` directory: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -The HertzBeat collector package will at `dist/hertzbeat-collector-{version}.tar.gz` +> Native collector packaging requires GraalVM for JDK 25 with the `native-image` tool available in `PATH`. + +The HertzBeat collector packages will be generated at `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` and a platform-specific native package such as `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` diff --git a/home/docs/community/how-to-release.md b/home/docs/community/how-to-release.md index 6deb97d01b2..c92feedccb6 100644 --- a/home/docs/community/how-to-release.md +++ b/home/docs/community/how-to-release.md @@ -10,7 +10,7 @@ This tutorial describes in detail how to release Apache HertzBeat™, take the r This release process is operated in the UbuntuOS(Windows,Mac), and the following tools are required: -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,23 @@ mvn clean install mvn clean package -Pcluster ``` +> Build the native collector binaries with GraalVM JDK 25 and `native-image` on matching GitHub-hosted runners or equivalent local hosts + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> The repository workflow `.github/workflows/collector-native-build.yml` can build and upload all supported native collector packages on GitHub-hosted runners. Release signing and final publishing remain a release manager responsibility. + The release package are here: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 Package the source code diff --git a/home/docs/download.md b/home/docs/download.md index fde53959f4b..b6d62fa5f06 100644 --- a/home/docs/download.md +++ b/home/docs/download.md @@ -22,9 +22,15 @@ Download the latest Apache HertzBeat™ release (v1.8.0) as server binary, colle |-------------|------|---------|----------| | **Server Binary** | ~200MB | Main monitoring server | Linux, macOS, Windows | | **Collector Binary** | ~50MB | Distributed collectors | Linux, macOS, Windows | -| **Source Code** | ~30MB | Build from source | Any with Java 21+ | +| **Source Code** | ~30MB | Build from source | Any with Java 25+ | | **Docker Compose** | ~5MB | Full stack deployment | Docker environments | +:::tip Native Collector Recommendation +If you do not need MySQL, OceanBase, Oracle, DB2, or other monitoring types that rely on external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. + +Trade-offs: native packages are platform-specific and do not support runtime `ext-lib` JDBC loading. See [Native Collector Guide](start/native-collector). +::: + :::tip Security Verification Verify downloads using GPG signatures and SHA512 checksums. See [Apache Verification Guide](https://www.apache.org/dyn/closer.cgi#verify) and [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS). ::: @@ -57,6 +63,9 @@ For older releases, please check the [archive](https://archive.apache.org/dist/i **Server Binary** - For most users. Includes the main HertzBeat monitoring server with web UI. **Collector Binary** - For distributed deployments. Deploy collectors in remote networks to report to the main server. +Native collector downloads are platform-specific, for example `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`. + +If you are deciding between JVM and native collector packages, start with [Native Collector Guide](start/native-collector). **Source Code** - For developers who want to build, modify, or contribute to HertzBeat. @@ -73,17 +82,19 @@ Import Apache HertzBeat KEYS first: `wget https://downloads.apache.org/hertzbeat ### What are the system requirements? **Server Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 4GB RAM minimum (8GB recommended) - 2 CPU cores minimum - 20GB disk space **Collector Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 2GB RAM minimum - 1 CPU core minimum - 5GB disk space +Native collector packages are published per target platform, while the JVM collector package remains cross-platform. + ### Can I use Docker instead of binary packages? Yes. Docker is the recommended installation method: diff --git a/home/docs/help/db2.md b/home/docs/help/db2.md index 5652094fa92..9258d261634 100644 --- a/home/docs/help/db2.md +++ b/home/docs/help/db2.md @@ -16,6 +16,13 @@ keywords: [ open source monitoring tool, open source database monitoring tool, m - Copy the JAR package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +DB2 monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for DB2 monitoring +- Do not use the native collector package for DB2 monitoring +::: + ### Configuration Parameters The following are the required configuration parameters for DB2 monitoring: diff --git a/home/docs/help/mysql.md b/home/docs/help/mysql.md index f21a1a1da87..7d62f7c0a46 100644 --- a/home/docs/help/mysql.md +++ b/home/docs/help/mysql.md @@ -14,6 +14,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +MySQL monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for MySQL monitoring +- Do not use the native collector package for MySQL monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/oceanbase.md b/home/docs/help/oceanbase.md index 510934771f5..d4fe52aeb18 100644 --- a/home/docs/help/oceanbase.md +++ b/home/docs/help/oceanbase.md @@ -13,6 +13,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +OceanBase monitoring depends on the external MySQL JDBC driver in `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for OceanBase monitoring +- Do not use the native collector package for OceanBase monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/oracle.md b/home/docs/help/oracle.md index 83d7854030e..562e68cfcfd 100644 --- a/home/docs/help/oracle.md +++ b/home/docs/help/oracle.md @@ -13,6 +13,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +Oracle monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for Oracle monitoring +- Do not use the native collector package for Oracle monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/risc-v.md b/home/docs/help/risc-v.md index 41d605afe84..a7d2ee5d467 100644 --- a/home/docs/help/risc-v.md +++ b/home/docs/help/risc-v.md @@ -48,7 +48,7 @@ Reference: [Ubuntu Official Documentation](https://canonical-ubuntu-boards.readt ## Install and Configure RISC-V-Compatible JDK -> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 21. +> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 25. **1. Download Temurin JDK** Download link: [https://adoptium.net/temurin/releases](https://adoptium.net/temurin/releases) @@ -66,11 +66,11 @@ sudo apt install -y tar wget sudo mkdir -p /usr/lib/jvm # Extract to system directory -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # Edit environment variables sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # Apply configuration @@ -91,6 +91,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > Notes: > -> 1. Replace `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` with your actual JDK filename. +> 1. Replace `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` with your actual JDK filename. > 2. Ensure the emulator has internet access to download HertzBeat. > 3. If issues arise, verify disk permissions and Java environment paths. diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md new file mode 100644 index 00000000000..9f16f217b15 --- /dev/null +++ b/home/docs/start/native-collector.md @@ -0,0 +1,75 @@ +--- +id: native-collector +title: Native Collector Guide +sidebar_label: Native Collector +description: When to choose the HertzBeat native collector package, its benefits, limitations, and deployment guidance. +--- + +## When should I choose the native collector? + +Choose the native collector package when your monitoring workload does not depend on loading external JDBC drivers from `ext-lib`. + +Typical native-friendly workloads include: + +- HTTP, HTTPS, website availability, and API checks +- Port, ping, SSL certificate, and other network probes +- Redis, Zookeeper, Kafka, and other non-JDBC monitoring types + +## Why use it? + +Compared with the JVM collector package, the native collector package is usually a better fit when you want: + +- Faster startup +- Lower baseline memory usage +- A simpler runtime without a bundled or preinstalled JDK + +## What are the trade-offs? + +The native collector package is not a drop-in replacement for every JVM collector scenario. + +- Native packages are platform-specific. You must choose the package that matches your OS and CPU architecture. +- The native collector does not support loading external JDBC driver JARs from `ext-lib` at runtime. +- If your deployment depends on JVM-style runtime classpath extension, keep using the JVM collector package. + +## When should I stay on the JVM collector? + +Use the JVM collector package if your monitoring depends on external JDBC drivers, especially: + +- MySQL, which requires `mysql-connector-j` +- OceanBase, which also depends on the MySQL JDBC driver +- Oracle, which requires `ojdbc8` and sometimes `orai18n` +- DB2, which requires `jcc` + +## Package naming + +The JVM collector package remains cross-platform: + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +The native collector package is platform-specific: + +- Linux or macOS: `apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows: `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +Examples: + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## Configuration consistency + +The native collector package uses the same `config/application.yml` layout as the JVM collector package. + +That means: + +- Collector connection settings are edited in the same place +- Virtual-thread related configuration is edited in the same place +- Native-only boot adjustments are applied by code at runtime instead of maintaining a second `application.yml` + +## Recommended decision + +- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. +- Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. + +For package deployment steps, refer to [Install HertzBeat via Package](package-deploy). diff --git a/home/docs/start/package-deploy.md b/home/docs/start/package-deploy.md index 0747d7821d9..f2e8981f7cd 100644 --- a/home/docs/start/package-deploy.md +++ b/home/docs/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: Install via Package :::tip You can install and run Apache HertzBeat™ on Linux Windows Mac system, and CPU supports X86/ARM64. -Since version 1.6.0 uses `Java 21` and the installation package no longer provides a built-in JDK version, use the new Hertzbeat according to the following situations: +The current branch uses `Java 25`, and the standard installation package no longer provides a built-in JDK. Use HertzBeat according to the following situations: -- When the default environment variable on your server is `Java 21`, you do not need to take any action for this step. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`, and if there are no other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and search the engine for how to set a new environment variable pointing to the new `Java 21`. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`,and you don't want to change the environment variable because if there are other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and rename the extracted folder to `java`, then copy it to the Hertzbeat extraction directory. +- When the default environment variable on your server is `Java 25`, you do not need to take any action for this step. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and if there are no other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, and set a new environment variable pointing to `Java 25`. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and you do not want to change the environment variable because there are other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, rename the extracted folder to `java`, and then copy it to the HertzBeat extraction directory. ::: @@ -64,11 +64,20 @@ HertzBeat Collector is a lightweight data collector used to collect and send dat Deploying multiple HertzBeat Collectors can achieve high availability, load balancing, and cloud-edge collaboration of data. ::: +:::tip Native Collector Recommendation +If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. + +Before choosing it, review the trade-offs in [Native Collector Guide](native-collector). +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. Download installation package - Download installation package `apache-hertzbeat-collector-xxx-bin.tar.gz` corresponding to your system environment + Download the collector package that matches your deployment mode: + - JVM collector package: `apache-hertzbeat-collector-xxx-bin.tar.gz` + - Native collector package for Linux or macOS: `apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Native collector package for Windows: `apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` - [Download Page](/docs/download) 2. Configure the collector configuration file @@ -77,6 +86,10 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # or + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # or + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` Configure the collector configuration yml file `config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` @@ -102,11 +115,31 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 3. Start the service - Run command `$ ./bin/startup.sh` or `bin/startup.bat` + Run `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. 4. Begin to explore HertzBeat Collector - Access `http://ip:1157` and you will see the registered new collector in dashboard + Open the HertzBeat server dashboard at `http://:1157` and confirm the new collector is registered. + +:::important Native Collector Limitations +The native collector package is suitable for monitoring types that do not rely on external JVM classpath extension. + +See [Native Collector Guide](native-collector) for package selection, package naming, and platform-specific trade-offs. + +`ext-lib`-based JDBC driver loading is a JVM collector capability. The native collector package does not support loading external JDBC driver JARs from `ext-lib` at runtime. + +If your monitoring depends on external JDBC drivers, use the JVM collector package instead of the native collector package. This currently includes: + +- MySQL, which requires `mysql-connector-j` +- OceanBase, which also relies on the MySQL JDBC driver +- Oracle, which requires `ojdbc8` and often `orai18n` +- DB2, which requires `jcc` + +Recommended deployment: + +- Use the native collector package for HTTP, website, port, ping, and similar non-JDBC monitoring types +- Use the JVM collector package when you need `ext-lib` driver extension +::: **HAVE FUN** @@ -117,15 +150,15 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 1. you need to prepare the JAVA environment in advance Install JAVA runtime environment-refer to [official website](https://www.oracle.com/java/technologies/downloads/) - requirement：JDK21 ENV + requirement：JDK25 ENV download JAVA installation package: [mirror website](https://mirrors.huaweicloud.com/openjdk/) After installation use command line to check whether you install it successfully. ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/docs/start/quickstart.md b/home/docs/start/quickstart.md index ddb24cab1e5..8e4830ec023 100644 --- a/home/docs/start/quickstart.md +++ b/home/docs/start/quickstart.md @@ -13,7 +13,7 @@ Install Apache HertzBeat™ in under 5 minutes using Docker with a single comman ## Installation Methods -HertzBeat provides four installation options: +HertzBeat provides multiple installation options: 1. **Docker** (Recommended) - Fastest setup, production-ready 2. **Binary Package** - Traditional deployment with manual configuration @@ -54,12 +54,13 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters(Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download Page](https://hertzbeat.apache.org/docs/download) + - If you do not need MySQL, OceanBase, Oracle, DB2, or other `ext-lib` JDBC drivers, prefer the native collector package for faster startup and lower memory usage. See [Native Collector Guide](native-collector). + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your target platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download Page](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml @@ -74,15 +75,17 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh` or `bin/startup.bat` - - Access `http://localhost:1157` and you will see the registered new collector in dashboard + - Native collector trade-offs: platform-specific packages, no runtime `ext-lib` JDBC loading, and less suitable for JVM-style runtime classpath extension. See [Native Collector Guide](native-collector). + - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - Run command `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. + - Access the HertzBeat server dashboard at `http://localhost:1157` and confirm the new collector is registered. Detailed config refer to [Install HertzBeat via Package](package-deploy) ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `manager` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` @@ -108,7 +111,7 @@ Detailed steps refer to [Artifact Hub](https://artifacthub.io/packages/helm/hert - 2 CPU cores - 4GB RAM - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ **Operating Systems:** Linux, macOS, Windows (via Docker or WSL) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index f9d36190316..06ee9518a8e 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: Virtual Threads description: Configure HertzBeat virtual-thread executors, defaults, rollback switches, and tuning guidance. --- -HertzBeat runs on JDK 21 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. +HertzBeat runs on JDK 25 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. ## 1. Where to Configure It diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md index fe530b8bc01..3748879cf55 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md @@ -52,7 +52,7 @@ limitations under the License. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md index 76b20f0d2ac..3df906d2f8e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md @@ -11,7 +11,7 @@ sidebar_label: 运行编译 ### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` @@ -37,7 +37,7 @@ sidebar_label: 运行编译 ## 生成二进制包 -> 需要 `maven3+`, `java21`, `node` 和 `pnpm` 环境. +> 需要 `maven3+`, `java25`, `node` 和 `pnpm` 环境. ### 前端打包 @@ -53,20 +53,23 @@ sidebar_label: 运行编译 ### 后端打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean package -Prelease` -HertzBeat 包将生成为 `dist/hertzbeat-{version}.tar.gz` +HertzBeat 包将生成为 `dist/apache-hertzbeat-{version}-bin.tar.gz` ### 采样器打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean install` 3. 切换到 `hertzbeat-collector` 目录: `cd hertzbeat-collector` -4. 在 `hertzbeat-collector` 目录下执行: `mvn clean package -Pcluster` +4. 在 `hertzbeat-collector` 目录下打 JVM 采集器安装包: `mvn clean package -Pcluster` +5. 在 `hertzbeat-collector` 目录下打 Native 采集器安装包: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -HertzBeat 采样器包将生成为 `dist/hertzbeat-collector-{version}.tar.gz` +> Native 采集器打包需要带有 `native-image` 命令的 GraalVM JDK 25 环境。 + +HertzBeat 采集器安装包将生成为 `dist/apache-hertzbeat-collector-{version}-bin.tar.gz`，以及类似 `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 这样的 Native 平台安装包 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md index b0d5ae4239e..e5a8594d134 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md @@ -10,7 +10,7 @@ sidebar_position: 4 此发布过程在 UbuntuOS(可在 Windows Mac) 中进行操作，并需要以下环境： -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,23 @@ mvn clean install mvn clean package -Pcluster ``` +> 在匹配的平台 GitHub Hosted Runner 或对应本地主机上，使用带 `native-image` 的 GraalVM JDK 25 构建 Native 采集器安装包 + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> 仓库中的 `.github/workflows/collector-native-build.yml` 可以在 GitHub Hosted Runner 上构建并上传全部受支持平台的 Native 采集器安装包。正式发布时的签名与最终上传仍由 release manager 负责。 + 生成的二进制包在: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 打包项目源代码 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md index 2325660f32d..1b467661198 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md @@ -22,9 +22,15 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 |--------|------|------|------| | **服务器二进制** | ~200MB | 主监控服务器 | Linux、macOS、Windows | | **采集器二进制** | ~50MB | 分布式采集器 | Linux、macOS、Windows | -| **源码** | ~30MB | 从源码构建 | 任何支持 Java 21+ 的平台 | +| **源码** | ~30MB | 从源码构建 | 任何支持 Java 25+ 的平台 | | **Docker Compose** | ~5MB | 全栈部署 | Docker 环境 | +:::tip Native 采集器推荐 +如果你不需要 MySQL、OceanBase、Oracle、DB2，或其他依赖 `ext-lib` 外部 JDBC 驱动的监控类型，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。 + +它的代价是安装包按平台区分，且不支持运行时 `ext-lib` JDBC 加载。详见 [Native 采集器指南](start/native-collector)。 +::: + :::tip 安全验证 使用 GPG 签名和 SHA512 校验和验证下载。参见 [Apache 验证指南](https://www.apache.org/dyn/closer.cgi#verify) 和 [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS)。 ::: @@ -57,6 +63,9 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 **服务器二进制** - 大多数用户使用。包含主 HertzBeat 监控服务器和 Web UI。 **采集器二进制** - 分布式部署使用。在远程网络部署采集器向主服务器上报。 +Native 采集器下载包按目标平台区分，例如 `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 或 `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`。 + +如果你正在 JVM 采集器和 Native 采集器之间做选择，建议先阅读 [Native 采集器指南](start/native-collector)。 **源码** - 开发者想要构建、修改或贡献 HertzBeat 时使用。 @@ -73,17 +82,19 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 ### 系统要求是什么？ **服务器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 4GB RAM 最低（推荐 8GB） - 2 CPU 核心 最低 - 20GB 磁盘空间 **采集器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 2GB RAM 最低 - 1 CPU 核心 最低 - 5GB 磁盘空间 +Native 采集器安装包会按目标平台分别发布，JVM 采集器安装包仍然保持跨平台。 + ### 可以用 Docker 代替二进制包吗？ 可以。Docker 是推荐的安装方法： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md index 0c2f9bb7e32..0dcba300aee 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md @@ -15,6 +15,13 @@ keywords: [ 开源监控系统, 开源数据库监控, DB2数据库监控 ] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下 - 重启 HertzBeat 服务 +:::important 采集器包选择 +DB2 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- DB2 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 DB2 监控 +::: + ### 配置参数 以下是 DB2 监控所需的配置参数： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md index 82b3f36b564..bc7e723a558 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, Mysql数据库监控] - 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. - 重启 HertzBeat 服务。 +:::important 采集器包选择 +MySQL 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- MySQL 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 MySQL 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md index edeeaeaf611..5893d9c1957 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, OceanBase 数据库监控] - 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. - 重启 HertzBeat 服务。 +:::important 采集器包选择 +OceanBase 监控同样依赖 `ext-lib` 目录下的 MySQL JDBC 驱动。 + +- OceanBase 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 OceanBase 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md index f15d3dc62b5..153ef72fcb4 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下。 - 重启 HertzBeat 服务。 +:::important 采集器包选择 +Oracle 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- Oracle 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 Oracle 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md index f931b561024..50b7da7a250 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md @@ -45,7 +45,7 @@ qemu-system-riscv64 \ ## 安装并配置支持 RISC-V 架构的 JDK -> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 21。 +> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 25。 **1. 下载 Temurin JDK** @@ -63,11 +63,11 @@ sudo apt install -y tar wget # 创建安装目录 sudo mkdir -p /usr/lib/jvm # 解压到系统目录 -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # 编辑环境变量，添加以下内容 sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # 使配置生效 @@ -88,6 +88,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > 注意事项： > -> 1. 请将 `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` 替换为您实际下载的 JDK 文件名。 +> 1. 请将 `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` 替换为您实际下载的 JDK 文件名。 > 2. 确保模拟器具备网络访问能力，以下载 HertzBeat。 > 3. 若遇到问题，请检查磁盘权限和 Java 环境路径配置。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md new file mode 100644 index 00000000000..1d26af32a47 --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -0,0 +1,75 @@ +--- +id: native-collector +title: Native 采集器指南 +sidebar_label: Native 采集器 +description: 说明 HertzBeat Native 采集器安装包适合什么场景、优缺点、限制和部署建议。 +--- + +## 什么场景适合使用 Native 采集器？ + +当你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动时，优先考虑 Native 采集器安装包。 + +比较适合 Native 采集器的场景包括： + +- HTTP、HTTPS、网站可用性、API 检查 +- 端口可用性、Ping、SSL 证书等网络探测 +- Redis、Zookeeper、Kafka 等非 JDBC 监控类型 + +## 为什么选择它？ + +相较 JVM 采集器安装包，Native 采集器安装包通常更适合以下诉求： + +- 启动更快 +- 常驻内存更低 +- 运行时更轻，不需要额外准备 bundled 或预装 JDK + +## 它的缺点和限制是什么？ + +Native 采集器并不是所有 JVM 采集器场景的无损替代。 + +- Native 安装包是平台相关的，必须选择与你操作系统和 CPU 架构匹配的包。 +- Native 采集器不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 +- 如果你的部署依赖 JVM 风格的运行时 classpath 扩展能力，仍然应该使用 JVM 采集器安装包。 + +## 哪些场景应该继续使用 JVM 采集器？ + +如果你的监控依赖外部 JDBC 驱动，请继续使用 JVM 采集器安装包，尤其包括： + +- MySQL，需要 `mysql-connector-j` +- OceanBase，同样依赖 MySQL JDBC 驱动 +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` + +## 安装包命名规则 + +JVM 采集器安装包仍然保持跨平台： + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +Native 采集器安装包按平台区分： + +- Linux 或 macOS：`apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows：`apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +例如： + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## 配置文件是否和 JVM 采集器一致？ + +Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/application.yml` 结构。 + +这意味着： + +- 采集器连接参数仍然在同一个位置修改 +- 虚拟线程相关配置仍然在同一个位置修改 +- Native 专用的启动调整通过代码在运行时生效，而不是长期维护第二份 `application.yml` + +## 推荐选择 + +- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 +- 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 + +具体安装步骤可参考 [通过安装包安装 HertzBeat](package-deploy)。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md index b34768ac984..3091205add9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: 安装包方式安装 :::tip Apache HertzBeat™ 支持在Linux Windows Mac系统安装运行，CPU支持X86/ARM64。 -由于1.6.0及以后版本使用 `Java 21` ，且安装包不再提供内置jdk的版本，参考以下情况使用新版Hertzbeat。 +当前分支默认使用 `Java 25`，且标准安装包不再提供内置 JDK。可参考以下情况使用 HertzBeat： -- 当你的服务器中默认环境变量为 `Java 21` 时，这一步你无需任何操作。 -- 当你的服务器中默认环境变量不为 `Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**没有**其他应用需要低版本 `Java` ，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并在搜索引擎搜索如何设置新的环境变量指向新的`Java 21`。 -- 当你的服务器中默认环境变量不为`Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**有**其他应用需要低版本 `Java` ，你不想更改环境变量，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并将解压后的文件夹重命名为`java`，复制到Hertzbeat的解压目录下。 +- 当你的服务器中默认环境变量为 `Java 25` 时，这一步无需任何操作。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**没有**其他应用需要低版本 `Java`，根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将环境变量指向新的 `Java 25`。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**有**其他应用需要低版本 `Java`，不希望修改全局环境变量，可根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将解压后的文件夹重命名为 `java`，复制到 HertzBeat 的解压目录下。 ::: @@ -63,11 +63,21 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 通过部署多个 HertzBeat Collector 可以实现数据的高可用，负载均衡和云边协同。 ::: +:::tip Native 采集器推荐 +如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。 + +在选择前，建议先阅读 [Native 采集器指南](native-collector) 了解它的限制和取舍。 +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. 下载安装包 - 从 [下载页面](/docs/download) 下载您系统环境对应的安装包版本 `apache-hertzbeat-collector-xxx-bin.tar.gz` + 按部署形态选择对应的采集器安装包： + - JVM 采集器安装包：`apache-hertzbeat-collector-xxx-bin.tar.gz` + - Linux 或 macOS 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Windows 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` + - 从 [下载页面](/docs/download) 下载 2. 设置配置文件 @@ -75,6 +85,10 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # 或 + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # 或 + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` 配置采集器的配置文件 `config/application.yml` 里面的 HertzBeat Server 连接 IP, 端口, 采集器名称(需保证唯一性)等参数。 @@ -100,14 +114,30 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 3. 启动 - 执行位于安装目录 hertzbeat-collector/bin/ 下的启动脚本 startup.sh, windows 环境下为 startup.bat - - ```shell - ./startup.sh - ``` + JVM 采集器安装包执行位于安装目录 `hertzbeat-collector/bin/` 下的启动脚本 `startup.sh`，Windows 环境下为 `startup.bat`；Linux 或 macOS 的 Native 采集器安装包执行 `./startup.sh`，Windows 的 Native 采集器安装包执行 `bin\\startup.bat` 4. 开始探索 HertzBeat Collector - 浏览器访问 [http://ip:1157/](http://ip:1157/) 即可开始探索使用，默认账户密码 admin/hertzbeat。 + 浏览器访问主 HertzBeat 服务 [http://manager-host:1157/](http://manager-host:1157/) 的概览页面，即可确认新采集器已注册。 + +:::important Native 采集器限制说明 +Native 采集器适合不依赖外部 JVM classpath 扩展的监控类型。 + +关于包选择、安装包命名和平台相关限制，详见 [Native 采集器指南](native-collector)。 + +基于 `ext-lib` 的 JDBC 驱动加载能力是 JVM 采集器的能力。Native 采集器当前不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 + +因此，凡是依赖外置 JDBC 驱动的监控类型，请使用 JVM 采集器，不要使用 Native 采集器。当前至少包括： + +- MySQL，需要 `mysql-connector-j` +- OceanBase，同样依赖 MySQL JDBC 驱动 +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` + +建议部署方式： + +- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型优先使用 Native 采集器 +- 需要 `ext-lib` 扩展驱动时使用 JVM 采集器 +::: **HAVE FUN** @@ -120,15 +150,15 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 1. 启动失败，需您提前准备JAVA运行环境 安装JAVA运行环境-可参考[官方网站](https://www.oracle.com/java/technologies/downloads/) - 要求：JAVA21环境 + 要求：JAVA25环境 下载JAVA安装包: [镜像站](https://mirrors.huaweicloud.com/openjdk/) 安装后命令行检查是否成功安装 ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md index 9723ac7357a..0031ecebd45 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md @@ -13,7 +13,7 @@ description: Apache HertzBeat 监控系统快速安装指南 - Docker、安装 ## 安装方式 -HertzBeat 提供四种安装选项： +HertzBeat 提供多种安装选项： 1. **Docker**（推荐）- 最快设置，生产就绪 2. **二进制包** - 传统部署，手动配置 @@ -58,12 +58,13 @@ HertzBeat 提供四种安装选项： #### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包`hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml`(可选) 3. 部署启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群(可选) - - 下载您系统环境对应采集器安装包`hertzbeat-collector-xx.tar.gz`到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) + - 如果你不需要 MySQL、OceanBase、Oracle、DB2 这类依赖 `ext-lib` JDBC 驱动的监控，优先选择 Native 采集器安装包，通常启动更快、内存更省。详见 [Native 采集器指南](native-collector)。 + - 下载您系统环境对应采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或匹配目标平台的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主HertzBeat服务的对外IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml @@ -78,15 +79,17 @@ HertzBeat 提供四种安装选项： manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` - - 浏览器访问主HertzBeat服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 + - Native 采集器的代价是安装包按平台区分、不支持运行时 `ext-lib` JDBC 加载，也不适合依赖 JVM 风格运行时 classpath 扩展的场景。详见 [Native 采集器指南](native-collector)。 + - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包 + - JVM 采集器安装包使用 `$ ./bin/startup.sh` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 + - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](package-deploy) #### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程`hertzbeat-startup`和前端工程`web-app` -2. 后端：需要`maven3+`, `java21`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 +2. 后端：需要`maven3+`, `java25`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 3. 前端：需要`nodejs npm angular-cli`环境，待本地后端启动后，在`web-app`目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` @@ -112,7 +115,7 @@ HertzBeat 提供四种安装选项： - 2 CPU 核心 - 4GB RAM - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ **支持系统：** Linux、macOS、Windows（通过 Docker 或 WSL） diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index ae69b63e0a9..020f5376b5f 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: 虚拟线程 description: 说明 HertzBeat 虚拟线程执行器的默认值、回滚开关和调优方式。 --- -HertzBeat 基于 JDK 21 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 +HertzBeat 基于 JDK 25 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 ## 1. 到哪里配置 diff --git a/home/sidebars.json b/home/sidebars.json index bda68ea497e..c40c129b177 100755 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -15,6 +15,7 @@ "start/docker-deploy", "start/docker-compose-deploy", "start/package-deploy", + "start/native-collector", { "type": "link", "label": "Install via Helm", diff --git a/home/src/components/StructuredData.js b/home/src/components/StructuredData.js index 97472f05e6a..cbeeb493187 100644 --- a/home/src/components/StructuredData.js +++ b/home/src/components/StructuredData.js @@ -55,7 +55,7 @@ export default function StructuredData() { "Cloud-edge collaboration for isolated networks", "Status page builder for service communication" ], - "softwareRequirements": "Docker 20.10+ or Java 21+", + "softwareRequirements": "Docker 20.10+ or Java 25+", "memoryRequirements": "4GB minimum, 8GB recommended", "processorRequirements": "2 CPU cores minimum", "storageRequirements": "10GB minimum" diff --git a/home/src/pages/faq.js b/home/src/pages/faq.js index cc6d74af086..cc45b8c4d58 100644 --- a/home/src/pages/faq.js +++ b/home/src/pages/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "What are HertzBeat's system requirements?", - answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 21+. Supported on Linux, macOS, Windows." + answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 25+. Supported on Linux, macOS, Windows." }, { question: "How do I upgrade HertzBeat?", diff --git a/home/src/pages/zh-cn/faq.js b/home/src/pages/zh-cn/faq.js index ae16a05b411..f422546dfca 100644 --- a/home/src/pages/zh-cn/faq.js +++ b/home/src/pages/zh-cn/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "HertzBeat 的系统要求是什么？", - answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 21+。支持系统：Linux、macOS、Windows。" + answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 25+。支持系统：Linux、macOS、Windows。" }, { question: "如何升级 HertzBeat？", diff --git a/home/static/llms-zh.txt b/home/static/llms-zh.txt index 0ae302d4691..8971f7a70f7 100644 --- a/home/static/llms-zh.txt +++ b/home/static/llms-zh.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU 核心 最低 - 4GB RAM 最低（推荐 8GB） - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ - 平台：Linux、macOS、Windows ## 支持的监控类型 diff --git a/home/static/llms.txt b/home/static/llms.txt index 18ab6d7bbf9..65ef12af761 100644 --- a/home/static/llms.txt +++ b/home/static/llms.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU cores minimum - 4GB RAM minimum (8GB recommended) - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ - Platforms: Linux, macOS, Windows ## Supported Monitoring Types diff --git a/pom.xml b/pom.xml index fdbdcd0960a..341a12c9066 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 1.8.0 - 21 + 25 ${java.version} ${java.version} 3.2.0 @@ -610,6 +610,10 @@ org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} + + + full + org.apache.maven.plugins diff --git a/script/assembly/collector/assembly-native.xml b/script/assembly/collector/assembly-native.xml new file mode 100644 index 00000000000..c341eaffd05 --- /dev/null +++ b/script/assembly/collector/assembly-native.xml @@ -0,0 +1,82 @@ + + + + ${native.package.id} + true + ${native.package.baseDirectory} + + ${native.package.format} + + + + + ${native.launcher.dir} + true + bin + 0755 + + + + src/main/resources + + application.yml + logback-spring.xml + banner.txt + + true + ${file.separator}config + + + + ../../ + ${file.separator} + + README.md + + + + ../../material/licenses/collector + ${file.separator} + + LICENSE + NOTICE + + + + ../../material/licenses/collector + licenses + + LICENSE-* + + + + + + + ${native.binary.source} + ${file.separator} + ${native.executable.packageName} + 0755 + + + diff --git a/script/assembly/collector/assembly.xml b/script/assembly/collector/assembly.xml index f4af4618f28..d02ee97e0cc 100644 --- a/script/assembly/collector/assembly.xml +++ b/script/assembly/collector/assembly.xml @@ -58,6 +58,9 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> banner.txt META-INF/** + + META-INF/spring.factories + true ${file.separator}config @@ -68,7 +71,7 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> target / - *.jar + ${project.build.finalName}.jar diff --git a/script/assembly/collector/bin-native-win/restart.bat b/script/assembly/collector/bin-native-win/restart.bat new file mode 100644 index 00000000000..50521a8d85b --- /dev/null +++ b/script/assembly/collector/bin-native-win/restart.bat @@ -0,0 +1,25 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +echo Restarting Apache HertzBeat ${project.artifactId} ... +call "%~dp0shutdown.bat" +timeout /t 2 /nobreak >nul +call "%~dp0startup.bat" +exit /b %ERRORLEVEL% diff --git a/script/assembly/collector/bin-native-win/shutdown.bat b/script/assembly/collector/bin-native-win/shutdown.bat new file mode 100644 index 00000000000..eb1dc08e316 --- /dev/null +++ b/script/assembly/collector/bin-native-win/shutdown.bat @@ -0,0 +1,49 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set SERVER_PORT=1159 + +set PID= +if exist "%PID_FILE%" ( + set /p PID=<"%PID_FILE%" +) + +if not defined PID ( + for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$conn = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1; if ($conn) { $conn.OwningProcess }"') do set PID=%%i +) + +if not defined PID ( + echo Apache HertzBeat %SERVER_NAME% is already stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %PID% -ErrorAction SilentlyContinue) { Stop-Process -Id %PID% -Force; exit 0 } exit 1" +del /q "%PID_FILE%" >nul 2>&1 + +echo Shutdown Apache HertzBeat %SERVER_NAME% Success! +exit /b 0 diff --git a/script/assembly/collector/bin-native-win/startup.bat b/script/assembly/collector/bin-native-win/startup.bat new file mode 100644 index 00000000000..33bfec4f48c --- /dev/null +++ b/script/assembly/collector/bin-native-win/startup.bat @@ -0,0 +1,102 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} +set BINARY_NAME=${native.executable.packageName} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set CONF_DIR=%DEPLOY_DIR%\config +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set APP_PATH=%DEPLOY_DIR%\%BINARY_NAME% +set SERVER_PORT=1159 +set STDOUT_LOG=%LOGS_DIR%\startup.out.log +set STDERR_LOG=%LOGS_DIR%\startup.err.log + +if "%1"=="status" goto status + +if not exist "%APP_PATH%" ( + echo ERROR: native executable not found: %APP_PATH% + exit /b 1 +) + +if not exist "%LOGS_DIR%" ( + mkdir "%LOGS_DIR%" +) + +if exist "%PID_FILE%" ( + set /p RUNNING_PID=<"%PID_FILE%" + powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" + if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% already started! + echo PID: %RUNNING_PID% + exit /b 1 + ) + del /q "%PID_FILE%" >nul 2>&1 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$portInUse = Get-NetTCPConnection -State Listen -LocalPort %SERVER_PORT% -ErrorAction SilentlyContinue; if ($portInUse) { exit 0 } exit 1" +if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% port %SERVER_PORT% is already used! + exit /b 1 +) + +echo You can review logs at hertzbeat\logs +echo Starting the HertzBeat %SERVER_NAME% ... + +for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$p = Start-Process -FilePath ''%APP_PATH%'' -ArgumentList ''--spring.config.location=%CONF_DIR%\'' -RedirectStandardOutput ''%STDOUT_LOG%'' -RedirectStandardError ''%STDERR_LOG%'' -PassThru; $p.Id"') do set APP_PID=%%i + +if not defined APP_PID ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + exit /b 1 +) + +>"%PID_FILE%" echo %APP_PID% + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$deadline = (Get-Date).AddSeconds(30); do { Start-Sleep -Seconds 1; if (-not (Get-Process -Id %APP_PID% -ErrorAction SilentlyContinue)) { exit 1 } $listening = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Where-Object { $_.OwningProcess -eq %APP_PID% }; if ($listening) { exit 0 } } while ((Get-Date) -lt $deadline); exit 0" +if errorlevel 1 ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + del /q "%PID_FILE%" >nul 2>&1 + exit /b 1 +) + +echo Service Start Success! +echo Service PID: %APP_PID% +exit /b 0 + +:status +if not exist "%PID_FILE%" ( + echo The HertzBeat %SERVER_NAME% is stopped + exit /b 0 +) + +set /p RUNNING_PID=<"%PID_FILE%" +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" +if errorlevel 1 ( + echo The HertzBeat %SERVER_NAME% is stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +echo The HertzBeat %SERVER_NAME% is running...! +echo PID: %RUNNING_PID% +exit /b 0 diff --git a/script/assembly/collector/bin-native/restart.sh b/script/assembly/collector/bin-native/restart.sh new file mode 100644 index 00000000000..af640ba89f1 --- /dev/null +++ b/script/assembly/collector/bin-native/restart.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +startTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Now!\033[0m" +./shutdown.sh +echo +sleep 2 +echo +./startup.sh +endTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Success！Spend $((endTime - startTime)) seconds \033[0m" diff --git a/script/assembly/collector/bin-native/shutdown.sh b/script/assembly/collector/bin-native/shutdown.sh new file mode 100644 index 00000000000..1c0478edbac --- /dev/null +++ b/script/assembly/collector/bin-native/shutdown.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +PID="$(find_running_pid)" +if [ -z "$PID" ]; then + echo "Apache HertzBeat ${SERVER_NAME} is already stopped" + rm -f "$PID_FILE" + exit 0 +fi + +kill "$PID" +for _ in $(seq 1 30); do + if ! kill -0 "$PID" 2>/dev/null; then + rm -f "$PID_FILE" + echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" + exit 0 + fi + sleep 1 +done + +kill -9 "$PID" 2>/dev/null +rm -f "$PID_FILE" +echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" diff --git a/script/assembly/collector/bin-native/startup.sh b/script/assembly/collector/bin-native/startup.sh new file mode 100644 index 00000000000..bd57563fad8 --- /dev/null +++ b/script/assembly/collector/bin-native/startup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +BIN_DIR="$(pwd)" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" +SERVER_PORT=1159 + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +RUNNING_PID="$(find_running_pid)" +if [ "$1" = "status" ]; then + if [ -n "$RUNNING_PID" ]; then + echo "The HertzBeat $SERVER_NAME is running...!" + echo "PID: $RUNNING_PID" + else + echo "The HertzBeat $SERVER_NAME is stopped" + fi + exit 0 +fi + +if [ ! -x "$APP_PATH" ]; then + echo "ERROR: native executable not found: $APP_PATH" + exit 1 +fi + +if [ -n "$RUNNING_PID" ]; then + echo "ERROR: The HertzBeat $SERVER_NAME already started!" + echo "PID: $RUNNING_PID" + exit 1 +fi + +mkdir -p "$LOGS_DIR" + +if command -v lsof >/dev/null 2>&1; then + SERVER_PORT_COUNT="$(lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | wc -l)" + if [ "$SERVER_PORT_COUNT" -gt 0 ]; then + echo "ERROR: The HertzBeat $SERVER_NAME port $SERVER_PORT is already used!" + exit 1 + fi +fi + +echo "You can review logs at hertzbeat/logs" +echo "Starting the HertzBeat $SERVER_NAME ..." +nohup "$APP_PATH" --spring.config.location="$CONF_DIR/" >"$LOGS_DIR/startup.log" 2>&1 & +APP_PID=$! +echo "$APP_PID" >"$PID_FILE" + +COUNT=0 +while [ $COUNT -lt 30 ]; do + sleep 1 + if ! kill -0 "$APP_PID" 2>/dev/null; then + echo "ERROR: Service start failed, check $LOGS_DIR/startup.log" + rm -f "$PID_FILE" + exit 1 + fi + if command -v lsof >/dev/null 2>&1 && lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | grep -q "$APP_PID"; then + break + fi + COUNT=$((COUNT + 1)) +done + +echo "Service Start Success!" +echo "Service PID: $APP_PID" diff --git a/script/assembly/collector/bin/startup.sh b/script/assembly/collector/bin/startup.sh index 1d2db5f9c27..dbdab8c581b 100644 --- a/script/assembly/collector/bin/startup.sh +++ b/script/assembly/collector/bin/startup.sh @@ -103,7 +103,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/assembly/server/bin/startup.sh b/script/assembly/server/bin/startup.sh index ebf25d7ae30..45bff0dc90f 100644 --- a/script/assembly/server/bin/startup.sh +++ b/script/assembly/server/bin/startup.sh @@ -107,7 +107,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/ci/github-actions/setup-deps/action.yml b/script/ci/github-actions/setup-deps/action.yml index 0e1698fe720..3ad4560a589 100644 --- a/script/ci/github-actions/setup-deps/action.yml +++ b/script/ci/github-actions/setup-deps/action.yml @@ -21,11 +21,11 @@ description: Install host system dependencies (with mvnd) runs: using: composite steps: - - name: Set up JDK 21 + - name: Set up JDK 25 uses: actions/setup-java@v4 with: distribution: "zulu" - java-version: 21 + java-version: 25 - name: Install mvnd shell: bash @@ -40,4 +40,4 @@ runs: - name: Verify mvnd installation shell: bash - run: mvnd --version \ No newline at end of file + run: mvnd --version diff --git a/script/docker/collector/Dockerfile b/script/docker/collector/Dockerfile index a6f89d8c89d..5a33eb5b392 100644 --- a/script/docker/collector/Dockerfile +++ b/script/docker/collector/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" @@ -26,7 +26,7 @@ RUN sed -i 's#http://#https://#g' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && apt-get install -y openssh-server RUN mkdir /var/run/sshd -ADD apache-hertzbeat-collector-*-bin.tar.gz /opt/ +ADD apache-hertzbeat-collector-${VERSION}-bin.tar.gz /opt/ ENV JAVA_OPTS "" ENV TZ=Asia/Shanghai diff --git a/script/docker/collector/build.sh b/script/docker/collector/build.sh index 278df32c783..03f93312b0d 100644 --- a/script/docker/collector/build.sh +++ b/script/docker/collector/build.sh @@ -21,13 +21,20 @@ cd `dirname $0` CURRENT_DIR=`pwd` # cd dist dir cd ../../../dist -# auto detect hertzbeat version -VERSION=`ls apache-hertzbeat-collector-*-bin.tar.gz| awk -F"-" '{print $4}'` +# auto detect the JVM collector package version +PACKAGE_FILE=$(find . -maxdepth 1 -name 'apache-hertzbeat-collector-*-bin.tar.gz' \ + ! -name 'apache-hertzbeat-collector-native-*-bin.tar.gz' | head -n 1) +VERSION=$(basename "$PACKAGE_FILE" | sed -e 's/^apache-hertzbeat-collector-//' -e 's/-bin\.tar\.gz$//') # use the version param if [ -n "$1" ]; then VERSION="$1"; fi +if [ -z "$VERSION" ]; then + echo "Can not find the JVM collector package under dist/. Build apache-hertzbeat-collector-{version}-bin.tar.gz first." + exit 1 +fi + # docker compile context CONTEXT_DIR=`pwd` diff --git a/script/docker/server/Dockerfile b/script/docker/server/Dockerfile index 773c21c0d15..6427e6300b9 100644 --- a/script/docker/server/Dockerfile +++ b/script/docker/server/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" diff --git a/script/ext-lib/README b/script/ext-lib/README index 5898fde6b91..0afb3b3bbec 100644 --- a/script/ext-lib/README +++ b/script/ext-lib/README @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder for JVM-based server / collector packages, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +jcc-11.5.9.0.jar +Note: + +- `ext-lib` is loaded by the JVM server package and the JVM collector package. +- The native collector package does not support loading external JDBC driver jars from `ext-lib` at runtime. +- If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers, use the JVM collector package. From 3809599a8c98093d0e6ac7d2e81af615559bc66b Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 23:29:57 +0800 Subject: [PATCH 11/14] build: fix native release packaging workflow --- .github/workflows/collector-native-build.yml | 22 +++++-------------- .../hertzbeat-collector-collector/pom.xml | 9 ++++++++ home/docs/community/how-to-release.md | 6 +++-- home/docs/start/native-collector.md | 5 +++++ .../current/community/how-to-release.md | 6 +++-- .../current/start/native-collector.md | 5 +++++ 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/collector-native-build.yml b/.github/workflows/collector-native-build.yml index 97a14b3e8fd..d44877dba8f 100644 --- a/.github/workflows/collector-native-build.yml +++ b/.github/workflows/collector-native-build.yml @@ -15,37 +15,26 @@ # specific language governing permissions and limitations # under the License. -name: Collector Native CI +name: Collector Native Release + +run-name: Native collector release build (${{ github.ref_name }}) on: workflow_dispatch: - push: - branches: [ action* ] - paths: - - '.github/workflows/collector-native-build.yml' - - 'pom.xml' - - 'hertzbeat-collector/**' - - 'script/assembly/collector/**' - pull_request: - branches: [ master, dev ] - paths: - - '.github/workflows/collector-native-build.yml' - - 'pom.xml' - - 'hertzbeat-collector/**' - - 'script/assembly/collector/**' jobs: build-native-collector: name: Native collector (${{ matrix.platform }}) permissions: contents: read + timeout-minutes: 120 runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - platform: linux-amd64 - runner: ubuntu-latest + runner: ubuntu-24.04 archive_ext: tar.gz - platform: linux-arm64 runner: ubuntu-24.04-arm @@ -97,3 +86,4 @@ jobs: with: name: apache-hertzbeat-collector-native-${{ matrix.platform }} path: ${{ steps.package.outputs.archive }} + retention-days: 14 diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index d40bdf60abb..e4f32ac721f 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -412,6 +412,15 @@ org.graalvm.buildtools native-maven-plugin 0.11.4 + + + build-native-image + package + + compile-no-fork + + + ${native.image.name} diff --git a/home/docs/community/how-to-release.md b/home/docs/community/how-to-release.md index c92feedccb6..9c92b940958 100644 --- a/home/docs/community/how-to-release.md +++ b/home/docs/community/how-to-release.md @@ -211,13 +211,15 @@ mvn clean install mvn clean package -Pcluster ``` -> Build the native collector binaries with GraalVM JDK 25 and `native-image` on matching GitHub-hosted runners or equivalent local hosts +> Build the native collector binary for the current host with GraalVM JDK 25 and `native-image` ```shell mvn clean package -pl hertzbeat-collector-collector -am -Pnative ``` -> The repository workflow `.github/workflows/collector-native-build.yml` can build and upload all supported native collector packages on GitHub-hosted runners. Release signing and final publishing remain a release manager responsibility. +> The repository workflow `.github/workflows/collector-native-build.yml` is a release helper, not a regular PR or push CI workflow. +> +> It is intentionally manual-only because multi-platform native builds are relatively slow and consume scarce Linux ARM, macOS, and Windows runners. During release preparation, open the Actions page, select `Collector Native Release`, run it from the release branch or tag, and then download the uploaded artifacts for signing and publishing. The release package are here: diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md index 9f16f217b15..e1affbbc040 100644 --- a/home/docs/start/native-collector.md +++ b/home/docs/start/native-collector.md @@ -72,4 +72,9 @@ That means: - Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. - Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. +## How are the official multi-platform packages built? + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` builds a native collector package for the current host only. +- The official Linux, macOS, and Windows native release packages are produced by manually running the `Collector Native Release` GitHub Actions workflow during release preparation, not on every push or pull request. + For package deployment steps, refer to [Install HertzBeat via Package](package-deploy). diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md index e5a8594d134..7415ca15f9e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md @@ -211,13 +211,15 @@ mvn clean install mvn clean package -Pcluster ``` -> 在匹配的平台 GitHub Hosted Runner 或对应本地主机上，使用带 `native-image` 的 GraalVM JDK 25 构建 Native 采集器安装包 +> 使用带 `native-image` 的 GraalVM JDK 25，为当前宿主机构建 Native 采集器安装包 ```shell mvn clean package -pl hertzbeat-collector-collector -am -Pnative ``` -> 仓库中的 `.github/workflows/collector-native-build.yml` 可以在 GitHub Hosted Runner 上构建并上传全部受支持平台的 Native 采集器安装包。正式发布时的签名与最终上传仍由 release manager 负责。 +> 仓库中的 `.github/workflows/collector-native-build.yml` 是 release 辅助工作流，不参与日常 PR 或 push 的常规 CI。 +> +> 之所以只保留手动触发，是因为跨平台 Native 构建耗时更长，也会占用相对稀缺的 Linux ARM、macOS 和 Windows Runner。准备发版时，请在 GitHub Actions 页面选择 `Collector Native Release`，基于 release 分支或 tag 手动触发，然后下载上传的产物用于签名和发布。 生成的二进制包在: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md index 1d26af32a47..6ef2037be31 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -72,4 +72,9 @@ Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/appli - 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 - 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 +## 官方多平台安装包是怎么构建的？ + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` 只会为当前宿主机构建一个 Native 采集器安装包。 +- 官方发布使用的 Linux、macOS、Windows Native 安装包，会在发布准备阶段手动触发 `Collector Native Release` GitHub Actions 工作流来生成，而不是在每次 push 或 pull request 时自动构建。 + 具体安装步骤可参考 [通过安装包安装 HertzBeat](package-deploy)。 From a5b406f20ab8a48d5c621f4731b0a361299d5355 Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 23:41:37 +0800 Subject: [PATCH 12/14] chore: add license header for spring factories --- .../src/main/resources/META-INF/spring.factories | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories index 16c40f8c6f3..731da64e615 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories @@ -1,2 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Intentionally left blank. # Native collector defaults are applied from Collector.main. From 0abb23e77f24a81452b247e5f8d312f514567949 Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 14 Mar 2026 11:26:02 +0800 Subject: [PATCH 13/14] feat: add MySQL R2DBC query engine support and update documentation --- README.md | 3 +- README_CN.md | 3 +- README_JP.md | 3 +- .../hertzbeat-collector-basic/pom.xml | 6 - .../collect/database/JdbcCommonCollect.java | 267 ++++++++---- .../database/query/JdbcQueryExecutor.java | 30 ++ .../query/JdbcQueryExecutorRegistry.java | 54 +++ .../database/query/JdbcQueryRowSet.java | 33 ++ .../hertzbeat-collector-collector/pom.xml | 31 +- .../mysql/MysqlCollectorProperties.java | 53 +++ .../mysql/MysqlJdbcDriverAvailability.java | 80 ++++ .../mysql/MysqlR2dbcJdbcQueryExecutor.java | 214 ++++++++++ .../strategy/CollectStrategyFactory.java | 10 + .../src/main/resources/application.yml | 7 + ...bcQueryAdapterTemplateIntegrationTest.java | 320 ++++++++++++++ .../MysqlJdbcDriverAvailabilityTest.java | 41 ++ ...ryAdapterCompatibilityIntegrationTest.java | 312 ++++++++++++++ ...bcQueryAdapterTemplateIntegrationTest.java | 322 ++++++++++++++ .../MysqlJdbcQueryParityIntegrationTest.java | 393 ++++++++++++++++++ .../MysqlR2dbcJdbcQueryExecutorTest.java | 143 +++++++ ...anbaseJdbcQueryAdapterIntegrationTest.java | 275 ++++++++++++ .../TidbJdbcQueryAdapterIntegrationTest.java | 240 +++++++++++ .../strategy/CollectStrategyFactoryTest.java | 34 ++ .../hertzbeat-collector-mysql-r2dbc/pom.xml | 77 ++++ .../mysql/r2dbc/MysqlQueryExecutor.java | 33 ++ .../mysql/r2dbc/MysqlR2dbcConfiguration.java | 55 +++ .../MysqlR2dbcConnectionFactoryProvider.java | 67 +++ .../mysql/r2dbc/MysqlR2dbcQueryExecutor.java | 156 +++++++ .../collector/mysql/r2dbc/QueryOptions.java | 60 +++ .../collector/mysql/r2dbc/QueryResult.java | 46 ++ .../mysql/r2dbc/ResultSetMapper.java | 95 +++++ .../collector/mysql/r2dbc/SqlGuard.java | 85 ++++ ...ysqlR2dbcQueryExecutorIntegrationTest.java | 201 +++++++++ .../MysqlSqlTemplateCompatibilityTest.java | 65 +++ .../mysql/r2dbc/ResultSetMapperTest.java | 81 ++++ .../collector/mysql/r2dbc/SqlGuardTest.java | 53 +++ hertzbeat-collector/pom.xml | 14 + .../pom.xml | 124 ++++++ .../AbstractMysqlR2dbcCollectE2eTest.java | 233 +++++++++++ ...MysqlR2dbcCollectCompatibilityE2eTest.java | 61 +++ .../mysql/MysqlR2dbcCollectE2eTest.java | 41 ++ hertzbeat-e2e/pom.xml | 1 + hertzbeat-startup/pom.xml | 18 + .../src/main/resources/application.yml | 7 + .../ReactorNettyCompatibilityTest.java | 54 +++ .../StartupMysqlR2dbcCompatibilityTest.java | 116 ++++++ home/docs/download.md | 2 +- home/docs/help/mariadb.md | 18 +- home/docs/help/mysql.md | 19 +- home/docs/help/oceanbase.md | 17 +- home/docs/help/tidb.md | 16 + home/docs/start/docker-deploy.md | 6 +- home/docs/start/native-collector.md | 8 +- home/docs/start/package-deploy.md | 7 +- home/docs/start/quickstart.md | 4 +- .../current/download.md | 2 +- .../current/help/mariadb.md | 18 +- .../current/help/mysql.md | 18 +- .../current/help/oceanbase.md | 17 +- .../current/help/tidb.md | 16 + .../current/start/docker-deploy.md | 6 +- .../current/start/native-collector.md | 8 +- .../current/start/package-deploy.md | 7 +- .../current/start/quickstart.md | 4 +- material/licenses/NOTICE | 16 + material/licenses/backend/LICENSE | 3 + material/licenses/collector/LICENSE | 3 + material/licenses/collector/NOTICE | 16 + pom.xml | 24 ++ script/application.yml | 7 + script/docker-compose/README.md | 7 +- .../hertzbeat-mysql-iotdb/README.md | 7 +- .../hertzbeat-mysql-iotdb/README_CN.md | 7 +- .../conf/application.yml | 7 + .../hertzbeat-mysql-iotdb/docker-compose.yaml | 1 + .../hertzbeat-mysql-iotdb/ext-lib/README | 7 +- .../hertzbeat-mysql-tdengine/README.md | 7 +- .../hertzbeat-mysql-tdengine/README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../hertzbeat-mysql-tdengine/ext-lib/README | 7 +- .../README.md | 7 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- .../hertzbeat-postgresql-greptimedb/README.md | 8 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- .../README.md | 8 +- .../README_CN.md | 7 +- .../conf/application.yml | 7 + .../docker-compose.yaml | 1 + .../ext-lib/README | 7 +- script/ext-lib/README | 6 +- 97 files changed, 4842 insertions(+), 197 deletions(-) create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java create mode 100644 hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java diff --git a/README.md b/README.md index e6b5b7dbf7a..eb8cac645f8 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,8 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - If you do not provide JDBC drivers in `ext-lib`, MySQL, MariaDB, and OceanBase can use the built-in query engine and run on the native collector package as well. TiDB follows the same rule for its SQL query metric set. + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC after restart for MySQL, MariaDB, and OceanBase. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics are unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. - Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access `http://localhost:1157` and you will see the registered new collector in dashboard diff --git a/README_CN.md b/README_CN.md index fda8d7ca640..80058e8f7b4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -145,7 +145,8 @@ manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包。 + - 如果没有在 `ext-lib` 中提供 JDBC 驱动，MySQL、MariaDB、OceanBase 可以直接使用内置查询引擎，也可以使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动。 - JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 diff --git a/README_JP.md b/README_JP.md index f8993253319..4487344ce90 100644 --- a/README_JP.md +++ b/README_JP.md @@ -148,7 +148,8 @@ - `mode: ${MODE:public}`：実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。 - `manager-host: ${MANAGER_HOST:127.0.0.1}`：メインhertzbeatサーバーのIP。 - `manager-port: ${MANAGER_PORT:1158}`：メインhertzbeatサーバポート。 - - `ext-lib` で MySQL、OceanBase、Oracle、DB2 などの外部 JDBC ドライバーを読み込む必要がある場合は、JVM コレクターのインストールパッケージを使用してください。 + - `ext-lib` に JDBC ドライバーを置かない場合、MySQL、MariaDB、OceanBase は組み込みのクエリエンジンを使って Native コレクターパッケージでも監視できます。TiDB も SQL クエリのメトリクスセットについては同じルールです。 + - `ext-lib` に `mysql-connector-j` を置いた場合は、再起動後に組み込みサーバーコレクターまたは JVM コレクターが MySQL、MariaDB、OceanBase で自動的に JDBC を優先します。TiDB も SQL クエリのメトリクスセットについては同じルールで、HTTP メトリクスは影響を受けません。Oracle と DB2 は引き続き外部 JDBC ドライバーに依存するため、JVM コレクターパッケージを使用してください。 - JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。 - メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。 diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index de107950e41..63c88e262df 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -67,12 +67,6 @@ commons-net commons-net - - - com.mysql - mysql-connector-j - provided - com.clickhouse diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java index 42529c72842..76e9421a6a2 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java @@ -20,7 +20,6 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; @@ -34,6 +33,9 @@ import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier; import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache; import org.apache.hertzbeat.collector.collect.common.cache.JdbcConnect; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; import org.apache.hertzbeat.collector.constants.CollectorConstants; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; @@ -205,31 +207,26 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException { public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { long startTime = System.currentTimeMillis(); JdbcProtocol jdbcProtocol = metrics.getJdbc(); - SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); - int timeout = CollectUtil.getTimeout(jdbcProtocol.getTimeout()); boolean reuseConnection = Boolean.parseBoolean(jdbcProtocol.getReuseConnection()); - Statement statement = null; - String databaseUrl; try { - if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { - int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - databaseUrl = constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); - } else { - databaseUrl = constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - } - - statement = getConnection(jdbcProtocol.getUsername(), - jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); switch (jdbcProtocol.getQueryType()) { - case QUERY_TYPE_ONE_ROW -> queryOneRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_MULTI_ROW -> queryMultiRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_COLUMNS -> queryOneRowByMatchTwoColumns(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case RUN_SCRIPT -> { - Connection connection = statement.getConnection(); - FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); - ScriptUtils.executeSqlScript(connection, rc); + case QUERY_TYPE_ONE_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1)) { + queryOneRow(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case QUERY_TYPE_MULTI_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryMultiRow(rowSet, metrics.getAliasFields(), builder, startTime); + } } + case QUERY_TYPE_COLUMNS -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryOneRowByMatchTwoColumns(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case RUN_SCRIPT -> runScript(metrics, timeout, reuseConnection); default -> { builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Not support database query type: " + jdbcProtocol.getQueryType()); @@ -261,23 +258,6 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { log.error("Jdbc error: {}.", errorMessage, e); builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Query Error: " + errorMessage); - } finally { - if (statement != null) { - Connection connection = null; - try { - connection = statement.getConnection(); - statement.close(); - } catch (Exception e) { - log.error("Jdbc close statement error: {}", e.getMessage()); - } - try { - if (!reuseConnection && connection != null) { - connection.close(); - } - } catch (Exception e) { - log.error("Jdbc close connection error: {}", e.getMessage()); - } - } } } @@ -286,6 +266,52 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_JDBC; } + private JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, boolean reuseConnection, int maxRows) throws Exception { + Optional executor = JdbcQueryExecutorRegistry.resolve(metrics); + if (executor.isPresent()) { + return executor.get().executeQuery(metrics, timeout, maxRows); + } + return executeJdbcQuery(metrics.getJdbc(), timeout, reuseConnection, maxRows); + } + + private JdbcQueryRowSet executeJdbcQuery(JdbcProtocol jdbcProtocol, int timeout, boolean reuseConnection, + int maxRows) throws Exception { + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + statement.setMaxRows(maxRows); + return new ResultSetJdbcQueryRowSet(statement, statement.executeQuery(jdbcProtocol.getSql()), reuseConnection); + } catch (Exception exception) { + closeStatementAndConnection(statement, reuseConnection); + throw exception; + } + } + + private void runScript(Metrics metrics, int timeout, boolean reuseConnection) throws Exception { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + Connection connection = statement.getConnection(); + FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); + ScriptUtils.executeSqlScript(connection, rc); + } finally { + closeStatementAndConnection(statement, reuseConnection); + } + } + + private String resolveDatabaseUrl(JdbcProtocol jdbcProtocol) throws Exception { + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + return constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); + } + return constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + } private Statement getConnection(String username, String password, String url, Integer timeout, boolean reuseConnection) throws Exception { CacheIdentifier identifier = CacheIdentifier.builder() @@ -343,29 +369,25 @@ private Statement getConnection(String username, String password, String url, In * query metrics：one tow three four * query sql：select one, tow, three, four from book limit 1; * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRow(Statement statement, String sql, List columns, + private void queryOneRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - statement.setMaxRows(1); - try (ResultSet resultSet = statement.executeQuery(sql)) { - if (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + if (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } } @@ -380,33 +402,30 @@ private void queryOneRow(Statement statement, String sql, List columns, * three - value3 * four - value4 * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List columns, + private void queryOneRowByMatchTwoColumns(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - HashMap values = new HashMap<>(columns.size()); - while (resultSet.next()) { - if (resultSet.getString(1) != null) { - values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2)); - } + HashMap values = new HashMap<>(columns.size()); + while (rowSet.next()) { + if (rowSet.getString(1) != null) { + values.put(rowSet.getString(1).toLowerCase().trim(), rowSet.getString(2)); } - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = values.get(column.toLowerCase()); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + } + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = values.get(column.toLowerCase()); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } /** @@ -416,28 +435,45 @@ private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List< * query sql：select one, tow, three, four from book; * and return multi row record mapping with the metrics * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryMultiRow(Statement statement, String sql, List columns, + private void queryMultiRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + while (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); + } + } + + private void closeStatementAndConnection(Statement statement, boolean reuseConnection) { + if (statement == null) { + return; + } + Connection connection = null; + try { + connection = statement.getConnection(); + statement.close(); + } catch (Exception exception) { + log.error("Jdbc close statement error: {}", exception.getMessage()); + } + try { + if (!reuseConnection && connection != null) { + connection.close(); + } + } catch (Exception exception) { + log.error("Jdbc close connection error: {}", exception.getMessage()); } } @@ -548,4 +584,53 @@ private String constructDatabaseUrl(JdbcProtocol jdbcProtocol, String host, Stri default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform()); }; } + + private static final class ResultSetJdbcQueryRowSet implements JdbcQueryRowSet { + + private final Statement statement; + private final java.sql.ResultSet resultSet; + private final boolean reuseConnection; + + private ResultSetJdbcQueryRowSet(Statement statement, java.sql.ResultSet resultSet, boolean reuseConnection) { + this.statement = statement; + this.resultSet = resultSet; + this.reuseConnection = reuseConnection; + } + + @Override + public boolean next() throws Exception { + return resultSet.next(); + } + + @Override + public String getString(String column) throws Exception { + return resultSet.getString(column); + } + + @Override + public String getString(int index) throws Exception { + return resultSet.getString(index); + } + + @Override + public void close() throws Exception { + Connection connection = null; + try { + connection = statement.getConnection(); + } catch (Exception ignored) { + // ignore + } + try { + resultSet.close(); + } finally { + try { + statement.close(); + } finally { + if (!reuseConnection && connection != null) { + connection.close(); + } + } + } + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java new file mode 100644 index 00000000000..3d3c9076da3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Adapter point for replacing only the SQL query execution part of JdbcCommonCollect. + */ +public interface JdbcQueryExecutor { + + boolean supports(Metrics metrics); + + JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java new file mode 100644 index 00000000000..bb9f1c0baa6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Static registry used by JdbcCommonCollect to discover optional query executors. + */ +public final class JdbcQueryExecutorRegistry { + + private static final List EXECUTORS = new CopyOnWriteArrayList<>(); + + private JdbcQueryExecutorRegistry() { + } + + public static void register(JdbcQueryExecutor executor) { + if (executor == null || EXECUTORS.contains(executor)) { + return; + } + EXECUTORS.add(executor); + } + + public static void unregister(JdbcQueryExecutor executor) { + if (executor == null) { + return; + } + EXECUTORS.remove(executor); + } + + public static Optional resolve(Metrics metrics) { + return EXECUTORS.stream() + .filter(executor -> executor.supports(metrics)) + .findFirst(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java new file mode 100644 index 00000000000..a93cef11d77 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +/** + * A minimal row cursor abstraction shared by JDBC and R2DBC-backed database queries. + */ +public interface JdbcQueryRowSet extends AutoCloseable { + + boolean next() throws Exception; + + String getString(String column) throws Exception; + + String getString(int index) throws Exception; + + @Override + void close() throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index e4f32ac721f..5fcca5107d9 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -48,6 +48,12 @@ ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + + org.apache.hertzbeat @@ -99,6 +105,23 @@ io.micrometer micrometer-registry-prometheus + + + org.springframework.boot + spring-boot-starter-test + test + + + com.mysql + mysql-connector-j + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + @@ -365,14 +388,6 @@ apache-hertzbeat-collector-native-${hzb.version} - - - src/main/resources - - META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect - - - org.apache.maven.plugins diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java new file mode 100644 index 00000000000..ac34a37ab84 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Collector-side MySQL query engine routing. + */ +@Getter +@Setter +@ConfigurationProperties(prefix = "hertzbeat.collector.mysql") +public class MysqlCollectorProperties { + + private QueryEngine queryEngine = QueryEngine.AUTO; + + public QueryEngine resolveQueryEngine(boolean mysqlJdbcDriverAvailable) { + if (queryEngine == QueryEngine.AUTO) { + return mysqlJdbcDriverAvailable ? QueryEngine.JDBC : QueryEngine.R2DBC; + } + return queryEngine; + } + + public boolean useR2dbc(boolean mysqlJdbcDriverAvailable) { + return resolveQueryEngine(mysqlJdbcDriverAvailable) == QueryEngine.R2DBC; + } + + /** + * Supported collector-side query engines. + */ + public enum QueryEngine { + AUTO, + JDBC, + R2DBC + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java new file mode 100644 index 00000000000..8c1466aedeb --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URL; +import java.security.CodeSource; +import java.util.Locale; +import org.springframework.stereotype.Component; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * Detects whether a MySQL JDBC driver is available from the external ext-lib path. + */ +@Component +public class MysqlJdbcDriverAvailability { + + private static final String[] MYSQL_DRIVER_CLASSES = { + "com.mysql.cj.jdbc.Driver", + "com.mysql.jdbc.Driver" + }; + + public boolean hasMysqlJdbcDriver() { + ClassLoader classLoader = ClassUtils.getDefaultClassLoader(); + for (String driverClass : MYSQL_DRIVER_CLASSES) { + if (!ClassUtils.isPresent(driverClass, classLoader)) { + continue; + } + try { + if (isExternalExtLibDriver(ClassUtils.forName(driverClass, classLoader))) { + return true; + } + } catch (ClassNotFoundException ignored) { + // Race-free enough for runtime detection: keep probing other known driver class names. + } + } + return false; + } + + boolean isExternalExtLibDriver(Class driverClass) { + String location = resolveLocation(driverClass); + return isExtLibLocation(location); + } + + static boolean isExtLibLocation(String location) { + if (!StringUtils.hasText(location)) { + return false; + } + String normalized = location + .replace('\\', '/') + .toLowerCase(Locale.ROOT); + return normalized.contains("/ext-lib/"); + } + + private String resolveLocation(Class driverClass) { + CodeSource codeSource = driverClass.getProtectionDomain().getCodeSource(); + if (codeSource != null && codeSource.getLocation() != null) { + return codeSource.getLocation().toExternalForm(); + } + String resourceName = ClassUtils.convertClassNameToResourcePath(driverClass.getName()) + ".class"; + ClassLoader classLoader = driverClass.getClassLoader(); + URL resource = classLoader != null ? classLoader.getResource(resourceName) : ClassLoader.getSystemResource(resourceName); + return resource != null ? resource.toExternalForm() : null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java new file mode 100644 index 00000000000..55837095c60 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URI; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.SshTunnel; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +/** + * MySQL-compatible query-only adapter that lets JdbcCommonCollect execute read-only queries through the built-in + * R2DBC path when no MySQL JDBC driver is present. + */ +@Component +public class MysqlR2dbcJdbcQueryExecutor implements JdbcQueryExecutor, InitializingBean, DisposableBean { + + private static final String QUERY_TYPE_ONE_ROW = "oneRow"; + private static final String QUERY_TYPE_MULTI_ROW = "multiRow"; + private static final String QUERY_TYPE_COLUMNS = "columns"; + + private final MysqlCollectorProperties properties; + private final MysqlQueryExecutor mysqlQueryExecutor; + private final MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability; + + public MysqlR2dbcJdbcQueryExecutor(MysqlCollectorProperties properties, + MysqlQueryExecutor mysqlQueryExecutor, + MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability) { + this.properties = properties; + this.mysqlQueryExecutor = mysqlQueryExecutor; + this.mysqlJdbcDriverAvailability = mysqlJdbcDriverAvailability; + } + + @Override + public boolean supports(Metrics metrics) { + if (metrics == null || metrics.getJdbc() == null) { + return false; + } + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + if (!isMysqlCompatiblePlatform(jdbcProtocol.getPlatform())) { + return false; + } + String queryType = jdbcProtocol.getQueryType(); + return properties.useR2dbc(mysqlJdbcDriverAvailability.hasMysqlJdbcDriver()) + && (QUERY_TYPE_ONE_ROW.equals(queryType) + || QUERY_TYPE_MULTI_ROW.equals(queryType) + || QUERY_TYPE_COLUMNS.equals(queryType)); + } + + private boolean isMysqlCompatiblePlatform(String platform) { + return "mysql".equalsIgnoreCase(platform) || "mariadb".equalsIgnoreCase(platform); + } + + @Override + public JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + QueryOptions options = buildQueryOptions(jdbcProtocol, timeout, maxRows); + QueryResult queryResult = mysqlQueryExecutor.execute(jdbcProtocol.getSql(), options); + if (queryResult.hasError()) { + throw new IllegalStateException("R2DBC MySQL query failed: " + queryResult.getError()); + } + return new QueryResultRowSet(queryResult); + } + + @Override + public void afterPropertiesSet() { + JdbcQueryExecutorRegistry.register(this); + } + + @Override + public void destroy() { + JdbcQueryExecutorRegistry.unregister(this); + } + + private QueryOptions buildQueryOptions(JdbcProtocol jdbcProtocol, int timeout, int maxRows) { + MysqlTarget target = resolveTarget(jdbcProtocol); + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + String host = target.host(); + int port = target.port(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + try { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, host, String.valueOf(port)); + host = "127.0.0.1"; + port = localPort; + } catch (Exception exception) { + throw new IllegalStateException("R2DBC MySQL query adapter failed to establish SSH tunnel", exception); + } + } + return QueryOptions.builder() + .host(host) + .port(port) + .username(jdbcProtocol.getUsername()) + .password(jdbcProtocol.getPassword()) + .database(target.database()) + .schema(target.database()) + .timeout(Duration.ofMillis(timeout)) + .maxRows(maxRows) + .fetchSize(256) + .readOnly(true) + .build(); + } + + private MysqlTarget resolveTarget(JdbcProtocol jdbcProtocol) { + if (StringUtils.hasText(jdbcProtocol.getUrl())) { + return parseJdbcUrl(jdbcProtocol.getUrl(), jdbcProtocol.getDatabase()); + } + if (!StringUtils.hasText(jdbcProtocol.getHost()) || !StringUtils.hasText(jdbcProtocol.getPort())) { + throw new IllegalArgumentException("R2DBC MySQL query adapter requires host/port or a jdbc:mysql URL"); + } + return new MysqlTarget(jdbcProtocol.getHost(), Integer.parseInt(jdbcProtocol.getPort()), jdbcProtocol.getDatabase()); + } + + private MysqlTarget parseJdbcUrl(String url, String fallbackDatabase) { + String trimmed = url.trim(); + if (!(trimmed.startsWith("jdbc:mysql://") || trimmed.startsWith("jdbc:mariadb://"))) { + throw new IllegalArgumentException("R2DBC MySQL query adapter only supports jdbc:mysql:// or jdbc:mariadb:// URLs"); + } + URI uri = URI.create(trimmed.substring("jdbc:".length())); + String host = uri.getHost(); + int port = uri.getPort() > 0 ? uri.getPort() : 3306; + if (!StringUtils.hasText(host)) { + throw new IllegalArgumentException("R2DBC MySQL query adapter URL must include a host"); + } + String path = uri.getPath(); + String database = StringUtils.hasText(path) && path.length() > 1 ? path.substring(1) : fallbackDatabase; + return new MysqlTarget(host, port, database); + } + + private record MysqlTarget(String host, int port, String database) { + } + + private static final class QueryResultRowSet implements JdbcQueryRowSet { + + private final List> rows; + private final Map columnIndexMap; + private int currentIndex = -1; + + private QueryResultRowSet(QueryResult queryResult) { + this.rows = queryResult.getRows(); + this.columnIndexMap = buildColumnIndexMap(queryResult.getColumns()); + } + + @Override + public boolean next() { + currentIndex++; + return currentIndex < rows.size(); + } + + @Override + public String getString(String column) { + Integer index = columnIndexMap.get(column.toLowerCase(Locale.ROOT)); + if (index == null) { + throw new IllegalArgumentException("Column not found in R2DBC MySQL result: " + column); + } + return getString(index + 1); + } + + @Override + public String getString(int index) { + if (currentIndex < 0 || currentIndex >= rows.size()) { + throw new IllegalStateException("R2DBC MySQL result cursor is not positioned on a row"); + } + int zeroBased = index - 1; + List row = rows.get(currentIndex); + if (zeroBased < 0 || zeroBased >= row.size()) { + throw new IllegalArgumentException("Column index out of bounds in R2DBC MySQL result: " + index); + } + return row.get(zeroBased); + } + + @Override + public void close() { + // QueryResult is fully materialized, so there is nothing left to close here. + } + + private static Map buildColumnIndexMap(List columns) { + Map indexMap = new HashMap<>(columns.size()); + for (int index = 0; index < columns.size(); index++) { + indexMap.put(columns.get(index).toLowerCase(Locale.ROOT), index); + } + return indexMap; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java index 4c6517ab905..28b9d2db88e 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java @@ -20,6 +20,7 @@ import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.springframework.boot.CommandLineRunner; import org.springframework.context.annotation.Configuration; @@ -29,6 +30,7 @@ /** * Specific metrics collection factory */ +@Slf4j @Configuration @Order(value = Ordered.HIGHEST_PRECEDENCE + 1) public class CollectStrategyFactory implements CommandLineRunner { @@ -49,10 +51,18 @@ public static AbstractCollect invoke(String protocol) { @Override public void run(String... args) throws Exception { + COLLECT_STRATEGY.clear(); // spi load and registry protocol and collect instance ServiceLoader loader = ServiceLoader.load(AbstractCollect.class, AbstractCollect.class.getClassLoader()); for (AbstractCollect collect : loader) { COLLECT_STRATEGY.put(collect.supportProtocol(), collect); } + if (COLLECT_STRATEGY.isEmpty()) { + throw new IllegalStateException( + "No collect strategies were registered. " + + "Verify META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect " + + "is present on the runtime classpath."); + } + log.info("Registered {} collect strategies: {}", COLLECT_STRATEGY.size(), COLLECT_STRATEGY.keySet()); } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 9615e547b94..08d98b88717 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -77,6 +77,13 @@ common: type: netty hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..1ac654969d0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MariadbJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mariadbTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mariadb:11.4")) + .withExposedPorts(3306) + .withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mariadbTemplateMetrics = loadMariadbTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMariadbTemplateThroughJdbcQueryAdapter() { + return mariadbTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) || "slow_sql".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mariadb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMariadbTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mariadb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mariaDb) throws Exception { + execRoot(mariaDb, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mariaDb); + } + + private void generateSlowQuery(GenericContainer mariaDb) throws Exception { + execUser(mariaDb, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mariaDb) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mariaDb.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mariaDb, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MariaDB entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MariaDB TCP login to become ready"); + } + + private void execRoot(GenericContainer mariaDb, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mariaDb, String database, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java new file mode 100644 index 00000000000..9c5cabe8428 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class MysqlJdbcDriverAvailabilityTest { + + @Test + void shouldTreatOnlyExtLibLocationsAsAutoJdbcSignal() { + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("/opt/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("file:/C:/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation("/Users/dev/.m2/repository/com/mysql/mysql-connector-j/9.0.0/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation(null)); + } + + @Test + void shouldIgnoreTestClasspathMysqlDriverWhenItIsNotFromExtLib() { + MysqlJdbcDriverAvailability availability = new MysqlJdbcDriverAvailability(); + + assertFalse(availability.hasMysqlJdbcDriver()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java new file mode 100644 index 00000000000..38d7265ddb8 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterCompatibilityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final Set REPRESENTATIVE_TEMPLATE_METRICS = Set.of("basic", "process_state"); + + private List representativeTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + representativeTemplateMetrics = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> REPRESENTATIVE_TEMPLATE_METRICS.contains(metric.getName())) + .collect(Collectors.toList()); + } + + @TestFactory + Stream shouldCollectRepresentativeTemplateMetricsAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyRepresentativeMetrics(target))); + } + + private void verifyRepresentativeMetrics(DatabaseTarget target) throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try (GenericContainer container = createContainer(target)) { + jdbcQueryExecutor.afterPropertiesSet(); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + for (Metrics templateMetric : representativeTemplateMetrics) { + Metrics metric = materializeMetric(templateMetric, container); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> target.name() + " " + metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> target.name() + " " + metric.getName() + " should keep the original parser output shape"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> target.name() + " basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + () -> target.name() + " basic.version should be collected"); + } + if ("process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, + () -> target.name() + " process_state should return at least one grouped state row"); + } + } + } finally { + jdbcQueryExecutor.destroy(); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the database entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..3d91c7107d6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mysqlTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mysqlTemplateMetrics = loadMysqlTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMysqlTemplateThroughJdbcQueryAdapter() { + return mysqlTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) + || "slow_sql".equals(metric.getName()) + || "account_expiry".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mysql); + } + + private void generateSlowQuery(GenericContainer mysql) throws Exception { + execUser(mysql, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mysql, String database, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java new file mode 100644 index 00000000000..650e6e92c5c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryParityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final String PARITY_TABLE = "collector_parity_metrics"; + + private Metrics basicTemplateMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + assertDoesNotThrow(() -> Class.forName("com.mysql.cj.jdbc.Driver")); + basicTemplateMetric = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-mysql.yml")); + } + + @AfterEach + void clearRegisteredExecutors() throws Exception { + Field executorsField = JdbcQueryExecutorRegistry.class.getDeclaredField("EXECUTORS"); + executorsField.setAccessible(true); + @SuppressWarnings("unchecked") + CopyOnWriteArrayList executors = (CopyOnWriteArrayList) executorsField.get(null); + executors.clear(); + } + + @TestFactory + Stream shouldMatchJdbcResultsForRepresentativeMysqlQueryShapes() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44")), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"))) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyParityAcrossTarget(target))); + } + + private void verifyParityAcrossTarget(DatabaseTarget target) throws Exception { + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initParityData(container); + + List parityMetrics = List.of( + materializeMetric(basicTemplateMetric, container), + buildColumnsParityMetric(container), + buildOneRowParityMetric(container), + buildMultiRowParityMetric(container)); + + for (Metrics parityMetric : parityMetrics) { + CollectRep.MetricsData jdbcResult = collectWithJdbc(parityMetric); + CollectRep.MetricsData r2dbcResult = collectWithR2dbc(parityMetric); + + assertEquals(CollectRep.Code.SUCCESS, jdbcResult.getCode(), + () -> target.name() + " JDBC baseline failed for " + parityMetric.getName() + ": " + jdbcResult.getMsg()); + assertEquals(CollectRep.Code.SUCCESS, r2dbcResult.getCode(), + () -> target.name() + " R2DBC path failed for " + parityMetric.getName() + ": " + r2dbcResult.getMsg()); + assertEquals(jdbcResult.getFields(), r2dbcResult.getFields(), + () -> target.name() + " field set differs for " + parityMetric.getName()); + assertEquals(normalizeRows(jdbcResult), normalizeRows(r2dbcResult), + () -> target.name() + " row payload differs for " + parityMetric.getName()); + assertFalse(r2dbcResult.getValues().isEmpty(), + () -> target.name() + " " + parityMetric.getName() + " should return at least one row"); + } + } + } + + private CollectRep.MetricsData collectWithJdbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } + + private CollectRep.MetricsData collectWithR2dbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try { + jdbcQueryExecutor.afterPropertiesSet(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } finally { + jdbcQueryExecutor.destroy(); + clearRegisteredExecutors(); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics buildColumnsParityMetric(GenericContainer container) { + return buildMetric( + "columns-parity", + List.of(field("version"), field("max_connections"), field("character_set_server")), + List.of("version", "max_connections", "character_set_server"), + "columns", + "SHOW VARIABLES WHERE Variable_name IN ('version', 'max_connections', 'character_set_server')", + container); + } + + private Metrics buildOneRowParityMetric(GenericContainer container) { + return buildMetric( + "one-row-parity", + List.of(field("answer"), field("label"), field("nullable_value")), + List.of("answer", "label", "nullable_value"), + "oneRow", + "SELECT 42 AS answer, 'adapter-parity' AS label, NULL AS nullable_value", + container); + } + + private Metrics buildMultiRowParityMetric(GenericContainer container) { + return buildMetric( + "multi-row-parity", + List.of(field("metric_name"), field("metric_value"), field("metric_note")), + List.of("metric_name", "metric_value", "metric_note"), + "multiRow", + "SELECT metric_name, metric_value, metric_note FROM " + PARITY_TABLE + " ORDER BY metric_name", + container); + } + + private Metrics buildMetric(String name, List fields, List aliasFields, + String queryType, String sql, GenericContainer container) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host(container.getHost()) + .port(String.valueOf(container.getMappedPort(3306))) + .platform("mysql") + .database(TEST_DATABASE) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .timeout(String.valueOf(Duration.ofSeconds(8).toMillis())) + .queryType(queryType) + .reuseConnection("false") + .url(buildJdbcUrl(container)) + .sql(sql) + .build(); + return Metrics.builder() + .name(name) + .protocol("jdbc") + .priority((byte) 1) + .fields(fields) + .aliasFields(aliasFields) + .jdbc(jdbcProtocol) + .build(); + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(buildJdbcUrl(container)); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + return new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + } + + private void initParityData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT SELECT ON " + TEST_DATABASE + ".* TO '" + TEST_USERNAME + "'@'%';" + + " DROP TABLE IF EXISTS " + TEST_DATABASE + "." + PARITY_TABLE + ";" + + " CREATE TABLE " + TEST_DATABASE + "." + PARITY_TABLE + " (" + + " metric_name VARCHAR(32) PRIMARY KEY," + + " metric_value VARCHAR(32) NOT NULL," + + " metric_note VARCHAR(32) NULL" + + " );" + + " INSERT INTO " + TEST_DATABASE + "." + PARITY_TABLE + + " (metric_name, metric_value, metric_note) VALUES" + + " ('alpha', '1', NULL)," + + " ('beta', '2', 'steady');" + + " FLUSH PRIVILEGES;"); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private String buildJdbcUrl(GenericContainer container) { + return "jdbc:mysql://%s:%d/%s?allowPublicKeyRetrieval=true&useSSL=false" + .formatted(container.getHost(), container.getMappedPort(3306), TEST_DATABASE); + } + + private List> normalizeRows(CollectRep.MetricsData metricsData) { + return metricsData.getValues().stream() + .map(valueRow -> new ArrayList<>(valueRow.getColumnsList())) + .sorted(Comparator.comparing(row -> String.join("\u0001", row))) + .collect(Collectors.toList()); + } + + private Metrics.Field field(String name) { + return Metrics.Field.builder().field(name).type((byte) 1).build(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } + + private record DatabaseTarget(String name, DockerImageName image) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java new file mode 100644 index 00000000000..6162d5e193d --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.junit.jupiter.api.Test; + +class MysqlR2dbcJdbcQueryExecutorTest { + + @Test + void shouldAutoRouteToR2dbcOnlyWhenMysqlJdbcDriverIsAbsent() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(false); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertTrue(executor.supports(metrics("mysql", "columns"))); + assertTrue(executor.supports(metrics("mariadb", "columns"))); + assertTrue(executor.supports(metrics("mysql", "multiRow"))); + assertFalse(executor.supports(metrics("mysql", "runScript"))); + assertFalse(executor.supports(metrics("postgresql", "columns"))); + } + + @Test + void shouldPreferJdbcWhenMysqlJdbcDriverIsPresentInAutoMode() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldHonorExplicitQueryEngineOverrides() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + assertTrue(executor.supports(metrics("mysql", "columns"))); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.JDBC); + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldExposeQueryResultsAsJdbcStyleRowSet() throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder() + .columns(List.of("Variable_name", "Value")) + .rows(List.of( + List.of("Threads_connected", "5"), + List.of("Uptime", "10"))) + .elapsedMs(11) + .rowCount(2) + .build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + try (JdbcQueryRowSet rowSet = executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)) { + assertTrue(rowSet.next()); + assertEquals("Threads_connected", rowSet.getString(1)); + assertEquals("5", rowSet.getString(2)); + assertEquals("5", rowSet.getString("value")); + + assertTrue(rowSet.next()); + assertEquals("Uptime", rowSet.getString("variable_name")); + assertEquals("10", rowSet.getString(2)); + assertFalse(rowSet.next()); + } + } + + @Test + void shouldFailFastWhenR2dbcQueryReturnsError() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder().error("query timeout").build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)); + assertTrue(exception.getMessage().contains("query timeout")); + } + + private Metrics metrics(String platform, String queryType) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host("127.0.0.1") + .port("3306") + .platform(platform) + .database("hzb") + .username("test") + .password("test123") + .queryType(queryType) + .sql("SHOW GLOBAL STATUS") + .timeout("6000") + .build(); + Metrics metrics = new Metrics(); + metrics.setProtocol("jdbc"); + metrics.setName("status"); + metrics.setJdbc(jdbcProtocol); + return metrics; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..e95cfa62052 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class OceanbaseJdbcQueryAdapterIntegrationTest { + + private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce:latest"; + private static final String OCEANBASE_USERNAME = "root@sys"; + private static final String OCEANBASE_PASSWORD = ""; + private static final String OCEANBASE_DATABASE = "oceanbase"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List oceanbaseTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse(OCEANBASE_IMAGE)) + .withExposedPorts(2881) + .withEnv("MODE", "MINI") + .withEnv("OB_MEMORY_LIMIT", "4096M") + .withEnv("OB_SYSTEM_MEMORY", "1024M") + .withEnv("OB_DATAFILE_SIZE", "2048M") + .withEnv("OB_LOG_DISK_SIZE", "2048M") + .withCommand("bash", "-lc", "/usr/sbin/sshd || true; /root/boot/start.sh || true; tail -f /dev/null") + .waitingFor(Wait.forListeningPort()) + .withStartupTimeout(Duration.ofMinutes(5)); + container.start(); + awaitSysLoginReady(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + oceanbaseTemplateMetrics = loadOceanbaseTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialOceanbaseTemplateThroughJdbcQueryAdapter() { + return oceanbaseTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, "basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("tenant".equals(metric.getName()) || "sql".equals(metric.getName()) || "process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("oceanbase") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(2881))); + jdbcProtocol.setUsername(OCEANBASE_USERNAME); + jdbcProtocol.setPassword(OCEANBASE_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(12).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + jdbcProtocol.setDatabase(OCEANBASE_DATABASE); + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadOceanbaseTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-oceanbase.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void awaitSysLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(3).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", obclientCommand("select 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for OceanBase observer bootstrap to finish and accept sys tenant logins. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for OceanBase sys tenant login to become ready"); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = "nohup sh -lc '" + obclientCommand("select sleep(15);").replace("'", "'\"'\"'") + " >/tmp/oceanbase-process-state.log 2>&1' >/dev/null 2>&1 &"; + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private String obclientCommand(String sql) { + StringBuilder command = new StringBuilder("obclient -h127.0.0.1 -P2881 -u") + .append(OCEANBASE_USERNAME) + .append(" -D") + .append(OCEANBASE_DATABASE) + .append(" -A "); + if (!OCEANBASE_PASSWORD.isEmpty()) { + command.append("-p").append(OCEANBASE_PASSWORD).append(' '); + } + command.append("-e \"").append(sql.replace("\"", "\\\"")).append('"'); + return command.toString(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..0d57526d95e --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TidbJdbcQueryAdapterIntegrationTest { + + private static final String TIDB_USERNAME = "root"; + private static final String TIDB_PASSWORD = ""; + private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private Metrics tidbBasicMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("pingcap/tidb:v7.5.1")) + .withCommand("--store=unistore", "--path=") + .withExposedPorts(4000, 10080) + .waitingFor(Wait.forLogMessage(".*server is running MySQL protocol.*", 1)); + container.start(); + waitForStatusEndpoint(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + tidbBasicMetric = loadTidbBasicMetric(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @Test + void shouldCollectTidbBasicMetricThroughMysqlCompatibleQueryAdapter() { + Metrics metric = materializeMetric(tidbBasicMetric); + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount()); + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst()); + assertTrue(metricsData.getValues().getFirst().getColumnsList().stream() + .anyMatch(value -> !Objects.equals(CommonConstants.NULL_VALUE, value) && !value.isEmpty()), + "TiDB basic should still return at least one concrete field value through the adapted query path"); + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(4000))); + jdbcProtocol.setUsername(TIDB_USERNAME); + jdbcProtocol.setPassword(TIDB_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setDatabase(null); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return metric; + } + + private Metrics loadTidbBasicMetric() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-tidb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + Job job = yaml.loadAs(reader, Job.class); + return job.getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-tidb.yml")); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("tidb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private void waitForStatusEndpoint() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String statusUrl = "http://" + container.getHost() + ":" + container.getMappedPort(10080) + "/status"; + while (System.currentTimeMillis() < deadline) { + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(statusUrl)) + .GET() + .timeout(Duration.ofSeconds(3)) + .build(); + HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) { + return; + } + } catch (Exception ignored) { + // Wait for the TiDB status endpoint to become available. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for TiDB status endpoint"); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java new file mode 100644 index 00000000000..45310f52abe --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.strategy; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.junit.jupiter.api.Test; + +class CollectStrategyFactoryTest { + + @Test + void shouldRegisterCommonProtocolsFromServiceLoader() throws Exception { + new CollectStrategyFactory().run(); + + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_JDBC)); + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_HTTP)); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml new file mode 100644 index 00000000000..6c631b18978 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-collector + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc + ${project.artifactId} + + + ${java.version} + ${java.version} + UTF-8 + + + + + io.asyncer + r2dbc-mysql + ${r2dbc-mysql.version} + + + org.springframework.boot + spring-boot-autoconfigure + + + org.projectlombok + lombok + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + testcontainers-junit-jupiter + ${testcontainers.version} + test + + + io.netty + netty-tcnative-boringssl-static + 2.0.69.Final + test + + + diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java new file mode 100644 index 00000000000..0c476039e89 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +/** + * Internal collector-side MySQL query executor. + */ +public interface MysqlQueryExecutor { + + /** + * Execute a single read-only SQL statement. + * + * @param sql SQL to execute + * @param options execution options and target connection settings + * @return normalized query result + */ + QueryResult execute(String sql, QueryOptions options); +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java new file mode 100644 index 00000000000..fcce4781bcd --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring beans for the collector-side MySQL R2DBC route. + */ +@Configuration(proxyBeanMethods = false) +public class MysqlR2dbcConfiguration { + + @Bean + @ConditionalOnMissingBean + public SqlGuard mysqlR2dbcSqlGuard() { + return new SqlGuard(); + } + + @Bean + @ConditionalOnMissingBean + public ResultSetMapper mysqlR2dbcResultSetMapper() { + return new ResultSetMapper(); + } + + @Bean + @ConditionalOnMissingBean + public MysqlR2dbcConnectionFactoryProvider mysqlR2dbcConnectionFactoryProvider() { + return new MysqlR2dbcConnectionFactoryProvider(); + } + + @Bean + @ConditionalOnMissingBean(MysqlQueryExecutor.class) + public MysqlQueryExecutor mysqlQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + return new MysqlR2dbcQueryExecutor(connectionFactoryProvider, resultSetMapper, sqlGuard); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java new file mode 100644 index 00000000000..3121538c33c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionConfiguration; +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.asyncer.r2dbc.mysql.constant.TlsVersions; +import org.springframework.util.StringUtils; + +/** + * Builds R2DBC MySQL connection factories for the collector. + */ +public class MysqlR2dbcConnectionFactoryProvider { + + /** + * Create a connection factory for a target MySQL instance. + * + * @param options target connection settings + * @return connection factory + */ + public MySqlConnectionFactory create(QueryOptions options) { + return create(options, SslMode.PREFERRED); + } + + public MySqlConnectionFactory create(QueryOptions options, SslMode sslMode) { + if (!StringUtils.hasText(options.getHost())) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a target host"); + } + MySqlConnectionConfiguration.Builder builder = MySqlConnectionConfiguration.builder() + .host(options.getHost()) + .port(options.getPort()) + .connectTimeout(options.getTimeout()) + .sslMode(sslMode) + .tcpKeepAlive(true) + .tcpNoDelay(true); + if (sslMode.startSsl()) { + // Pin to TLSv1.2 because JDK 25 + the current server matrix is unreliable when TLSv1.3 is negotiated. + builder.tlsVersion(TlsVersions.TLS1_2); + } + if (StringUtils.hasText(options.getUsername())) { + builder.user(options.getUsername()); + } + if (options.getPassword() != null) { + builder.password(options.getPassword()); + } + if (StringUtils.hasText(options.resolvedDatabase())) { + builder.database(options.resolvedDatabase()); + } + return MySqlConnectionFactory.from(builder.build()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java new file mode 100644 index 00000000000..ab74d430204 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.r2dbc.spi.Connection; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.Locale; +import java.util.concurrent.TimeoutException; +import reactor.core.publisher.Mono; + +/** + * Collector-side MySQL query executor backed by R2DBC. + */ +public class MysqlR2dbcQueryExecutor implements MysqlQueryExecutor { + + private final MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider; + private final ResultSetMapper resultSetMapper; + private final SqlGuard sqlGuard; + + public MysqlR2dbcQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + this.connectionFactoryProvider = connectionFactoryProvider; + this.resultSetMapper = resultSetMapper; + this.sqlGuard = sqlGuard; + } + + @Override + public QueryResult execute(String sql, QueryOptions options) { + String normalizedSql = sqlGuard.normalizeAndValidate(sql); + QueryResult firstAttempt = executeOnce(normalizedSql, options, SslMode.PREFERRED); + if (!firstAttempt.hasError() || !shouldRetryWithoutSsl(firstAttempt.getError())) { + return firstAttempt; + } + QueryResult fallbackAttempt = executeOnce(normalizedSql, options, SslMode.DISABLED); + if (!fallbackAttempt.hasError()) { + return fallbackAttempt; + } + if (requiresSslCompatibleAuth(fallbackAttempt.getError())) { + return QueryResult.builder() + .error(fallbackAttempt.getError() + + ". This route currently needs a TLS-compatible runtime or a mysql_native_password monitoring user.") + .build(); + } + return fallbackAttempt; + } + + private QueryResult executeOnce(String sql, QueryOptions options, SslMode sslMode) { + MySqlConnectionFactory connectionFactory = connectionFactoryProvider.create(options, sslMode); + Duration timeout = options.getTimeout().plusSeconds(1); + Connection connection = null; + try { + connection = Mono.from(connectionFactory.create()).block(timeout); + if (connection == null) { + return QueryResult.builder().error("R2DBC MySQL collector route could not create a connection").build(); + } + QueryResult result = map(connection, sql, options).block(timeout); + if (result == null) { + return QueryResult.builder().error("R2DBC MySQL collector route returned no result").build(); + } + return result; + } catch (IllegalArgumentException exception) { + throw exception; + } catch (Exception exception) { + String message = extractErrorMessage(exception, options.getTimeout()); + return QueryResult.builder() + .error(message) + .build(); + } finally { + if (connection != null) { + safelyClose(connection); + } + } + } + + private void safelyClose(Connection connection) { + try { + Mono.from(connection.close()) + .timeout(Duration.ofSeconds(1), Mono.empty()) + .onErrorResume(_ -> Mono.empty()) + .block(Duration.ofSeconds(2)); + } catch (Exception ignored) { + // Best-effort close only. Query completion must not be turned into a failure by cleanup. + } + } + + private String extractErrorMessage(Exception exception, Duration timeout) { + if (isTimeoutException(exception)) { + return "Query timed out after " + timeout.toMillis() + "ms"; + } + String message = exception.getMessage(); + if ((message == null || message.isBlank()) && exception.getCause() != null) { + message = exception.getCause().getMessage(); + } + return message == null || message.isBlank() ? exception.getClass().getSimpleName() : message; + } + + private boolean isTimeoutException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof TimeoutException) { + return true; + } + String message = current.getMessage(); + if (message != null && message.toLowerCase(Locale.ROOT).contains("timeout on blocking read")) { + return true; + } + current = current.getCause(); + } + return false; + } + + private boolean shouldRetryWithoutSsl(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("handshake_failure") + || normalized.contains("ssl/tls handshake") + || normalized.contains("closedchannelexception") + || normalized.contains("connection unexpectedly closed"); + } + + private boolean requiresSslCompatibleAuth(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("caching_sha2_password") + || normalized.contains("must require ssl"); + } + + private Mono map(Connection connection, String sql, QueryOptions options) { + Statement statement = connection.createStatement(sql); + // Keep the query path read-only and deterministic, and let JdbcCommonCollect own the existing parser flow. + return resultSetMapper.map(statement, options.getTimeout(), options.getMaxRows()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java new file mode 100644 index 00000000000..ffad2d25617 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.time.Duration; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.springframework.util.StringUtils; + +/** + * Collector-only execution options for a target MySQL query. + */ +@Getter +@Builder(toBuilder = true) +@ToString(exclude = "password") +public class QueryOptions { + + private final String host; + @Builder.Default + private final int port = 3306; + private final String username; + private final String password; + private final String database; + private final String schema; + @Builder.Default + private final Duration timeout = Duration.ofSeconds(6); + @Builder.Default + private final int maxRows = 1000; + @Builder.Default + private final int fetchSize = 256; + @Builder.Default + private final boolean readOnly = true; + + public String resolvedDatabase() { + if (StringUtils.hasText(database)) { + return database; + } + if (StringUtils.hasText(schema)) { + return schema; + } + return null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java new file mode 100644 index 00000000000..93c5e580a04 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Collections; +import java.util.List; + +import lombok.Builder; +import lombok.Getter; + +/** + * Normalized query response. + */ +@Getter +@Builder(toBuilder = true) +public class QueryResult { + + @Builder.Default + private final List columns = Collections.emptyList(); + @Builder.Default + private final List> rows = Collections.emptyList(); + @Builder.Default + private final long elapsedMs = 0L; + @Builder.Default + private final int rowCount = 0; + private final String error; + + public boolean hasError() { + return error != null && !error.isBlank(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java new file mode 100644 index 00000000000..5ad8971c527 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Maps R2DBC results into the collector-neutral result model. + */ +public class ResultSetMapper { + + /** + * Execute and map a single statement. + * + * @param statement statement to execute + * @param timeout timeout for the query + * @param maxRows max rows to materialize + * @return normalized query result + */ + public Mono map(Statement statement, Duration timeout, int maxRows) { + long startNanos = System.nanoTime(); + ResultAccumulator accumulator = new ResultAccumulator(); + return Flux.from(statement.execute()) + .concatMap(result -> Flux.from(result.map(accumulator::mapRow))) + .take(maxRows) + .collectList() + .map(rows -> accumulator.toQueryResult(rows, elapsedMillis(startNanos))) + .timeout(timeout); + } + + private static long elapsedMillis(long startNanos) { + return Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + } + + private static ColumnLayout extractColumns(RowMetadata metadata) { + List columns = new ArrayList<>(); + metadata.getColumnMetadatas().forEach(columnMetadata -> columns.add(columnMetadata.getName())); + return new ColumnLayout(List.copyOf(columns), columns.size()); + } + + private static List extractRow(Row row, int columnCount) { + List values = new ArrayList<>(columnCount); + for (int index = 0; index < columnCount; index++) { + Object value = row.get(index); + values.add(value == null ? null : String.valueOf(value)); + } + return values; + } + + private record ColumnLayout(List columns, int columnCount) { + } + + private static final class ResultAccumulator { + + private ColumnLayout columnLayout; + + private List mapRow(Row row, RowMetadata metadata) { + if (columnLayout == null) { + columnLayout = extractColumns(metadata); + } + return extractRow(row, columnLayout.columnCount()); + } + + private QueryResult toQueryResult(List> rows, long elapsedMs) { + return QueryResult.builder() + .columns(columnLayout == null ? List.of() : columnLayout.columns()) + .rows(rows) + .rowCount(rows.size()) + .elapsedMs(elapsedMs) + .build(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java new file mode 100644 index 00000000000..47cb257a236 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Minimal SQL guard for the built-in read-only MySQL collector route. + */ +public class SqlGuard { + + private static final Pattern TRAILING_SEMICOLONS = Pattern.compile(";\\s*$"); + private static final Pattern COMMENTS = Pattern.compile("(/\\*|\\*/|--|#)"); + private static final Pattern FORBIDDEN = Pattern.compile( + "\\b(insert|update|delete|replace|merge|alter|drop|truncate|create|call)\\b"); + + /** + * Normalize a single read-only SQL statement and reject obvious unsafe statements. + * + * @param sql sql text + * @return normalized sql text + */ + public String normalizeAndValidate(String sql) { + if (sql == null || sql.isBlank()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + String normalized = TRAILING_SEMICOLONS.matcher(sql.trim()).replaceAll("").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + if (normalized.indexOf(';') >= 0) { + throw new IllegalArgumentException("R2DBC MySQL collector route only allows a single SQL statement"); + } + if (COMMENTS.matcher(normalized).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route does not allow SQL comments"); + } + + String lower = normalized.toLowerCase(Locale.ROOT); + if (!(lower.startsWith("select") || lower.startsWith("show"))) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports SELECT or SHOW statements"); + } + + String stripped = stripQuotedContent(lower); + if (FORBIDDEN.matcher(stripped).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports read-only statements"); + } + return normalized; + } + + private String stripQuotedContent(String sql) { + StringBuilder builder = new StringBuilder(sql.length()); + char quote = 0; + for (int i = 0; i < sql.length(); i++) { + char current = sql.charAt(i); + if (quote == 0 && (current == '\'' || current == '"' || current == '`')) { + quote = current; + builder.append(' '); + continue; + } + if (quote != 0 && current == quote) { + quote = 0; + builder.append(' '); + continue; + } + builder.append(quote == 0 ? current : ' '); + } + return builder.toString(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java new file mode 100644 index 00000000000..532c433c7f3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +class MysqlR2dbcQueryExecutorIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + + private final MysqlQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), new ResultSetMapper(), new SqlGuard()); + + @TestFactory + Stream shouldRunReadOnlyQueriesAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7", DockerImageName.parse("mysql:5.7.44"), false, false), + new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false), + new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true, false)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyReadOnlyQueries(target))); + } + + @Test + void shouldRejectIllegalSqlBeforeConnecting() { + assertThrows(IllegalArgumentException.class, () -> executor.execute("DELETE FROM sample_metrics", QueryOptions.builder() + .host("127.0.0.1") + .port(3306) + .username("test") + .password("test123") + .database("hzb") + .build())); + } + + @Test + void shouldReturnTimeoutErrorOnSlowQuery() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + QueryResult result = executor.execute("SELECT SLEEP(3)", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(1))); + assertTrue(result.hasError()); + } + } + + @Test + void shouldSupportMysql8DefaultCachingSha2Users() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + QueryResult result = executor.execute("SELECT 1 AS value", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5))); + assertFalse(result.hasError(), () -> "mysql-8.0-default-auth failed: " + result.getError()); + assertEquals(List.of("value"), result.getColumns()); + assertEquals(List.of(List.of("1")), result.getRows()); + } + } + + private void verifyReadOnlyQueries(DatabaseTarget target) throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + + QueryOptions options = buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5)); + + QueryResult selectResult = executor.execute("SELECT 1 AS value", options); + assertFalse(selectResult.hasError(), + () -> target.name() + " SELECT 1 failed: " + selectResult.getError()); + assertEquals(List.of("value"), selectResult.getColumns()); + assertEquals(List.of(List.of("1")), selectResult.getRows()); + + QueryResult showResult = executor.execute("SHOW VARIABLES LIKE 'version%'", options); + assertFalse(showResult.hasError(), + () -> target.name() + " SHOW VARIABLES failed: " + showResult.getError()); + assertTrue(showResult.getRowCount() > 0); + + QueryResult businessResult = executor.execute("SELECT label FROM sample_metrics WHERE id = 1", options); + assertFalse(businessResult.hasError(), + () -> target.name() + " business SQL failed: " + businessResult.getError()); + assertEquals(List.of(List.of("alpha")), businessResult.getRows()); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", "root123"); + return container; + } + container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", "root123"); + if (target.mysqlNativePasswordUser()) { + container.withCommand("--default-authentication-plugin=mysql_native_password"); + } + return container; + } + + private QueryOptions buildOptions(GenericContainer container, String database, Duration timeout) { + return QueryOptions.builder() + .host(normalizeLoopbackHost(container.getHost())) + .port(container.getMappedPort(3306)) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .database(database) + .timeout(timeout) + .maxRows(1000) + .fetchSize(128) + .readOnly(true) + .build(); + } + + private void awaitTcpLoginReady(GenericContainer container) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT 1\""); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", command); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // The entrypoint may still be switching from the temporary bootstrap server to the final one. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String normalizeLoopbackHost(String host) { + return host; + } + + private void initSchema(GenericContainer container) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"CREATE TABLE IF NOT EXISTS sample_metrics (id INT PRIMARY KEY, label VARCHAR(32));", + "REPLACE INTO sample_metrics (id, label) VALUES (1, 'alpha');\""); + container.execInContainer("sh", "-lc", command); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb, boolean mysqlNativePasswordUser) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java new file mode 100644 index 00000000000..687cf0613c0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +class MysqlSqlTemplateCompatibilityTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + @SuppressWarnings("unchecked") + void shouldAcceptCurrentOfficialMysqlTemplateSql() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + assertTrue(Files.isRegularFile(template), "MySQL monitor template must exist"); + + Yaml yaml = new Yaml(); + int sqlCount = 0; + try (Reader reader = Files.newBufferedReader(template)) { + Map root = yaml.load(reader); + List> metrics = (List>) root.get("metrics"); + for (Map metric : metrics) { + Map jdbc = (Map) metric.get("jdbc"); + if (jdbc == null) { + continue; + } + Object sql = jdbc.get("sql"); + if (!(sql instanceof String sqlText)) { + continue; + } + String normalized = sqlGuard.normalizeAndValidate(sqlText); + assertFalse(normalized.isBlank(), "Normalized SQL should not be blank"); + sqlCount++; + } + } + assertTrue(sqlCount > 0, "MySQL monitor template should contain SQL statements"); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java new file mode 100644 index 00000000000..2ec085b46a6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.r2dbc.spi.ColumnMetadata; +import io.r2dbc.spi.Result; +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.List; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; + +class ResultSetMapperTest { + + private final ResultSetMapper mapper = new ResultSetMapper(); + + @Test + @SuppressWarnings("unchecked") + void shouldCacheColumnLayoutAcrossRows() { + Statement statement = mock(Statement.class); + Result result = mock(Result.class); + RowMetadata metadata = mock(RowMetadata.class); + Row firstRow = mock(Row.class); + Row secondRow = mock(Row.class); + ColumnMetadata firstColumn = mock(ColumnMetadata.class); + ColumnMetadata secondColumn = mock(ColumnMetadata.class); + + when(firstColumn.getName()).thenReturn("id"); + when(secondColumn.getName()).thenReturn("label"); + doReturn(List.of(firstColumn, secondColumn)).when(metadata).getColumnMetadatas(); + when(firstRow.get(0)).thenReturn(1); + when(firstRow.get(1)).thenReturn("alpha"); + when(secondRow.get(0)).thenReturn(2); + when(secondRow.get(1)).thenReturn("beta"); + when(result.map(any(BiFunction.class))).thenAnswer(invocation -> { + BiFunction> mapping = + (BiFunction>) invocation.getArgument(0); + return Flux.just( + mapping.apply(firstRow, metadata), + mapping.apply(secondRow, metadata)); + }); + doReturn(Flux.just(result)).when(statement).execute(); + + QueryResult queryResult = mapper.map(statement, Duration.ofSeconds(1), 10).block(); + + assertNotNull(queryResult); + assertEquals(List.of("id", "label"), queryResult.getColumns()); + assertEquals(List.of( + List.of("1", "alpha"), + List.of("2", "beta")), queryResult.getRows()); + assertEquals(2, queryResult.getRowCount()); + verify(metadata, times(1)).getColumnMetadatas(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java new file mode 100644 index 00000000000..b46235c47cc --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class SqlGuardTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + void shouldNormalizeTrailingSemicolon() { + assertEquals("SELECT 1", sqlGuard.normalizeAndValidate("SELECT 1;")); + } + + @Test + void shouldAllowShowStatement() { + assertEquals("SHOW VARIABLES LIKE 'version%'", sqlGuard.normalizeAndValidate("SHOW VARIABLES LIKE 'version%'")); + } + + @Test + void shouldRejectWriteStatement() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("DELETE FROM test")); + } + + @Test + void shouldRejectMultipleStatements() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1; SELECT 2")); + } + + @Test + void shouldRejectComments() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1 -- comment")); + } +} diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index bbb03e841d5..f1e1454623c 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -31,12 +31,14 @@ 25 ${java.version} ${java.version} + 2024.0.3 hertzbeat-collector-basic hertzbeat-collector-common hertzbeat-collector-collector + hertzbeat-collector-mysql-r2dbc hertzbeat-collector-mongodb hertzbeat-collector-nebulagraph hertzbeat-collector-rocketmq @@ -45,6 +47,13 @@ + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + org.apache.hertzbeat hertzbeat-collector-common @@ -55,6 +64,11 @@ hertzbeat-collector-basic ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + org.apache.hertzbeat hertzbeat-collector-mongodb diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml new file mode 100644 index 00000000000..49b2f517428 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-e2e + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc-e2e + + + ${java.version} + ${java.version} + UTF-8 + 2024.0.3 + + + + + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + + + + + + + org.apache.hertzbeat + hertzbeat-startup + ${hertzbeat.version} + test + + + com.mysql + mysql-connector-j + + + org.springframework.boot + spring-boot-starter-webflux + + + + + org.apache.hertzbeat + hertzbeat-collector-common-e2e + ${hertzbeat.version} + test + test-jar + + + org.apache.hertzbeat + hertzbeat-collector-basic + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-common + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-collector + ${hertzbeat.version} + test + + + io.projectreactor.netty + reactor-netty-core + test + + + + org.testcontainers + testcontainers-junit-jupiter + test + + + org.testcontainers + testcontainers-mysql + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + false + + + + + + diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..aff3ffb50cc --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; +import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlCollectorProperties; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlJdbcDriverAvailability; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlR2dbcJdbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.job.protocol.Protocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.Assertions; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Shared MySQL-compatible E2E support for the collector-side R2DBC adapter. + */ +abstract class AbstractMysqlR2dbcCollectE2eTest extends AbstractCollectE2eTest { + + protected static final String TEST_DATABASE = "hzb"; + protected static final String TEST_USERNAME = "test"; + protected static final String TEST_PASSWORD = "test123"; + protected static final String ROOT_PASSWORD = "root123"; + + protected GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + + protected void setUpTarget(DatabaseTarget target) throws Exception { + super.setUp(); + collect = new JdbcCommonCollect(); + metrics = new Metrics(); + + container = createContainer(target); + container.start(); + awaitTcpLoginReady(); + initMonitoringData(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + } + + protected void tearDownTarget() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + jdbcQueryExecutor = null; + } + if (container != null) { + container.stop(); + container = null; + } + } + + protected void assertMysqlJdbcDriverAbsent() { + Assertions.assertThrows(ClassNotFoundException.class, () -> Class.forName("com.mysql.cj.jdbc.Driver")); + } + + protected void collectMysqlTemplate(Set metricFilter) throws Exception { + Job mysqlJob = appService.getAppDefine("mysql"); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : mysqlJob.getMetrics()) { + if (metricFilter != null && !metricFilter.contains(metricsDef.getName())) { + continue; + } + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, + configmapFromPreCollectData.isEmpty() ? new HashMap<>() : configmapFromPreCollectData.getFirst()); + String metricName = metricsDef.getName(); + if ("process_state".equals(metricName)) { + startBackgroundSleepQuery(); + } + if ("slow_sql".equals(metricName)) { + generateSlowQuery(); + } + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true); + configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData); + } + } + + @Override + protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef); + metrics.setJdbc(jdbcProtocol); + CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder(); + metricsData.setApp("mysql"); + return collectMetricsData(metrics, metricsDef, metricsData); + } + + @Override + protected Protocol buildProtocol(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = metricsDef.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setDatabase(TEST_DATABASE); + jdbcProtocol.setTimeout("8000"); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return jdbcProtocol; + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer mysql = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return mysql.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return mysql.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private void initMonitoringData() throws Exception { + execRoot("GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(); + } + + private void generateSlowQuery() throws Exception { + execUser(TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(String database, String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + protected record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java new file mode 100644 index 00000000000..7c34f8a6947 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.utility.DockerImageName; + +/** + * Compatibility E2E coverage for the collector-side MySQL R2DBC adapter. + */ +class MysqlR2dbcCollectCompatibilityE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + private static final Set MARIADB_REPRESENTATIVE_METRICS = + Set.of("basic", "process_state", "slow_sql"); + + @TestFactory + Stream shouldCollectMysqlTemplateAcrossCompatibilityMatrixWithoutMysqlJdbcDriver() { + return Stream.of( + new MatrixTarget( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + null), + new MatrixTarget( + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + null), + new MatrixTarget( + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true), + MARIADB_REPRESENTATIVE_METRICS)) + .map(target -> DynamicTest.dynamicTest(target.databaseTarget().name(), () -> verifyTarget(target))); + } + + private void verifyTarget(MatrixTarget target) throws Exception { + setUpTarget(target.databaseTarget()); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(target.metricFilter()); + } finally { + tearDownTarget(); + } + } + + private record MatrixTarget(DatabaseTarget databaseTarget, Set metricFilter) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..9bd4ebafe02 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.testcontainers.utility.DockerImageName; + +/** + * E2E test for collector-side MySQL monitoring through the R2DBC query adapter. + */ +@Slf4j +class MysqlR2dbcCollectE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + @Test + void shouldCollectMysqlTemplateWithoutMysqlJdbcDriver() throws Exception { + DatabaseTarget target = new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false); + setUpTarget(target); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(null); + } finally { + tearDownTarget(); + } + } +} diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index 445a8a50776..6249cf0d447 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -31,6 +31,7 @@ hertzbeat-collector-common-e2e hertzbeat-collector-kafka-e2e hertzbeat-collector-basic-e2e + hertzbeat-collector-mysql-r2dbc-e2e hertzbeat-log-e2e diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index b881e374d4c..90d3a3a663f 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -116,6 +116,18 @@ netty-all + + + io.projectreactor.netty + reactor-netty-core + 1.2.3 + + + io.projectreactor.netty + reactor-netty-http + 1.2.3 + + org.flywaydb @@ -150,6 +162,12 @@ spring-boot-starter-test test + + org.testcontainers + testcontainers + ${testcontainers.version} + test + diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java new file mode 100644 index 00000000000..b7e5c02e7c2 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.fail; + +import io.netty.channel.ChannelOption; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.netty.Connection; +import reactor.netty.tcp.TcpClient; + +/** + * Guards the startup runtime against Reactor Netty / Netty mismatches that only show up when a client loop + * is created for MySQL R2DBC collection. + */ +class ReactorNettyCompatibilityTest { + + @Test + void reactorNettyClientLoopCanBeCreated() { + Connection connection = null; + try { + connection = TcpClient.create() + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 1000) + .host("127.0.0.1") + .port(9) + .connect() + .onErrorResume(throwable -> Mono.empty()) + .block(Duration.ofSeconds(3)); + } catch (NoClassDefFoundError error) { + fail("Startup runtime is missing a Reactor Netty dependency needed by MySQL R2DBC collection", error); + } finally { + if (connection != null) { + connection.disposeNow(); + } + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java new file mode 100644 index 00000000000..b93b6c934dd --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.time.Duration; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Verifies that the startup runtime can execute a real MySQL R2DBC query. + */ +class StartupMysqlR2dbcCompatibilityTest { + + private static final String DATABASE = "hzb"; + private static final String USERNAME = "test"; + private static final String PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + + @AfterEach + void tearDown() { + if (container != null) { + container.stop(); + container = null; + } + } + + @Test + void shouldExecuteMysqlQueryOnStartupRuntimeClasspath() throws Exception { + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", DATABASE) + .withEnv("MYSQL_USER", USERNAME) + .withEnv("MYSQL_PASSWORD", PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(); + + MysqlR2dbcQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()); + QueryOptions options = QueryOptions.builder() + .host(container.getHost()) + .port(container.getMappedPort(3306)) + .database(DATABASE) + .username(USERNAME) + .password(PASSWORD) + .timeout(Duration.ofSeconds(8)) + .build(); + + QueryResult result = executor.execute("SELECT 1 AS ok", options); + + assertFalse(result.hasError(), () -> "Unexpected query error: " + result.getError()); + assertEquals(1, result.getRowCount()); + assertEquals("1", result.getRows().getFirst().getFirst()); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand("SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String mysqlCliCommand(String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + USERNAME, + "-p" + PASSWORD, + DATABASE, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } +} diff --git a/home/docs/download.md b/home/docs/download.md index b6d62fa5f06..e167b88e3c1 100644 --- a/home/docs/download.md +++ b/home/docs/download.md @@ -26,7 +26,7 @@ Download the latest Apache HertzBeat™ release (v1.8.0) as server binary, colle | **Docker Compose** | ~5MB | Full stack deployment | Docker environments | :::tip Native Collector Recommendation -If you do not need MySQL, OceanBase, Oracle, DB2, or other monitoring types that rely on external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. +If you do not need external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase are included in this native-friendly path when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. Trade-offs: native packages are platform-specific and do not support runtime `ext-lib` JDBC loading. See [Native Collector Guide](start/native-collector). ::: diff --git a/home/docs/help/mariadb.md b/home/docs/help/mariadb.md index 6bd75d08c48..1d8a155815c 100644 --- a/home/docs/help/mariadb.md +++ b/home/docs/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MariaDB database. Support MariaDB5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +MariaDB follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +MariaDB monitoring supports both JVM and native deployment now. + +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package +::: ### Configuration parameter diff --git a/home/docs/help/mysql.md b/home/docs/help/mysql.md index 7d62f7c0a46..51bcce08e17 100644 --- a/home/docs/help/mysql.md +++ b/home/docs/help/mysql.md @@ -7,18 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MySQL database. Support MYSQL5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.4.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0) -- It is recommended that you use the latest available mysql-connector-java version as there are regular security fixes to JDBC drivers. -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +HertzBeat now supports two MySQL query paths: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. +- The automatic decision only checks `ext-lib`. If you want to force one path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. :::important Collector package selection -MySQL monitoring requires external JDBC driver loading from `ext-lib`. +MySQL monitoring supports both JVM and native deployment now. -- Use HertzBeat server built-in collector or the JVM collector package for MySQL monitoring -- Do not use the native collector package for MySQL monitoring +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package ::: ### Configuration parameter diff --git a/home/docs/help/oceanbase.md b/home/docs/help/oceanbase.md index d4fe52aeb18..3bfcd3cf292 100644 --- a/home/docs/help/oceanbase.md +++ b/home/docs/help/oceanbase.md @@ -7,17 +7,20 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of OceanBase database. Support OceanBase 4.0+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +OceanBase now follows the same automatic routing as the MySQL-compatible query path: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. :::important Collector package selection -OceanBase monitoring depends on the external MySQL JDBC driver in `ext-lib`. +OceanBase monitoring now supports both JVM and native deployment. -- Use HertzBeat server built-in collector or the JVM collector package for OceanBase monitoring -- Do not use the native collector package for OceanBase monitoring +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL-compatible query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package ::: ### Configuration parameter diff --git a/home/docs/help/tidb.md b/home/docs/help/tidb.md index 73d8195aab4..78b741ea2c1 100644 --- a/home/docs/help/tidb.md +++ b/home/docs/help/tidb.md @@ -15,6 +15,22 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo **Protocol Use: HTTP and JDBC** +### Driver selection + +TiDB monitoring keeps the HTTP part unchanged, and the SQL query part now follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC for the SQL query metric set. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine for the SQL query metric set. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +The TiDB template mixes HTTP metrics and MySQL-compatible SQL queries. + +- HTTP metric sets are unaffected by JDBC driver selection +- The built-in SQL query engine can collect the default TiDB `basic` metric set without `mysql-connector-j` +- If you explicitly place `mysql-connector-j` in `ext-lib`, the JVM collector or built-in server collector will still prefer JDBC for the SQL query path +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/start/docker-deploy.md b/home/docs/start/docker-deploy.md index 69c2310c023..db5bdd7a0ba 100644 --- a/home/docs/start/docker-deploy.md +++ b/home/docs/start/docker-deploy.md @@ -19,6 +19,7 @@ It is necessary to have Docker environment in your environment. If not installed ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ It is necessary to have Docker environment in your environment. If not installed - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (optional) Mount the configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (optional) Mount the account configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR package [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR packages when you need runtime JDBC extension. `mysql-jdbc` is only needed if you explicitly want the JDBC path for MySQL-compatible monitoring; [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) and [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) are still required for Oracle monitoring. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path used by the built-in collector. Supported values: `auto`, `jdbc`, `r2dbc`. - `--name hertzbeat` : (optional) Naming container name hertzbeat - `--restart=always` : (optional) Configure the container to restart automatically. - `apache/hertzbeat` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat` instead. @@ -71,6 +73,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -81,6 +84,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a - `-e MODE=public` : set the running mode(public or private), public cluster or private - `-e MANAGER_HOST=127.0.0.1` : Important, Set the main hertzbeat server ip host, must use the server host instead of 127.0.0.1. - `-e MANAGER_PORT=1158` : (optional) Set the main hertzbeat server port, default 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path. Supported values: `auto`, `jdbc`, `r2dbc`. - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `--name hertzbeat-collector` : Naming container name hertzbeat-collector - `apache/hertzbeat-collector` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat-collector) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat-collector` instead. diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md index e1affbbc040..36527ce068f 100644 --- a/home/docs/start/native-collector.md +++ b/home/docs/start/native-collector.md @@ -13,6 +13,8 @@ Typical native-friendly workloads include: - HTTP, HTTPS, website availability, and API checks - Port, ping, SSL certificate, and other network probes +- MySQL, MariaDB, and OceanBase when you do not rely on runtime `ext-lib` JDBC loading +- TiDB when you do not rely on runtime `ext-lib` JDBC loading for its SQL query metric set - Redis, Zookeeper, Kafka, and other non-JDBC monitoring types ## Why use it? @@ -35,10 +37,9 @@ The native collector package is not a drop-in replacement for every JVM collecto Use the JVM collector package if your monitoring depends on external JDBC drivers, especially: -- MySQL, which requires `mysql-connector-j` -- OceanBase, which also depends on the MySQL JDBC driver - Oracle, which requires `ojdbc8` and sometimes `orai18n` - DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path ## Package naming @@ -69,8 +70,9 @@ That means: ## Recommended decision -- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. +- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring, for MySQL, MariaDB, and OceanBase without `ext-lib`, or for TiDB when its SQL query metric set can use the built-in MySQL-compatible query engine. - Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. +- For MySQL-compatible monitoring on the JVM collector, `auto` only checks `ext-lib`. If you need to force a path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. ## How are the official multi-platform packages built? diff --git a/home/docs/start/package-deploy.md b/home/docs/start/package-deploy.md index f2e8981f7cd..c9486dbd053 100644 --- a/home/docs/start/package-deploy.md +++ b/home/docs/start/package-deploy.md @@ -65,7 +65,7 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala ::: :::tip Native Collector Recommendation -If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. +If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can also use the native collector package directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. Before choosing it, review the trade-offs in [Native Collector Guide](native-collector). ::: @@ -130,14 +130,13 @@ See [Native Collector Guide](native-collector) for package selection, package na If your monitoring depends on external JDBC drivers, use the JVM collector package instead of the native collector package. This currently includes: -- MySQL, which requires `mysql-connector-j` -- OceanBase, which also relies on the MySQL JDBC driver - Oracle, which requires `ojdbc8` and often `orai18n` - DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path Recommended deployment: -- Use the native collector package for HTTP, website, port, ping, and similar non-JDBC monitoring types +- Use the native collector package for HTTP, website, port, ping, similar non-JDBC monitoring types, and for MySQL, MariaDB, or OceanBase without `ext-lib` - Use the JVM collector package when you need `ext-lib` driver extension ::: diff --git a/home/docs/start/quickstart.md b/home/docs/start/quickstart.md index 8e4830ec023..d9d061cb626 100644 --- a/home/docs/start/quickstart.md +++ b/home/docs/start/quickstart.md @@ -59,7 +59,7 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache 3. Run command `$ ./bin/startup.sh` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters(Optional) - - If you do not need MySQL, OceanBase, Oracle, DB2, or other `ext-lib` JDBC drivers, prefer the native collector package for faster startup and lower memory usage. See [Native Collector Guide](native-collector). + - If you do not need external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can use the built-in query engine directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. See [Native Collector Guide](native-collector). - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your target platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download Page](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` @@ -76,7 +76,7 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ``` - Native collector trade-offs: platform-specific packages, no runtime `ext-lib` JDBC loading, and less suitable for JVM-style runtime classpath extension. See [Native Collector Guide](native-collector). - - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC for MySQL, MariaDB, and OceanBase after restart. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics stay unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. - Run command `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access the HertzBeat server dashboard at `http://localhost:1157` and confirm the new collector is registered. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md index 1b467661198..0c226c424de 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md @@ -26,7 +26,7 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 | **Docker Compose** | ~5MB | 全栈部署 | Docker 环境 | :::tip Native 采集器推荐 -如果你不需要 MySQL、OceanBase、Oracle、DB2，或其他依赖 `ext-lib` 外部 JDBC 驱动的监控类型，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。 +如果你不需要 `ext-lib` 外部 JDBC 驱动，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时也属于这条 Native 友好路径；TiDB 的 SQL 查询指标也遵循同样规则。 它的代价是安装包按平台区分，且不支持运行时 `ext-lib` JDBC 加载。详见 [Native 采集器指南](start/native-collector)。 ::: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md index dd8b3f7d868..6b2223a6396 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, MariaDB数据库监控] > 对MariaDB数据库的通用性能指标进行采集监控。支持MariaDB5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +MariaDB 现在和 MySQL 一样支持自动分流： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +MariaDB 监控现在既支持 JVM 部署，也支持 Native 部署。 + +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 +::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md index bc7e723a558..17bf69eff4d 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md @@ -7,17 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, Mysql数据库监控] > 对MYSQL数据库的通用性能指标进行采集监控。支持MYSQL5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +HertzBeat 现在支持两条 MySQL 查询链路： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置 MySQL 查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 +- 自动分流只检查 `ext-lib`。如果你想显式指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 :::important 采集器包选择 -MySQL 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 +MySQL 监控现在既支持 JVM 部署，也支持 Native 部署。 -- MySQL 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 -- 不要使用 Native 采集器安装包执行 MySQL 监控 +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置 MySQL 查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 ::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md index 5893d9c1957..415b6445a7a 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md @@ -7,17 +7,20 @@ keywords: [开源监控系统, 开源数据库监控, OceanBase 数据库监控] > 对 OceanBase 数据库的通用性能指标进行采集监控。支持 OceanBase 4.0+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +OceanBase 现在和 MySQL 兼容查询路径一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 :::important 采集器包选择 -OceanBase 监控同样依赖 `ext-lib` 目录下的 MySQL JDBC 驱动。 +OceanBase 监控现在同样支持 JVM 和 Native 两种部署方式。 -- OceanBase 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 -- 不要使用 Native 采集器安装包执行 OceanBase 监控 +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置的 MySQL 兼容查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，仍然请选择 JVM 采集器安装包 ::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md index 69bdd6fd40b..76280d8ebc9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md @@ -7,6 +7,22 @@ keywords: [开源监控系统, 开源数据库监控, TiDB数据库监控] > 使用 HTTP 和 JDBC 协议对 TiDB 的通用性能指标进行采集监控。 +### 驱动选择 + +TiDB 监控里的 HTTP 部分保持不变，SQL 查询部分现在和 MySQL 一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会对 SQL 查询指标自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会对 SQL 查询指标自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +TiDB 默认模板同时包含 HTTP 指标和 MySQL 兼容 SQL 查询。 + +- HTTP 指标集合不受 JDBC 驱动选择影响 +- 内置 SQL 查询引擎已经可以在不放 `mysql-connector-j` 的情况下采集默认 TiDB `basic` 指标集合 +- 如果你明确把 `mysql-connector-j` 放进 `ext-lib`，JVM 采集器或主程序内置采集器仍会对 SQL 查询路径优先走 JDBC +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md index 06fa83e496e..2265696f10c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md @@ -19,6 +19,7 @@ sidebar_label: Docker方式安装 ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ sidebar_label: Docker方式安装 - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (可选) 挂载日志文件到本地主机方便查看 - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (可选) 挂载配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (可选) 挂载账户配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 挂载外部的第三方 JAR 包 [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 在你需要运行时 JDBC 扩展时挂载外部第三方 JAR 包。`mysql-jdbc` 只在你明确希望 MySQL 兼容监控继续走 JDBC 时才需要；Oracle 监控仍然需要 [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) 和 [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar)。 + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖主程序内置采集器的 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `--name hertzbeat` : (可选) 命名容器名称为 hertzbeat - `--restart=always` : (可选) 配置容器自动重启。 - `apache/hertzbeat` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat`代替。 @@ -69,6 +71,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -79,6 +82,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 - `-e MODE=public` : 配置运行模式(public or private), 公共集群模式或私有云边模式。 - `-e MANAGER_HOST=127.0.0.1` : 重要, 配置连接的 HertzBeat Server 地址，127.0.0.1 需替换为 HertzBeat Server 对外 IP 地址。 - `-e MANAGER_PORT=1158` : (可选) 配置连接的 HertzBeat Server 端口，默认 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (可选)挂载日志文件到本地主机方便查看 - `--name hertzbeat-collector` : 命名容器名称为 hertzbeat-collector - `apache/hertzbeat-collector` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat-collector)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat-collector`代替。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md index 6ef2037be31..9399673a846 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -13,6 +13,8 @@ description: 说明 HertzBeat Native 采集器安装包适合什么场景、优 - HTTP、HTTPS、网站可用性、API 检查 - 端口可用性、Ping、SSL 证书等网络探测 +- 不依赖运行时 `ext-lib` JDBC 加载的 MySQL、MariaDB、OceanBase +- SQL 查询指标不依赖运行时 `ext-lib` JDBC 加载的 TiDB - Redis、Zookeeper、Kafka 等非 JDBC 监控类型 ## 为什么选择它？ @@ -35,10 +37,9 @@ Native 采集器并不是所有 JVM 采集器场景的无损替代。 如果你的监控依赖外部 JDBC 驱动，请继续使用 JVM 采集器安装包，尤其包括： -- MySQL，需要 `mysql-connector-j` -- OceanBase，同样依赖 MySQL JDBC 驱动 - Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` - DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 ## 安装包命名规则 @@ -69,8 +70,9 @@ Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/appli ## 推荐选择 -- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 +- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包；MySQL、MariaDB、OceanBase 在不使用 `ext-lib` 时适合直接选择 Native 采集器安装包，TiDB 的 SQL 查询指标在不使用 `ext-lib` 时也可以走内置 MySQL 兼容查询引擎。 - 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 +- 对 MySQL 兼容监控来说，`auto` 只检查 `ext-lib`。如果你想手动指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 ## 官方多平台安装包是怎么构建的？ diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md index 3091205add9..c3d8120c564 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md @@ -64,7 +64,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 ::: :::tip Native 采集器推荐 -如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。 +如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时，也可以直接使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 在选择前，建议先阅读 [Native 采集器指南](native-collector) 了解它的限制和取舍。 ::: @@ -128,14 +128,13 @@ Native 采集器适合不依赖外部 JVM classpath 扩展的监控类型。 因此，凡是依赖外置 JDBC 驱动的监控类型，请使用 JVM 采集器，不要使用 Native 采集器。当前至少包括： -- MySQL，需要 `mysql-connector-j` -- OceanBase，同样依赖 MySQL JDBC 驱动 - Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` - DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 建议部署方式： -- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型优先使用 Native 采集器 +- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型，以及不依赖 `ext-lib` 的 MySQL、MariaDB、OceanBase，优先使用 Native 采集器 - 需要 `ext-lib` 扩展驱动时使用 JVM 采集器 ::: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md index 0031ecebd45..d4d2b7c854c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md @@ -63,7 +63,7 @@ HertzBeat 提供多种安装选项： 3. 部署启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群(可选) - - 如果你不需要 MySQL、OceanBase、Oracle、DB2 这类依赖 `ext-lib` JDBC 驱动的监控，优先选择 Native 采集器安装包，通常启动更快、内存更省。详见 [Native 采集器指南](native-collector)。 + - 如果你不需要 `ext-lib` 外置 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时可以直接使用内置查询引擎；TiDB 的 SQL 查询指标也遵循同样规则。详见 [Native 采集器指南](native-collector)。 - 下载您系统环境对应采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或匹配目标平台的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主HertzBeat服务的对外IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` @@ -80,7 +80,7 @@ HertzBeat 提供多种安装选项： ``` - Native 采集器的代价是安装包按平台区分、不支持运行时 `ext-lib` JDBC 加载，也不适合依赖 JVM 风格运行时 classpath 扩展的场景。详见 [Native 采集器指南](native-collector)。 - - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动 - JVM 采集器安装包使用 `$ ./bin/startup.sh` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 diff --git a/material/licenses/NOTICE b/material/licenses/NOTICE index 66e5d1db5f7..3e8e5d05a62 100644 --- a/material/licenses/NOTICE +++ b/material/licenses/NOTICE @@ -1162,6 +1162,22 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== ======================================================================== Apache ECharts diff --git a/material/licenses/backend/LICENSE b/material/licenses/backend/LICENSE index be7c0505ed8..3440fc8c31d 100644 --- a/material/licenses/backend/LICENSE +++ b/material/licenses/backend/LICENSE @@ -265,6 +265,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-api/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-impl/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-jackson/0.11.2 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.12.3 Apache-2.0 @@ -305,6 +306,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.26.0 Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/LICENSE b/material/licenses/collector/LICENSE index b141b35c26a..ea272bc1567 100644 --- a/material/licenses/collector/LICENSE +++ b/material/licenses/collector/LICENSE @@ -222,6 +222,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6 Apache-2.0 https://mvnrepository.com/artifact/commons-net/commons-net/3.10.0 Apache-2.0 https://mvnrepository.com/artifact/commons-validator/commons-validator/1.7 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.12.3 Apache-2.0 @@ -259,6 +260,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-sctp/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/NOTICE b/material/licenses/collector/NOTICE index c56b217a965..1d686114d90 100644 --- a/material/licenses/collector/NOTICE +++ b/material/licenses/collector/NOTICE @@ -746,3 +746,19 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== diff --git a/pom.xml b/pom.xml index 341a12c9066..a10511c3a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 3.7.1 4.1.117.Final 8.4.0 + 1.4.1 12.10.2.jre11 21.5.0.0 4.6.1 @@ -183,6 +184,7 @@ 2.25.0 2.25.0-alpha 1.3.1-alpha + 2.0.3 @@ -625,6 +627,28 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-java-25 + validate + + enforce + + + + + [25,) + Apache HertzBeat now requires JDK 25 or newer. Please switch JAVA_HOME before building or testing. + + + + + + org.apache.maven.plugins diff --git a/script/application.yml b/script/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/script/application.yml +++ b/script/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/README.md b/script/docker-compose/README.md index 99908b2cc4c..3a7b931bb9d 100644 --- a/script/docker-compose/README.md +++ b/script/docker-compose/README.md @@ -2,9 +2,14 @@ Suggest the [HertzBeat + GreptimeDB + Postgresql Solution](hertzbeat-postgresql-greptimedb) for the best performance and stability. +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If you place `mysql-connector-j` in `ext-lib`, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC driver jars in `ext-lib`. + - Use Postgresql + GreptimeDB as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+GreptimeDB Solution](hertzbeat-postgresql-greptimedb) - Use Postgresql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+VictoriaMetrics Solution](hertzbeat-postgresql-victoria-metrics) - Use Mysql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+Mysql+VictoriaMetrics Solution](hertzbeat-mysql-victoria-metrics) - Use Mysql + IoTDB as Hertzbeat dependent storage -> [HertzBeat+Mysql+IoTDB Solution](hertzbeat-mysql-iotdb) - Use Mysql + Tdengine as Hertzbeat dependent storage -> [HertzBeat+Mysql+Tdengine Solution](hertzbeat-mysql-tdengine) - diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README.md b/script/docker-compose/hertzbeat-mysql-iotdb/README.md index b5b94efcec2..b6c8fc30504 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-iotdb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/ hertzbeat-mysql-iotdb) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md index 9f145fe769b..e310d72ce40 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-iotdb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-iotdb) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index c4c9d80797e..33b0537468e 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml index 9a1af1589e8..472cca7f2c8 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml @@ -68,6 +68,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README.md b/script/docker-compose/hertzbeat-mysql-tdengine/README.md index 27c1febaba5..0b7d988438b 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-tdengine` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md index 329344585d3..ad2d2758dad 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-tdengine` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 8a6dfe1528f..8d795ed6434 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml index 33afa63dadd..83e151bb550 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md index 02a6829ea8f..8d5a2c52410 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md index 6602b34a318..30bf8f11e96 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index d9118c6c560..f92abb9f989 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml index da4188bd442..acebf9889b6 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md index 5a84321adcb..8c2dd54d1e2 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-greptimedb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md index 609ba130f64..2b375c99d2c 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgresql-greptimedb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index 0a1feb682dc..6cc14900303 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml index be1603c409d..2c3782607ba 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml @@ -87,6 +87,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md index ff4edb7525e..d791a30dd13 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md index f809587e05d..d6982c63092 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgre-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index ed4c59bcbb4..3108f5b9944 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -235,6 +235,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml index fa28d86ea79..fda39244719 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml @@ -70,6 +70,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/ext-lib/README b/script/ext-lib/README index 0afb3b3bbec..30800cd2328 100644 --- a/script/ext-lib/README +++ b/script/ext-lib/README @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder for JVM-based server / collector packages, for example: +Please move external libs to this folder only when you need JVM runtime extension, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar @@ -23,5 +23,7 @@ jcc-11.5.9.0.jar Note: - `ext-lib` is loaded by the JVM server package and the JVM collector package. +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, the JVM server package or JVM collector package prefers JDBC after restart. - The native collector package does not support loading external JDBC driver jars from `ext-lib` at runtime. -- If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers, use the JVM collector package. +- If you need Oracle or DB2 monitoring, or you explicitly want the JDBC path for MySQL-compatible monitoring, use the JVM collector package. From 454a304ba1160b987e329fdbd05e9e67cd885d7d Mon Sep 17 00:00:00 2001 From: Logic Date: Sat, 14 Mar 2026 11:47:10 +0800 Subject: [PATCH 14/14] feat: add MySQL R2DBC query engine support and update documentation --- .../nativex/NativeCollectorDefaults.java | 20 +++++++++---------- .../src/main/resources/application.yml | 3 --- .../nativex/NativeCollectorDefaultsTest.java | 12 +++++++++-- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java index 5f97c36ecbf..1926c48cfb8 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java @@ -22,16 +22,18 @@ import org.springframework.core.NativeDetector; /** - * Applies the native-only collector defaults without forking {@code application.yml}. + * Applies collector defaults without forking {@code application.yml}. */ public final class NativeCollectorDefaults { static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude"; - static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + static final String JVM_AUTOCONFIGURE_EXCLUDES = String.join(",", "org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration", "org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration", - "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration", + "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration"); + static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + JVM_AUTOCONFIGURE_EXCLUDES, "org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration", "org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration", @@ -43,16 +45,12 @@ private NativeCollectorDefaults() { } public static void applyTo(SpringApplication application) { - Map defaultProperties = defaultProperties(NativeDetector.inNativeImage()); - if (!defaultProperties.isEmpty()) { - application.setDefaultProperties(defaultProperties); - } + application.setDefaultProperties(defaultProperties(NativeDetector.inNativeImage())); } static Map defaultProperties(boolean nativeImage) { - if (!nativeImage) { - return Map.of(); - } - return Map.of(AUTOCONFIGURE_EXCLUDE_PROPERTY, NATIVE_AUTOCONFIGURE_EXCLUDES); + return Map.of( + AUTOCONFIGURE_EXCLUDE_PROPERTY, + nativeImage ? NATIVE_AUTOCONFIGURE_EXCLUDES : JVM_AUTOCONFIGURE_EXCLUDES); } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 08d98b88717..03702ef5627 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -27,9 +27,6 @@ spring: timeout-per-shutdown-phase: 10s jackson: default-property-inclusion: ALWAYS - # need to disable spring boot mongodb auto config, or default mongodb connection tried and failed... - autoconfigure: - exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration management: endpoints: web: diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java index 2ffa4beeebc..811259ecc16 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java @@ -33,7 +33,15 @@ void shouldProvideNativeAutoconfigureExcludesWhenNativeImage() { } @Test - void shouldNotProvideNativeSpecificPropertiesForJvmCollector() { - assertTrue(NativeCollectorDefaults.defaultProperties(false).isEmpty()); + void shouldProvideJvmAutoconfigureExcludesForJvmCollector() { + Map properties = NativeCollectorDefaults.defaultProperties(false); + assertEquals(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void nativeAutoconfigureExcludesShouldExtendJvmExcludes() { + assertTrue(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES + .startsWith(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES)); } }