From d1d0d102a32216fb29d162f2183b81cb2185f076 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:21:05 +0800 Subject: [PATCH 01/12] feat: migrate blocking executors to virtual threads --- hertzbeat-alerter/pom.xml | 5 + .../hertzbeat/alert/AlerterWorkerPool.java | 120 ++++- .../periodic/PeriodicAlertRuleScheduler.java | 163 ++++++- .../realtime/window/AlarmEvaluator.java | 62 ++- .../alert/notice/AlertNoticeDispatch.java | 32 +- .../alert/reduce/AlarmCommonReduce.java | 67 +-- .../alert/reduce/AlarmGroupReduce.java | 157 ++++++- .../alert/reduce/AlarmInhibitReduce.java | 166 ++++++- .../alert/AlerterWorkerPoolTest.java | 176 +++++-- .../PeriodicAlertRuleSchedulerTest.java | 202 +++++++++ .../realtime/window/AlarmEvaluatorTest.java | 105 +++++ .../alert/notice/AlertNoticeDispatchTest.java | 31 ++ .../alert/reduce/AlarmCommonReduceTest.java | 79 +++- .../alert/reduce/AlarmGroupReduceTest.java | 118 ++++- .../alert/reduce/AlarmInhibitReduceTest.java | 124 ++++- .../collector/dispatch/WorkerPoolTest.java | 70 ++- .../dispatch/entrance/CollectServerTest.java | 73 +++ .../collector/dispatch/CommonDispatcher.java | 4 +- .../src/main/resources/application.yml | 8 + .../hertzbeat-collector-common/pom.xml | 4 + .../common/cache/GlobalConnectionCache.java | 116 ++++- .../collect/common/http/CommonHttpClient.java | 131 +++++- .../collector/dispatch/WorkerPool.java | 74 ++- .../dispatch/entrance/CollectServer.java | 125 ++++- .../cache/GlobalConnectionCacheTest.java | 175 +++++++ .../CommonHttpClientVirtualThreadTest.java | 121 +++++ .../hertzbeat-collector-rocketmq/pom.xml | 8 +- .../rocketmq/RocketmqSingleCollectImpl.java | 43 +- .../rocketmq/RocketmqSingleCollectTest.java | 62 +++ .../common/concurrent/AdmissionMode.java | 39 ++ .../common/concurrent/ManagedExecutor.java | 36 ++ .../common/concurrent/ManagedExecutors.java | 428 ++++++++++++++++++ .../concurrent/ManagedExecutorsTest.java | 184 ++++++++ .../hertzbeat/common/config/CommonConfig.java | 2 +- .../config/VirtualThreadProperties.java | 168 +++++++ .../common/support/CommonThreadPool.java | 66 ++- .../common/support/CommonThreadPoolTest.java | 121 +++-- .../hertzbeat/log/notice/LogSseManager.java | 10 +- .../log/notice/LogSseManagerTest.java | 33 +- .../component/sd/ServiceDiscoveryWorker.java | 2 +- .../component/status/CalculateStatus.java | 428 ++++++++++++------ .../manager/scheduler/ManagerWorkerPool.java | 74 ++- .../manager/scheduler/netty/ManageServer.java | 123 ++++- .../component/status/CalculateStatusTest.java | 148 ++++++ .../scheduler/ManagerWorkerPoolTest.java | 105 +++++ .../scheduler/netty/ManageServerTest.java | 149 ++++++ .../remoting/netty/NettyRemotingClient.java | 2 +- .../remoting/netty/NettyRemotingServer.java | 2 +- .../apache/hertzbeat/startup/AsyncConfig.java | 41 ++ .../src/main/resources/application-test.yml | 31 ++ .../src/main/resources/application.yml | 31 ++ .../hertzbeat/startup/AsyncConfigTest.java | 83 ++++ .../warehouse/WarehouseWorkerPool.java | 72 ++- .../warehouse/store/DataStorageDispatch.java | 4 +- .../history/tsdb/doris/DorisDataStorage.java | 2 +- .../duckdb/DuckdbDatabaseDataStorage.java | 224 ++++++--- .../warehouse/WarehouseWorkerPoolTest.java | 67 ++- .../duckdb/DuckdbDatabaseDataStorageTest.java | 148 ++++++ 58 files changed, 4849 insertions(+), 595 deletions(-) create mode 100644 hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java create mode 100644 hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java create mode 100644 hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java create mode 100644 hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java create mode 100644 hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java create mode 100644 hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java create mode 100644 hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java diff --git a/hertzbeat-alerter/pom.xml b/hertzbeat-alerter/pom.xml index 7af87836e9d..c3a3890d2c1 100644 --- a/hertzbeat-alerter/pom.xml +++ b/hertzbeat-alerter/pom.xml @@ -40,6 +40,11 @@ hertzbeat-common-core provided + + org.apache.hertzbeat + hertzbeat-common-spring + provided + org.apache.hertzbeat diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java index 23426ddcf06..b6b743b8a2e 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java @@ -18,12 +18,20 @@ package org.apache.hertzbeat.alert; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,16 +39,25 @@ */ @Component @Slf4j -public class AlerterWorkerPool { +public class AlerterWorkerPool implements DisposableBean { private ThreadPoolExecutor workerExecutor; - private ThreadPoolExecutor notifyExecutor; - private ThreadPoolExecutor logWorkerExecutor; + private ManagedExecutor notifyExecutor; + private ManagedExecutor logWorkerExecutor; + private Map notifyChannelPermits; + private int notifyMaxConcurrentPerChannel; public AlerterWorkerPool() { + this(VirtualThreadProperties.defaults()); + } + + @Autowired + public AlerterWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; initWorkExecutor(); - initNotifyExecutor(); - initLogWorkerExecutor(); + initNotifyExecutor(properties); + initLogWorkerExecutor(properties); } private void initWorkExecutor() { @@ -61,16 +78,32 @@ private void initWorkExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initNotifyExecutor() { + private void initNotifyExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter notifyExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.AlerterProperties alerterProperties = properties.getAlerter(); + VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.getNotify(); + notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.getNotifyMaxConcurrentPerChannel()); + notifyChannelPermits = new ConcurrentHashMap<>(8); + notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-", + notifyProperties.getMode(), notifyProperties.getMaxConcurrentJobs(), handler); + return; + } + notifyMaxConcurrentPerChannel = 0; + notifyChannelPermits = null; + notifyExecutor = ManagedExecutors.wrap("notify-worker", createLegacyNotifyExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyNotifyExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter notifyExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("notify-worker-%d") .build(); - notifyExecutor = new ThreadPoolExecutor(6, + return new ThreadPoolExecutor(6, 6, 10, TimeUnit.SECONDS, @@ -79,16 +112,27 @@ private void initNotifyExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initLogWorkerExecutor() { + private void initLogWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter logWorkerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties logWorkerProperties = properties.getAlerter().getLogWorker(); + logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-", + logWorkerProperties.getMaxConcurrentJobs(), logWorkerProperties.getQueueCapacity(), handler); + return; + } + logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyLogWorkerExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter logWorkerExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("log-worker-%d") .build(); - logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, + return new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1000), threadFactory, new ThreadPoolExecutor.AbortPolicy()); @@ -113,6 +157,41 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { notifyExecutor.execute(runnable); } + /** + * Executes the given runnable task using the notify executor with per-channel concurrency control. + * + * @param channelType notification channel type + * @param runnable the task to be executed + * @throws RejectedExecutionException if the task cannot be accepted for execution + */ + public void executeNotify(byte channelType, Runnable runnable) throws RejectedExecutionException { + if (notifyChannelPermits == null) { + notifyExecutor.execute(runnable); + return; + } + Semaphore semaphore = notifyChannelPermits.computeIfAbsent(channelType, + key -> new Semaphore(notifyMaxConcurrentPerChannel)); + if (!semaphore.tryAcquire()) { + throw new RejectedExecutionException( + "notify-worker rejected task because channel concurrency limit was reached for type " + channelType); + } + boolean submitted = false; + try { + notifyExecutor.execute(() -> { + try { + runnable.run(); + } finally { + semaphore.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + semaphore.release(); + } + } + } + /** * Executes the given runnable task using the logWorkerExecutor. * @@ -122,4 +201,11 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { public void executeLogJob(Runnable runnable) throws RejectedExecutionException { logWorkerExecutor.execute(runnable); } + + @Override + public void destroy() { + workerExecutor.shutdownNow(); + notifyExecutor.close(); + logWorkerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 978fb3d234a..9ad94c4d4dd 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -25,12 +25,19 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; @@ -40,18 +47,33 @@ */ @Slf4j @Component -public class PeriodicAlertRuleScheduler implements CommandLineRunner { +public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean { private final MetricsPeriodicAlertCalculator metricsCalculator; private final LogPeriodicAlertCalculator logCalculator; private final AlertDefineDao alertDefineDao; private final ScheduledExecutorService scheduledExecutor; - private final Map> scheduledFutures; + private final ExecutorService periodicExecutor; + private final Semaphore periodicPermits; + private final boolean virtualThreadsEnabled; + private final Map scheduledTasks; public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, LogPeriodicAlertCalculator logCalculator, AlertDefineDao alertDefineDao) { + this(metricsCalculator, logCalculator, alertDefineDao, VirtualThreadProperties.defaults()); + } + + @Autowired + public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, + LogPeriodicAlertCalculator logCalculator, + AlertDefineDao alertDefineDao, + VirtualThreadProperties virtualThreadProperties) { this.metricsCalculator = metricsCalculator; this.logCalculator = logCalculator; this.alertDefineDao = alertDefineDao; + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Scheduled periodic alert threshold has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled periodic alert threshold has uncaughtException."); @@ -61,17 +83,27 @@ public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculat .setNameFormat("periodic-alert-threshold-worker-%d") .build(); this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); - this.scheduledFutures = new ConcurrentHashMap<>(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.virtualThreadsEnabled = properties.isEnabled(); + int maxConcurrentPeriodicTasks = Math.max(1, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + this.periodicExecutor = virtualThreadsEnabled + ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("periodic-alert-task-", 0) + .uncaughtExceptionHandler(handler) + .factory()) + : null; + this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null; + this.scheduledTasks = new ConcurrentHashMap<>(); } public void cancelSchedule(Long ruleId) { if (ruleId == null) { return; } - ScheduledFuture future = scheduledFutures.get(ruleId); - if (future != null) { - future.cancel(true); - scheduledFutures.remove(ruleId); + ScheduledTaskState state = scheduledTasks.remove(ruleId); + if (state != null) { + state.cancel(); } } @@ -83,14 +115,12 @@ public void updateSchedule(AlertDefine rule) { cancelSchedule(rule.getId()); if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) || rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate(() -> { - if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { - metricsCalculator.calculate(rule); - } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - logCalculator.calculate(rule); - } - }, 0, rule.getPeriod(), java.util.concurrent.TimeUnit.SECONDS); - scheduledFutures.put(rule.getId(), future); + ScheduledTaskState state = new ScheduledTaskState(rule); + ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate( + virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), + 0, rule.getPeriod(), TimeUnit.SECONDS); + state.setScheduledFuture(future); + scheduledTasks.put(rule.getId(), state); } } @@ -106,4 +136,107 @@ public void run(String... args) throws Exception { updateSchedule(rule); } } + + @Override + public void destroy() { + scheduledTasks.values().forEach(ScheduledTaskState::cancel); + scheduledTasks.clear(); + scheduledExecutor.shutdownNow(); + if (periodicExecutor != null) { + periodicExecutor.shutdownNow(); + } + } + + private void executeRule(AlertDefine rule) { + if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { + metricsCalculator.calculate(rule); + } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { + logCalculator.calculate(rule); + } + } + + private final class ScheduledTaskState { + + private final AlertDefine rule; + private ScheduledFuture scheduledFuture; + private Future runningFuture; + private boolean running; + private boolean pending; + private boolean cancelled; + + private ScheduledTaskState(AlertDefine rule) { + this.rule = rule; + } + + private synchronized void setScheduledFuture(ScheduledFuture scheduledFuture) { + this.scheduledFuture = scheduledFuture; + } + + private synchronized void trigger() { + if (cancelled) { + return; + } + if (running) { + pending = true; + return; + } + running = true; + submitLocked(); + } + + private synchronized void cancel() { + cancelled = true; + pending = false; + ScheduledFuture periodicFuture = scheduledFuture; + Future currentFuture = runningFuture; + if (periodicFuture != null) { + periodicFuture.cancel(true); + } + if (currentFuture != null) { + currentFuture.cancel(true); + } + } + + private void submitLocked() { + try { + runningFuture = periodicExecutor.submit(() -> { + boolean permitAcquired = false; + try { + periodicPermits.acquire(); + permitAcquired = true; + if (!Thread.currentThread().isInterrupted()) { + executeRule(rule); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); + } finally { + if (permitAcquired) { + periodicPermits.release(); + } + onComplete(); + } + }); + } catch (RuntimeException e) { + running = false; + throw e; + } + } + + private synchronized void onComplete() { + runningFuture = null; + if (cancelled) { + running = false; + pending = false; + return; + } + if (!pending) { + running = false; + return; + } + pending = false; + submitLocked(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java index 1e245631d4f..1d210ac47c8 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java @@ -20,6 +20,11 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.apache.hertzbeat.alert.util.AlertTemplateUtil; @@ -32,10 +37,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * Alarm Evaluator - Final alarm logic trigger @@ -47,36 +48,48 @@ */ @Component @Slf4j -public class AlarmEvaluator { +public class AlarmEvaluator implements DisposableBean { private static final String WINDOW_START_TIME = "window_start_time"; private static final String WINDOW_END_TIME = "window_end_time"; private static final String MATCHING_LOGS_COUNT = "matching_logs_count"; private final AlarmCommonReduce alarmCommonReduce; - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce) { + this(alarmCommonReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmCommonReduce = alarmCommonReduce; - initAlarmEvaluator(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initAlarmEvaluator(properties); } - public void initAlarmEvaluator() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getWindowEvaluator(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-", + queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2, 10, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } public void sendAndProcessWindowData(WindowAggregator.WindowData windowData) { @@ -314,4 +327,9 @@ private void addLogEntryToMap(LogEntry logEntry, Map context) { } } } -} \ No newline at end of file + + @Override + public void destroy() { + workerExecutor.close(); + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java index 6d9207216b7..b294e817cd8 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterWorkerPool; import org.apache.hertzbeat.alert.config.AlertSseManager; @@ -121,14 +122,27 @@ public void dispatchAlarm(GroupAlert groupAlert) { } private void sendNotify(GroupAlert alert) { - matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> workerPool.executeNotify(() -> rule.getReceiverId() - .forEach(receiverId -> { - try { - sendNoticeMsg(getOneReceiverById(receiverId), - getOneTemplateById(rule.getTemplateId()), alert); - } catch (AlertNoticeException e) { - log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); - } - })))); + matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> { + NoticeTemplate noticeTemplate = getOneTemplateById(rule.getTemplateId()); + rule.getReceiverId().forEach(receiverId -> { + NoticeReceiver receiver = getOneReceiverById(receiverId); + if (receiver == null || receiver.getType() == null) { + log.warn("DispatchTask skip invalid receiver, receiverId: {}, alertId: {}", receiverId, alert.getId()); + return; + } + try { + workerPool.executeNotify(receiver.getType(), () -> { + try { + sendNoticeMsg(receiver, noticeTemplate, alert); + } catch (AlertNoticeException e) { + log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); + } + }); + } catch (RejectedExecutionException e) { + log.warn("DispatchTask rejected notify task, receiverId: {}, type: {}, message: {}", + receiverId, receiver.getType(), e.getMessage()); + } + }); + })); } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 0ef336921ac..4d6b2edb0e5 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -18,16 +18,16 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; /** @@ -35,33 +35,45 @@ */ @Service @Slf4j -public class AlarmCommonReduce { +public class AlarmCommonReduce implements DisposableBean { private final AlarmGroupReduce alarmGroupReduce; - - private ThreadPoolExecutor workerExecutor; + + private final ManagedExecutor workerExecutor; public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) { - initWorkExecutor(); + this(alarmGroupReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmGroupReduce = alarmGroupReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initWorkExecutor(properties); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getReduce(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-", + queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2, 2, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } @@ -103,10 +115,6 @@ Runnable reduceAlarmTask(SingleAlert alert) { * Fingerprint is based on labels excluding timestamp related fields */ private String generateAlertFingerprint(Map labels) { - // Remove timestamp related fields - labels.remove("timestamp"); - labels.remove("start_at"); - labels.remove("active_at"); return labels.entrySet().stream() .filter(e -> !"timestamp".equals(e.getKey()) && !"starts_at".equals(e.getKey()) && !"actives_at".equals(e.getKey()) @@ -116,4 +124,9 @@ private String generateAlertFingerprint(Map labels) { .map(e -> e.getKey() + ":" + e.getValue()) .collect(Collectors.joining(",")); } + + @Override + public void destroy() { + workerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 94c6cb873e2..e43b2e43bcb 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -26,17 +26,22 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -45,7 +50,7 @@ */ @Component @Slf4j -public class AlarmGroupReduce { +public class AlarmGroupReduce implements DisposableBean { /** * Default initial group wait time 30s @@ -88,16 +93,60 @@ public class AlarmGroupReduce { */ private final Map groupCacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService workerExecutor; + + private final ScheduledDispatchTask checkTask; + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) { + this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties) { + this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true); + } + + AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.alarmInhibitReduce = alarmInhibitReduce; this.groupDefines = new ConcurrentHashMap<>(8); this.groupCacheMap = new ConcurrentHashMap<>(8); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.scheduledExecutor = createScheduler(); + this.workerExecutor = createVirtualExecutor(properties); + this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups); List groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue(); refreshGroupDefines(groupConverges); - startCheckAndSendGroups(); + if (autoStart) { + startCheckAndSendGroups(); + } } private void startCheckAndSendGroups() { + scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCheckAndSendGroups() { + checkTask.dispatch(); + } + + void beforeCheckAndSendGroupsRun() { + } + + @Override + public void destroy() { + scheduledExecutor.shutdownNow(); + if (workerExecutor != null) { + workerExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Check alarm groups calculate has uncaughtException."); @@ -106,21 +155,36 @@ private void startCheckAndSendGroups() { .setDaemon(true) .setNameFormat("alarm-group-calculate-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - long now = System.currentTimeMillis(); - groupCacheMap.forEach((groupKey, cache) -> { - if (shouldSendGroup(cache, now)) { - sendGroupAlert(cache); - cache.setLastSendTime(now); - cache.getAlertFingerprints().clear(); - } - }); - } catch (Exception e) { - log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); - } - }, 10000, CHECK_INTERVAL, java.util.concurrent.TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) { + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("alarm-group-calculate-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Check alarm groups calculate worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCheckAndSendGroups() { + beforeCheckAndSendGroupsRun(); + try { + long now = System.currentTimeMillis(); + groupCacheMap.forEach((groupKey, cache) -> { + if (shouldSendGroup(cache, now)) { + sendGroupAlert(cache); + cache.setLastSendTime(now); + cache.getAlertFingerprints().clear(); + } + }); + } catch (Exception e) { + log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); + } } /** @@ -339,4 +403,63 @@ private static class GroupAlertCache { private long lastSendTime; private long lastRepeatTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index 08cdf7b3b78..c8a51392692 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -18,25 +18,28 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.Collections; -import java.util.stream.Collectors; - import lombok.Data; import lombok.AllArgsConstructor; @@ -46,7 +49,7 @@ */ @Component @Slf4j -public class AlarmInhibitReduce { +public class AlarmInhibitReduce implements DisposableBean { /** * Interval for checking and cleaning up expired source alerts @@ -73,22 +76,69 @@ public class AlarmInhibitReduce { /** * Default TTL for source alerts (4 hours) */ - private static long SOURCE_ALERT_TTL = 4 * 60 * 60 * 1000L; + private final long sourceAlertTtl; + + private final ScheduledExecutorService cleanupScheduler; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao , AlerterProperties alerterProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true); + } + + AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties, + boolean autoStart) { this.alarmSilenceReduce = alarmSilenceReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) { - SOURCE_ALERT_TTL = alerterProperties.getInhibit().getTtl(); + this.sourceAlertTtl = alerterProperties.getInhibit().getTtl(); + } else { + this.sourceAlertTtl = 4 * 60 * 60 * 1000L; } inhibitRules = new ConcurrentHashMap<>(8); sourceAlertCache = new ConcurrentHashMap<>(8); + this.cleanupScheduler = createCleanupScheduler(); + this.cleanupExecutor = createCleanupExecutor(properties); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache); List inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue(); refreshInhibitRules(inhibits); - startScheduledCleanupCache(); + if (autoStart) { + startScheduledCleanupCache(); + } } private void startScheduledCleanupCache() { + cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanupCacheRun() { + } + + @Override + public void destroy() { + cleanupScheduler.shutdownNow(); + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createCleanupScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled clean up inhibit cache has uncaughtException."); @@ -97,17 +147,30 @@ private void startScheduledCleanupCache() { .setDaemon(true) .setNameFormat("inhibit-clean-up-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - // Scheduled cleanup of all expired source alerts - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - sourceAlertCache.values().forEach(this::cleanupExpiredEntries); - // Remove empty rule caches - sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); - } catch (Exception e) { - log.error("Error during scheduled cleanup", e); - } - }, CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) { + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("inhibit-clean-up-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Scheduled clean up inhibit cache worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCleanupCache() { + beforeCleanupCacheRun(); + try { + sourceAlertCache.values().forEach(this::cleanupExpiredEntries); + sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + } catch (Exception e) { + log.error("Error during scheduled cleanup", e); + } } /** @@ -264,7 +327,7 @@ private void cacheSourceAlert(SingleAlert alert, AlertInhibit rule) { SourceAlertEntry entry = new SourceAlertEntry( alert, System.currentTimeMillis(), - System.currentTimeMillis() + SOURCE_ALERT_TTL + System.currentTimeMillis() + sourceAlertTtl ); ruleCache.put(alert.getFingerprint(), entry); cleanupExpiredEntries(ruleCache); @@ -315,4 +378,63 @@ private static class SourceAlertEntry { private final long createTime; private final long expiryTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java index 444b85d79dc..434a999eb65 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java @@ -18,9 +18,18 @@ package org.apache.hertzbeat.alert; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.BeforeEach; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -28,60 +37,159 @@ */ class AlerterWorkerPoolTest { - private static final int NUMBER_OF_THREADS = 10; + private static final int NUMBER_OF_TASKS = 10; + private AlerterWorkerPool pool; - private AtomicInteger counter; - private CountDownLatch latch; - @BeforeEach - void setUp() { - pool = new AlerterWorkerPool(); - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); + @AfterEach + void tearDown() { + if (pool != null) { + pool.destroy(); + } } @Test void executeJob() throws InterruptedException { - for (int i = 0; i < NUMBER_OF_THREADS; i++) { + pool = new AlerterWorkerPool(); + AtomicInteger counter = new AtomicInteger(); + CountDownLatch latch = new CountDownLatch(NUMBER_OF_TASKS); + + for (int i = 0; i < NUMBER_OF_TASKS; i++) { pool.executeJob(() -> { counter.incrementAndGet(); latch.countDown(); }); } - latch.await(); - assertEquals(NUMBER_OF_THREADS, counter.get()); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals(NUMBER_OF_TASKS, counter.get()); } @Test - void executeNotify() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeNotify(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); + void executeNotifyRunsOnVirtualThread() throws Exception { + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeNotify((byte) 1, () -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); + notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + notifyProperties.setMaxConcurrentJobs(1); + alerterProperties.setNotify(notifyProperties); + alerterProperties.setNotifyMaxConcurrentPerChannel(8); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 2, () -> { + })); + } finally { + release.countDown(); } - latch.await(); + } + + @Test + void executeNotifyRejectsWhenChannelLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); + notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + notifyProperties.setMaxConcurrentJobs(8); + alerterProperties.setNotify(notifyProperties); + alerterProperties.setNotifyMaxConcurrentPerChannel(1); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertEquals(NUMBER_OF_THREADS, counter.get()); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 1, () -> { + })); + } finally { + release.countDown(); + } } @Test void executeLogJob() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeLogJob(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); - } - latch.await(); + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeLogJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } - assertEquals(NUMBER_OF_THREADS, counter.get()); + @Test + void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); + VirtualThreadProperties.QueueProperties logWorkerProperties = new VirtualThreadProperties.QueueProperties(); + logWorkerProperties.setMaxConcurrentJobs(1); + logWorkerProperties.setQueueCapacity(1); + alerterProperties.setLogWorker(logWorkerProperties); + properties.setAlerter(alerterProperties); + pool = new AlerterWorkerPool(properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + pool.executeLogJob(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + pool.executeLogJob(secondStarted::countDown); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeLogJob(() -> { + })); + } finally { + releaseFirst.countDown(); + } + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java new file mode 100644 index 00000000000..a21f66fc2a8 --- /dev/null +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.calculate.periodic; + +import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link PeriodicAlertRuleScheduler}. + */ +@ExtendWith(MockitoExtension.class) +class PeriodicAlertRuleSchedulerTest { + + @Mock + private MetricsPeriodicAlertCalculator metricsCalculator; + + @Mock + private LogPeriodicAlertCalculator logCalculator; + + @Mock + private AlertDefineDao alertDefineDao; + + private PeriodicAlertRuleScheduler scheduler; + + @BeforeEach + void setUp() { + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + VirtualThreadProperties.defaults()); + } + + @AfterEach + void tearDown() { + if (scheduler != null) { + scheduler.destroy(); + } + } + + @Test + void updateScheduleRunsPeriodicCalculationOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(1L); + scheduler.updateSchedule(rule); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void updateScheduleKeepsSingleInFlightExecutionPerRule() throws InterruptedException { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + int count = invocations.incrementAndGet(); + try { + if (count == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (count == 2) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(2L)); + + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + Thread.sleep(1200L); + assertEquals(1, maxConcurrent.get()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void cancelScheduleInterruptsRunningVirtualTask() throws InterruptedException { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + doAnswer(invocation -> { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(3L); + scheduler.updateSchedule(rule); + + assertTrue(started.await(5, TimeUnit.SECONDS)); + scheduler.cancelSchedule(rule.getId()); + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + } + + @Test + void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException { + scheduler.destroy(); + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + periodicProperties(1)); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + AlertDefine rule = invocation.getArgument(0); + try { + if (rule.getId().equals(4L)) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (rule.getId().equals(5L)) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(4L)); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + scheduler.updateSchedule(metricRule(5L)); + Thread.sleep(200L); + assertEquals(1, maxConcurrent.get()); + assertEquals(1L, secondStarted.getCount()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private AlertDefine metricRule(Long id) { + return AlertDefine.builder() + .id(id) + .name("periodic-rule-" + id) + .type(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) + .period(1) + .enable(true) + .build(); + } + + private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { + VirtualThreadProperties properties = VirtualThreadProperties.defaults(); + properties.getAlerter().setPeriodicMaxConcurrentJobs(maxConcurrentJobs); + return properties; + } +} diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java index e27cb833f32..394b9b77d02 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java @@ -19,9 +19,11 @@ import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.apache.hertzbeat.common.entity.log.LogEntry; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,14 +34,20 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -98,6 +106,13 @@ void setUp() { windowData.addMatchingLog(matchingEvent); } + @AfterEach + void tearDown() { + if (alarmEvaluator != null) { + alarmEvaluator.destroy(); + } + } + @Test void testProcessWindowDataWithIndividualMode() throws InterruptedException { // Given - alert define with individual mode @@ -343,4 +358,94 @@ void testMultipleMatchingLogsInWindow() throws InterruptedException { assertEquals(2, alerts.size()); assertEquals(2, alerts.get(0).getTriggerTimes()); // Each alert should have trigger times = total count } + + @Test + void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + alertDefine.setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + alarmEvaluator.sendAndProcessWindowData(windowData); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); + queueProperties.setMaxConcurrentJobs(1); + properties.getAlerter().setWindowEvaluator(queueProperties); + alarmEvaluator.destroy(); + alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + WindowAggregator.WindowData firstWindow = cloneWindowDataWithBody("first"); + WindowAggregator.WindowData secondWindow = cloneWindowDataWithBody("second"); + firstWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + secondWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + + alarmEvaluator.sendAndProcessWindowData(firstWindow); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmEvaluator.sendAndProcessWindowData(secondWindow); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + + private WindowAggregator.WindowData cloneWindowDataWithBody(String body) { + LogEntry logEntry = LogEntry.builder() + .timeUnixNano(System.currentTimeMillis() * 1_000_000L) + .severityText("ERROR") + .body(body) + .build(); + AlertDefine define = AlertDefine.builder() + .id(alertDefine.getId()) + .name(alertDefine.getName()) + .type(alertDefine.getType()) + .expr(alertDefine.getExpr()) + .times(alertDefine.getTimes()) + .template(alertDefine.getTemplate()) + .labels(alertDefine.getLabels()) + .annotations(alertDefine.getAnnotations()) + .enable(alertDefine.isEnable()) + .build(); + MatchingLogEvent event = MatchingLogEvent.builder() + .logEntry(logEntry) + .alertDefine(define) + .eventTimestamp(System.currentTimeMillis()) + .workerTimestamp(System.currentTimeMillis()) + .build(); + WindowAggregator.WindowData clonedWindowData = new WindowAggregator.WindowData( + new WindowAggregator.WindowKey(define.getId(), System.currentTimeMillis() - 60000, System.currentTimeMillis()), + define); + clonedWindowData.addMatchingLog(event); + return clonedWindowData; + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java index 8102afcdc0b..ff4c92ef4cb 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java @@ -19,7 +19,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyByte; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -149,4 +152,32 @@ void testSendNoticeMsgNoHandler() { assertFalse(alertNoticeDispatch.sendNoticeMsg(receiver, null, alert)); } + + @Test + void testDispatchAlarmUsesTypedNotifyExecution() { + NoticeTemplate template = new NoticeTemplate(); + template.setId(1L); + template.setName("default-template"); + + when(alertStoreHandler.store(alert)).thenReturn(alert); + when(noticeConfigService.getReceiverFilterRule(alert)).thenReturn(Collections.singletonList( + org.apache.hertzbeat.common.entity.alerter.NoticeRule.builder() + .receiverId(Collections.singletonList(1L)) + .templateId(1L) + .build())); + when(noticeConfigService.getReceiverById(1L)).thenReturn(receiver); + when(noticeConfigService.getOneTemplateById(1L)).thenReturn(template); + doNothing().when(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(1); + task.run(); + return null; + }).when(workerPool).executeNotify(anyByte(), any(Runnable.class)); + + alertNoticeDispatch.dispatchAlarm(alert); + + verify(workerPool).executeNotify(eq((byte) 1), any(Runnable.class)); + verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + verify(emitterManager).broadcast(any(String.class)); + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index a98f36dfff8..8dae8999511 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -17,7 +17,20 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -40,14 +53,76 @@ class AlarmCommonReduceTest { @BeforeEach void setUp() { - - testAlert = SingleAlert.builder().build(); + testAlert = SingleAlert.builder().labels(new HashMap<>(Map.of("alertname", "test"))).build(); alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce); } + @AfterEach + void tearDown() { + if (alarmCommonReduce != null) { + alarmCommonReduce.destroy(); + } + } + @Test void testReduceAndSendAlarm() { alarmCommonReduce.reduceAndSendAlarm(testAlert); } + @Test + void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(testAlert); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); + queueProperties.setMaxConcurrentJobs(1); + properties.getAlerter().setReduce(queueProperties); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "first"))).build()); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "second"))).build()); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index a8e8dc05338..a042032cab4 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -38,6 +38,9 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; @@ -47,9 +50,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -73,7 +82,15 @@ void setUp() { MockitoAnnotations.openMocks(this); when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); - alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao); + alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmGroupReduce != null) { + alarmGroupReduce.destroy(); + } } @Test @@ -112,6 +129,42 @@ void whenMatchingGroupRule_shouldGroup() { verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait } + @Test + void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private Map createLabels(String... keyValues) { Map labels = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { @@ -119,4 +172,67 @@ private Map createLabels(String... keyValues) { } return labels; } + + private static final class TestAlarmGroupReduce extends AlarmGroupReduce { + + private final CountDownLatch virtualThreadLatch; + + private final AtomicBoolean virtualThread; + + private final CountDownLatch firstStarted; + + private final CountDownLatch releaseFirst; + + private final CountDownLatch secondStarted; + + private final AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent; + + private final AtomicInteger invocations; + + private TestAlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties properties, CountDownLatch virtualThreadLatch, + AtomicBoolean virtualThread, CountDownLatch firstStarted, + CountDownLatch releaseFirst, CountDownLatch secondStarted, + AtomicInteger maxConcurrent, AtomicInteger invocations) { + super(alarmInhibitReduce, alertGroupConvergeDao, properties, false); + this.virtualThreadLatch = virtualThreadLatch; + this.virtualThread = virtualThread; + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + this.invocations = invocations; + this.concurrent = maxConcurrent == null ? null : new AtomicInteger(); + } + + @Override + void beforeCheckAndSendGroupsRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null || invocations == null) { + return; + } + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1 && firstStarted != null && releaseFirst != null) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2 && secondStarted != null) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java index 18bc17eb578..7b264dffb7d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java @@ -39,6 +39,7 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -49,14 +50,19 @@ import java.util.Map; import java.util.ArrayList; import java.util.List; - +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -89,7 +95,15 @@ void setUp() { inhibitProperties.setTtl(60000); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmInhibitReduce != null) { + alarmInhibitReduce.destroy(); + } } @Test @@ -290,7 +304,9 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { AlerterProperties.InhibitProperties inhibitProperties = new AlerterProperties.InhibitProperties(); inhibitProperties.setTtl(100); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); AlertInhibit rule = AlertInhibit.builder() .id(1L) @@ -324,6 +340,42 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { verify(alarmSilenceReduce).silenceAlarm(targetGroupAlert); } + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmInhibitReduce.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmInhibitReduce.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmInhibitReduce.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private GroupAlert createGroupAlert(String status, Map labels, List alerts) { return GroupAlert.builder() .status(status) @@ -347,4 +399,68 @@ private SingleAlert createSingleAlert(String status, String fingerprint, Map { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - assertDoesNotThrow(() -> workerPool.executeJob(mockTask)); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); } @Test - void testExecuteJobThrowsException() { + void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties collectorProperties = new VirtualThreadProperties.PoolProperties(); + collectorProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + collectorProperties.setMaxConcurrentJobs(1); + properties.setCollector(collectorProperties); + workerPool = new WorkerPool(properties); - workerPool = mock(WorkerPool.class); - doThrow(new RejectedExecutionException()).when(workerPool).executeJob(mockTask); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + workerPool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(mockTask)); + try { + assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(() -> { + })); + } finally { + release.countDown(); + } } - - @Test - void testDestroy() { - assertDoesNotThrow(() -> workerPool.destroy()); - } - } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java index 3112e1e5baf..c56d53e948f 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java @@ -18,17 +18,25 @@ package org.apache.hertzbeat.collector.dispatch.entrance; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.netty.channel.Channel; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties; import org.apache.hertzbeat.collector.dispatch.DispatchProperties; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; @@ -142,4 +150,69 @@ void testOnChannelActive() { assertNotNull(scheduledExecutor); } + @Test + void testDispatchHeartbeatRunsOnVirtualThread() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testDispatchHeartbeatDoesNotRunConcurrently() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + collectServer.dispatchHeartbeat("collector1"); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private DispatchProperties properties() { + return properties; + } + } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java index c03709bad7f..f915f2ee4eb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java @@ -114,7 +114,7 @@ public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, public void start() { try { // Pull the collection task from the task queue and put it into the thread pool for execution - workerPool.executeJob(() -> { + workerPool.executeLongRunning(() -> { Thread.currentThread().setName("metrics-task-dispatcher"); while (!Thread.currentThread().isInterrupted()) { MetricsCollect metricsCollect = null; @@ -379,4 +379,4 @@ protected static class MetricsTime { private Metrics metrics; private Timeout timeout; } -} \ No newline at end of file +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index eb1b92e0de8..43057c241bf 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -75,3 +75,11 @@ push: common: queue: type: netty + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index d4cffe23fb7..b9a80815d3a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -51,6 +51,10 @@ org.apache.hertzbeat hertzbeat-common-core + + org.apache.hertzbeat + hertzbeat-common-spring + org.apache.sshd diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java index ed936372334..9b6699622b2 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java @@ -20,14 +20,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import lombok.extern.slf4j.Slf4j; - import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; /** * Singleton LRU global resource cache for client-server connections @@ -56,10 +57,20 @@ public class GlobalConnectionCache { */ private final ConcurrentLinkedHashMap> cacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; + /** * Private constructor to prevent instantiation */ private GlobalConnectionCache() { + this(true); + } + + GlobalConnectionCache(boolean autoStart) { cacheMap = new ConcurrentLinkedHashMap.Builder>() .maximumWeightedCapacity(Integer.MAX_VALUE) .listener((key, value) -> { @@ -72,7 +83,13 @@ private GlobalConnectionCache() { log.info("GlobalConnectionCache discarded key: {}, value: {}.", key, value); }) .build(); - initCacheMonitor(); + this.scheduledExecutor = createCacheMonitorScheduler(); + this.cleanupExecutor = createCleanupExecutor(); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanTimeoutOrUnHealthyCache); + if (autoStart) { + initCacheMonitor(); + Runtime.getRuntime().addShutdownHook(new Thread(this::destroy)); + } } /** @@ -95,18 +112,42 @@ public static GlobalConnectionCache getInstance() { * Initialize the cache monitor for cleaning up expired connections */ private void initCacheMonitor() { + scheduledExecutor.scheduleWithFixedDelay(this::dispatchCleanupCache, 2, 100, TimeUnit.SECONDS); + } + + /** + * Clean and remove timeout or unhealthy cache entries + */ + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanTimeoutOrUnHealthyCacheRun() { + } + + void destroy() { + scheduledExecutor.shutdownNow(); + cleanupExecutor.shutdownNow(); + } + + private ScheduledExecutorService createCacheMonitorScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("connection-cache-timeout-detector-%d") .setDaemon(true) .build(); - ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(this::cleanTimeoutOrUnHealthyCache, 2, 100, TimeUnit.SECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); } - /** - * Clean and remove timeout or unhealthy cache entries - */ - private void cleanTimeoutOrUnHealthyCache() { + private ExecutorService createCleanupExecutor() { + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("connection-cache-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("Connection cache cleanup has uncaughtException.", throwable)) + .factory()); + } + + private void runCleanTimeoutOrUnHealthyCache() { + beforeCleanTimeoutOrUnHealthyCacheRun(); try { cacheMap.forEach((key, value) -> { Long[] cacheTime = timeoutMap.get(key); @@ -204,4 +245,59 @@ public void removeCache(Object key) { log.error("Connection close error for key {}: {}", key, e.getMessage(), e); } } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java index 22cb0e8a1bd..54001cdae0d 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java @@ -22,6 +22,7 @@ import java.security.cert.CertificateExpiredException; import java.security.cert.X509Certificate; import java.util.Date; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -52,6 +53,14 @@ public class CommonHttpClient { private static PoolingHttpClientConnectionManager connectionManager; + private static ScheduledExecutorService scheduledExecutor; + + private static ExecutorService cleanupExecutor; + + private static ScheduledDispatchTask cleanupTask; + + private static volatile Runnable beforeCleanupHook; + /** * all max total connection */ @@ -137,15 +146,9 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr // clean up available but idle connections .evictIdleConnections(100, TimeUnit.SECONDS) .build(); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("http-connection-pool-cleaner-%d") - .setDaemon(true) - .build(); - ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(() -> { - connectionManager.closeExpiredConnections(); - connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); - }, 40L, 40L, TimeUnit.SECONDS); + initializeCleanupExecutors(); + scheduledExecutor.scheduleWithFixedDelay(CommonHttpClient::dispatchConnectionPoolCleanup, + 40L, 40L, TimeUnit.SECONDS); // shutdown hook Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close)); @@ -156,13 +159,119 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr public static CloseableHttpClient getHttpClient() { return httpClient; } + + public static PoolingHttpClientConnectionManager getConnectionManager() { + return connectionManager; + } + + static void dispatchConnectionPoolCleanup() { + if (cleanupTask != null) { + cleanupTask.dispatch(); + } + } + + static void setConnectionManagerForTest(PoolingHttpClientConnectionManager manager) { + connectionManager = manager; + } + + static void setBeforeCleanupHookForTest(Runnable hook) { + beforeCleanupHook = hook; + } public static void close() { try { - httpClient.close(); + if (httpClient != null) { + httpClient.close(); + } } catch (Exception e) { log.error("close http client error", e); } + if (scheduledExecutor != null) { + scheduledExecutor.shutdownNow(); + } + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private static void initializeCleanupExecutors() { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("http-connection-pool-cleaner-%d") + .setDaemon(true) + .build(); + scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); + cleanupExecutor = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("http-connection-pool-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("HTTP connection pool cleanup has uncaughtException.", throwable)) + .factory()); + cleanupTask = new ScheduledDispatchTask(cleanupExecutor, CommonHttpClient::runConnectionPoolCleanup); + } + + private static void runConnectionPoolCleanup() { + Runnable hook = beforeCleanupHook; + if (hook != null) { + hook.run(); + } + if (connectionManager == null) { + return; + } + connectionManager.closeExpiredConnections(); + connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } } - } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java index a11ff4cfe44..7ba179ce40d 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java @@ -18,15 +18,19 @@ package org.apache.hertzbeat.collector.dispatch; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.stereotype.Component; - +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * Collection task worker thread pool @@ -35,25 +39,55 @@ @Slf4j public class WorkerPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public WorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - // thread factory - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + @Autowired + public WorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getCollector(); + return ManagedExecutors.newVirtualExecutor("collector-worker", "collect-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("collector-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("collector-long-running", "collect-long-running-", + (thread, throwable) -> { + log.error("[Important] WorkerPool longRunningExecutor has uncaughtException.", throwable); log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("collect-worker-%d") .build(); int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors()); int maxSize = Runtime.getRuntime().availableProcessors() * 16; - workerExecutor = new ThreadPoolExecutor(coreSize, + return new ThreadPoolExecutor(coreSize, maxSize, 10, TimeUnit.SECONDS, @@ -72,10 +106,20 @@ public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + /** + * Run the long-lived dispatcher job outside of the collector admission limit. + * + * @param runnable dispatcher task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + @Override public void destroy() throws Exception { - if (workerExecutor != null) { - workerExecutor.shutdownNow(); + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); } } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 9b34a11d4bd..112209c8b76 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -32,6 +32,7 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProcessor; import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; @@ -40,11 +41,13 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -70,11 +73,29 @@ public class CollectServer implements CommandLineRunner { private ScheduledExecutorService scheduledExecutor; + private final ExecutorService heartbeatExecutor; + + private final Object heartbeatLock = new Object(); + + private boolean heartbeatRunning; + + private boolean heartbeatPending; + public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, final CommonThreadPool threadPool, final CollectorInfoProperties infoProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, VirtualThreadProperties.defaults()); + } + + @Autowired + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final CommonThreadPool threadPool, + final CollectorInfoProperties infoProperties, + final VirtualThreadProperties virtualThreadProperties) { if (properties == null || properties.getEntrance() == null || properties.getEntrance().getNetty() == null) { log.error("init error, please config dispatch entrance netty props in application.yml"); throw new IllegalArgumentException("please config dispatch entrance netty props"); @@ -87,6 +108,7 @@ public CollectServer(final CollectJobService collectJobService, this.timerDispatch = timerDispatch; this.collectJobService.setCollectServer(this); this.infoProperties = infoProperties; + this.heartbeatExecutor = createHeartbeatExecutor(virtualThreadProperties); this.init(properties, threadPool); } @@ -107,7 +129,12 @@ private void init(final DispatchProperties properties, final CommonThreadPool th } public void shutdown() { - this.scheduledExecutor.shutdownNow(); + if (this.scheduledExecutor != null) { + this.scheduledExecutor.shutdownNow(); + } + if (this.heartbeatExecutor != null) { + this.heartbeatExecutor.shutdownNow(); + } this.remotingClient.shutdown(); } @@ -120,6 +147,21 @@ public void sendMsg(final ClusterMsg.Message message) { this.remotingClient.sendMsg(message); } + void dispatchHeartbeat(String identity) { + if (heartbeatExecutor == null) { + sendHeartbeat(identity); + return; + } + synchronized (heartbeatLock) { + if (heartbeatRunning) { + heartbeatPending = true; + return; + } + heartbeatRunning = true; + } + submitHeartbeat(identity); + } + @Override public void run(String... args) throws Exception { this.remotingClient.start(); @@ -161,19 +203,8 @@ public void onChannelActive(Channel channel) { .build(); scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); // schedule send heartbeat message - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() - .setIdentity(identity) - .setDirection(ClusterMsg.Direction.REQUEST) - .setType(ClusterMsg.MessageType.HEARTBEAT) - .build(); - CollectServer.this.sendMsg(heartbeat); - log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); - } catch (Exception e) { - log.error("schedule send heartbeat to server error.{}", e.getMessage()); - } - }, 5, 5, TimeUnit.SECONDS); + scheduledExecutor.scheduleAtFixedRate(() -> CollectServer.this.dispatchHeartbeat(identity), + 5, 5, TimeUnit.SECONDS); } } @@ -182,4 +213,70 @@ public void onChannelIdle(Channel channel) { log.info("handle idle event triggered. collector is going offline."); } } + + private ExecutorService createHeartbeatExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("heartbeat-worker-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("HeartBeat worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void submitHeartbeat(String identity) { + boolean submitted = false; + try { + heartbeatExecutor.execute(() -> { + try { + sendHeartbeat(identity); + } finally { + onHeartbeatComplete(identity); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (heartbeatLock) { + heartbeatRunning = false; + heartbeatPending = false; + } + } + } + } + + private void onHeartbeatComplete(String identity) { + boolean shouldRunAgain; + synchronized (heartbeatLock) { + if (heartbeatPending) { + heartbeatPending = false; + shouldRunAgain = true; + } else { + heartbeatRunning = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submitHeartbeat(identity); + } + } + + private void sendHeartbeat(String identity) { + try { + ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() + .setIdentity(identity) + .setDirection(ClusterMsg.Direction.REQUEST) + .setType(ClusterMsg.MessageType.HEARTBEAT) + .build(); + CollectServer.this.sendMsg(heartbeat); + log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); + } catch (Exception e) { + log.error("schedule send heartbeat to server error.{}", e.getMessage()); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java new file mode 100644 index 00000000000..d066434d410 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link GlobalConnectionCache}. + */ +class GlobalConnectionCacheTest { + + private TestGlobalConnectionCache globalConnectionCache; + + @BeforeEach + void setUp() { + globalConnectionCache = new TestGlobalConnectionCache(); + } + + @AfterEach + void tearDown() { + if (globalConnectionCache != null) { + globalConnectionCache.destroy(); + } + } + + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + globalConnectionCache.setVirtualThreadHook(latch, virtualThread); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + globalConnectionCache.setConcurrencyHook(firstStarted, releaseFirst, secondStarted, maxConcurrent); + + globalConnectionCache.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + globalConnectionCache.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchCleanupCacheClosesExpiredConnections() throws Exception { + TestConnection connection = new TestConnection(); + globalConnectionCache.addCache("expired", connection, -1L); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(connection.closed.await(5, TimeUnit.SECONDS)); + assertEquals(1, connection.closeCount.get()); + assertTrue(globalConnectionCache.getCache("expired", false).isEmpty()); + } + + private static final class TestGlobalConnectionCache extends GlobalConnectionCache { + + private CountDownLatch virtualThreadLatch; + + private AtomicBoolean virtualThread; + + private CountDownLatch firstStarted; + + private CountDownLatch releaseFirst; + + private CountDownLatch secondStarted; + + private AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent = new AtomicInteger(); + + private final AtomicInteger invocations = new AtomicInteger(); + + private TestGlobalConnectionCache() { + super(false); + } + + private void setVirtualThreadHook(CountDownLatch latch, AtomicBoolean flag) { + this.virtualThreadLatch = latch; + this.virtualThread = flag; + } + + private void setConcurrencyHook(CountDownLatch firstStarted, CountDownLatch releaseFirst, + CountDownLatch secondStarted, AtomicInteger maxConcurrent) { + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + } + + @Override + void beforeCleanTimeoutOrUnHealthyCacheRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null) { + return; + } + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } + + private static final class TestConnection extends AbstractConnection { + + private final CountDownLatch closed = new CountDownLatch(1); + + private final AtomicInteger closeCount = new AtomicInteger(); + + @Override + public Object getConnection() { + return new Object(); + } + + @Override + public void closeConnection() { + closeCount.incrementAndGet(); + closed.countDown(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java new file mode 100644 index 00000000000..2ec28df4528 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for CommonHttpClient cleanup dispatch. + */ +class CommonHttpClientVirtualThreadTest { + + private final PoolingHttpClientConnectionManager originalConnectionManager = CommonHttpClient.getConnectionManager(); + + @AfterEach + void tearDown() { + CommonHttpClient.setBeforeCleanupHookForTest(null); + CommonHttpClient.setConnectionManagerForTest(originalConnectionManager); + } + + @Test + void dispatchConnectionPoolCleanupRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchConnectionPoolCleanupDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchConnectionPoolCleanupClosesExpiredAndIdleConnections() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(manager).closeExpiredConnections(); + CommonHttpClient.setConnectionManagerForTest(manager); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(manager, times(1)).closeExpiredConnections(); + verify(manager, times(1)).closeIdleConnections(40, TimeUnit.SECONDS); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml index ef3d9261cf3..92d05010fe1 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml @@ -28,12 +28,6 @@ hertzbeat-collector-rocketmq ${project.artifactId} - - 17 - 17 - UTF-8 - - org.apache.hertzbeat @@ -47,4 +41,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java index c8c3889dd85..a8077ca6091 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java @@ -19,7 +19,6 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -28,10 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -40,6 +35,8 @@ import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; import org.apache.hertzbeat.collector.util.JsonPathParser; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; @@ -55,7 +52,6 @@ import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.springframework.beans.factory.DisposableBean; @@ -68,10 +64,11 @@ public class RocketmqSingleCollectImpl extends AbstractCollect implements DisposableBean { private static final int WAIT_TIMEOUT = 10; + static final int QUEUE_CAPACITY = 5000; private static final Set SYSTEM_GROUP_SET = new HashSet<>(); - private final ExecutorService executorService; + private final ManagedExecutor executorService; static { // system consumer group @@ -86,24 +83,28 @@ public class RocketmqSingleCollectImpl extends AbstractCollect implements Dispos } public RocketmqSingleCollectImpl() { + this(createExecutor()); + } + + RocketmqSingleCollectImpl(ManagedExecutor executorService) { + this.executorService = executorService; + } + + private static ManagedExecutor createExecutor() { Runtime runtime = Runtime.getRuntime(); int corePoolSize = Math.max(8, runtime.availableProcessors()); int maximumPoolSize = Math.max(16, runtime.availableProcessors()); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("RocketMQCollectGroup has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("rocketMQ-collector-%d") - .build(); - this.executorService = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(5000), threadFactory, new ThreadPoolExecutor.DiscardOldestPolicy()); + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("RocketMQCollectGroup has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + return ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-collector", "rocketmq-collector-", + corePoolSize, maximumPoolSize, QUEUE_CAPACITY, handler); } @Override public void destroy() { - ThreadUtils.shutdownGracefully(this.executorService, 10L, TimeUnit.SECONDS); + this.executorService.close(); } /** @@ -270,7 +271,7 @@ private void collectConsumerData(DefaultMQAdminExt mqAdminExt, RocketmqCollectDa if (SYSTEM_GROUP_SET.contains(consumerGroup)) { continue; } - executorService.submit(() -> { + executeConsumerTask(() -> { RocketmqCollectData.ConsumerInfo consumerInfo = new RocketmqCollectData.ConsumerInfo(); consumerInfoList.add(consumerInfo); consumerInfo.setConsumerGroup(consumerGroup); @@ -369,4 +370,8 @@ private void fillBuilder(RocketmqCollectData rocketmqCollectData, CollectRep.Met builder.addValueRow(valueRowBuilder.build()); } } + + void executeConsumerTask(Runnable runnable) { + executorService.execute(runnable); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java index 9996e727276..b40ed2599d7 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java @@ -19,12 +19,20 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,6 +47,13 @@ public void setUp() throws Exception { collect = new RocketmqSingleCollectImpl(); } + @AfterEach + void tearDown() { + if (collect != null) { + collect.destroy(); + } + } + @Test void preCheck() { // metrics is null @@ -97,4 +112,51 @@ void collect() { void supportProtocol() { assertEquals(DispatchConstants.PROTOCOL_ROCKETMQ, collect.supportProtocol()); } + + @Test + void executeConsumerTaskRunsOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + collect.executeConsumerTask(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeConsumerTaskDiscardsOldestWhenQueueIsFull() throws InterruptedException { + ManagedExecutor executor = ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-test", + "rocketmq-test-", 1, 1, 1, (thread, throwable) -> { + }); + RocketmqSingleCollectImpl testCollect = new RocketmqSingleCollectImpl(executor); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch thirdStarted = new CountDownLatch(1); + AtomicBoolean secondExecuted = new AtomicBoolean(false); + try { + testCollect.executeConsumerTask(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + testCollect.executeConsumerTask(() -> secondExecuted.set(true)); + testCollect.executeConsumerTask(thirdStarted::countDown); + + releaseFirst.countDown(); + assertTrue(thirdStarted.await(5, TimeUnit.SECONDS)); + assertFalse(secondExecuted.get()); + } finally { + releaseFirst.countDown(); + testCollect.destroy(); + } + } } diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java new file mode 100644 index 00000000000..5f1a0e4fec2 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +/** + * Task admission mode for managed executors. + */ +public enum AdmissionMode { + + /** + * Start each task on its own virtual thread without an executor-level concurrency cap. + */ + UNBOUNDED_VT, + + /** + * Reject immediately when the configured concurrency cap has been reached. + */ + LIMIT_AND_REJECT, + + /** + * Block the submitter until a permit is available. + */ + LIMIT_AND_BLOCK +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java new file mode 100644 index 00000000000..c86d7a97d60 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.Executor; + +/** + * Executor with a stable logical name and close semantics. + */ +public interface ManagedExecutor extends Executor, AutoCloseable { + + /** + * Logical executor name for logging and metrics tags. + * + * @return executor name + */ + String name(); + + @Override + void close(); +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java new file mode 100644 index 00000000000..24226deadae --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.ArrayDeque; +import java.util.Objects; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Factory methods for managed executors. + */ +public final class ManagedExecutors { + + private ManagedExecutors() { + } + + /** + * Wrap an existing executor service. + * + * @param name executor name + * @param executorService delegate executor service + * @return managed executor wrapper + */ + public static ManagedExecutor wrap(String name, ExecutorService executorService) { + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a per-task virtual-thread executor with optional admission control. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param mode admission mode + * @param maxConcurrentTasks max concurrent tasks for limited modes + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newVirtualExecutor(String name, String threadNamePrefix, AdmissionMode mode, + int maxConcurrentTasks, Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, semaphore(mode, maxConcurrentTasks), mode); + } + + /** + * Create a platform-thread-per-task executor for a small number of long-running tasks. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newPlatformExecutor(String name, String threadNamePrefix, + Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a queued executor that preserves queue semantics while executing tasks on virtual threads. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity, {@code <= 0} means unbounded + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newQueuedVirtualExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for queued executors"); + } + return new QueuedVirtualManagedExecutor(name, threadNamePrefix, maxConcurrentTasks, queueCapacity, handler); + } + + /** + * Create a virtual-thread executor that preserves {@link java.util.concurrent.ThreadPoolExecutor} + * core/max/queue semantics with discard-oldest overflow handling. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param coreConcurrentTasks core concurrent tasks + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newDiscardOldestVirtualExecutor(String name, String threadNamePrefix, + int coreConcurrentTasks, int maxConcurrentTasks, + int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + if (coreConcurrentTasks <= 0) { + throw new IllegalArgumentException("coreConcurrentTasks must be greater than zero"); + } + if (maxConcurrentTasks < coreConcurrentTasks) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than or equal to coreConcurrentTasks"); + } + if (queueCapacity <= 0) { + throw new IllegalArgumentException("queueCapacity must be greater than zero"); + } + return new DiscardOldestVirtualManagedExecutor(name, threadNamePrefix, coreConcurrentTasks, + maxConcurrentTasks, queueCapacity, handler); + } + + private static Semaphore semaphore(AdmissionMode mode, int maxConcurrentTasks) { + if (mode == AdmissionMode.UNBOUNDED_VT) { + return null; + } + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for limited executors"); + } + return new Semaphore(maxConcurrentTasks); + } + + private static final class DefaultManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final Semaphore permits; + private final AdmissionMode admissionMode; + + private DefaultManagedExecutor(String name, ExecutorService delegate, Semaphore permits, + AdmissionMode admissionMode) { + this.name = Objects.requireNonNull(name, "name"); + this.delegate = Objects.requireNonNull(delegate, "delegate"); + this.permits = permits; + this.admissionMode = Objects.requireNonNull(admissionMode, "admissionMode"); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + acquirePermit(); + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + releasePermit(); + } + }); + submitted = true; + } finally { + if (!submitted) { + releasePermit(); + } + } + } + + @Override + public void close() { + delegate.shutdownNow(); + } + + private void acquirePermit() { + if (permits == null) { + return; + } + switch (admissionMode) { + case LIMIT_AND_REJECT: + if (!permits.tryAcquire()) { + throw new RejectedExecutionException(name + " rejected task because concurrency limit was reached"); + } + break; + case LIMIT_AND_BLOCK: + try { + permits.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException(name + " interrupted while waiting for an execution permit", e); + } + break; + case UNBOUNDED_VT: + break; + default: + throw new IllegalStateException("Unsupported admission mode: " + admissionMode); + } + } + + private void releasePermit() { + if (permits != null) { + permits.release(); + } + } + } + + private static final class QueuedVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final ExecutorService dispatcher; + private final BlockingDeque queue; + private final Semaphore permits; + private final Semaphore permitSignals; + private final AtomicBoolean closed; + + private QueuedVirtualManagedExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.queue = queueCapacity > 0 ? new LinkedBlockingDeque<>(queueCapacity) : new LinkedBlockingDeque<>(); + this.permits = new Semaphore(maxConcurrentTasks); + this.permitSignals = new Semaphore(0); + this.closed = new AtomicBoolean(false); + ThreadFactory dispatcherFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix + "dispatcher-", 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.dispatcher = Executors.newSingleThreadExecutor(dispatcherFactory); + this.dispatcher.execute(this::dispatchLoop); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + if (closed.get()) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (!queue.offerLast(command)) { + throw new RejectedExecutionException(name + " rejected task because queue capacity was reached"); + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + dispatcher.shutdownNow(); + delegate.shutdownNow(); + queue.clear(); + } + + private void dispatchLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + Runnable command = queue.takeFirst(); + if (!permits.tryAcquire()) { + queue.putFirst(command); + permitSignals.acquire(); + continue; + } + submit(command); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + permits.release(); + permitSignals.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + permits.release(); + permitSignals.release(); + } + } + } + } + + private static final class DiscardOldestVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final int coreConcurrentTasks; + private final int maxConcurrentTasks; + private final ArrayDeque queue; + private final int queueCapacity; + private final Object lock; + private boolean closed; + private int runningTasks; + + private DiscardOldestVirtualManagedExecutor(String name, String threadNamePrefix, int coreConcurrentTasks, + int maxConcurrentTasks, int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.coreConcurrentTasks = coreConcurrentTasks; + this.maxConcurrentTasks = maxConcurrentTasks; + this.queueCapacity = queueCapacity; + this.queue = new ArrayDeque<>(queueCapacity); + this.lock = new Object(); + this.closed = false; + this.runningTasks = 0; + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + Runnable taskToStart = null; + synchronized (lock) { + if (closed) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (runningTasks < coreConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else if (queue.size() < queueCapacity) { + queue.offerLast(command); + return; + } else if (runningTasks < maxConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else { + queue.pollFirst(); + queue.offerLast(command); + return; + } + } + submit(taskToStart); + } + + @Override + public void close() { + synchronized (lock) { + if (closed) { + return; + } + closed = true; + queue.clear(); + } + delegate.shutdownNow(); + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + onTaskComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + runningTasks--; + } + throw new RejectedExecutionException(name + " rejected task because delegate submission failed"); + } + } + } + + private void onTaskComplete() { + Runnable nextTask = null; + synchronized (lock) { + if (closed) { + runningTasks--; + return; + } + nextTask = queue.pollFirst(); + if (nextTask == null) { + runningTasks--; + return; + } + } + submit(nextTask); + } + } +} diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java new file mode 100644 index 00000000000..9f104b5e5fe --- /dev/null +++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/concurrent/ManagedExecutorsTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ManagedExecutors}. + */ +class ManagedExecutorsTest { + + @Test + void shouldRunTaskOnVirtualThread() throws Exception { + ManagedExecutor executor = ManagedExecutors.newVirtualExecutor("test", "test-vt-", + AdmissionMode.UNBOUNDED_VT, 0, (thread, throwable) -> { + }); + try { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + executor.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } finally { + executor.close(); + } + } + + @Test + void shouldRejectTaskWhenAdmissionLimitReached() throws Exception { + ManagedExecutor executor = ManagedExecutors.newVirtualExecutor("limited", "limited-vt-", + AdmissionMode.LIMIT_AND_REJECT, 1, (thread, throwable) -> { + }); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + executor.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { + })); + } finally { + release.countDown(); + executor.close(); + } + } + + @Test + void shouldQueueTasksWhileKeepingVirtualThreadExecution() throws Exception { + ManagedExecutor executor = ManagedExecutors.newQueuedVirtualExecutor("queued", "queued-vt-", + 1, 0, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicBoolean firstVirtual = new AtomicBoolean(false); + AtomicBoolean secondVirtual = new AtomicBoolean(false); + try { + executor.execute(() -> { + firstVirtual.set(Thread.currentThread().isVirtual()); + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(() -> { + secondVirtual.set(Thread.currentThread().isVirtual()); + secondStarted.countDown(); + }); + + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertTrue(firstVirtual.get()); + assertTrue(secondVirtual.get()); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } + + @Test + void shouldRejectTaskWhenQueuedExecutorCapacityReached() throws Exception { + ManagedExecutor executor = ManagedExecutors.newQueuedVirtualExecutor("queued", "queued-vt-", + 1, 1, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + try { + executor.execute(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(secondStarted::countDown); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + assertThrows(RejectedExecutionException.class, () -> executor.execute(() -> { + })); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } + + @Test + void shouldDiscardOldestTaskWhenDiscardOldestExecutorQueueIsFull() throws Exception { + ManagedExecutor executor = ManagedExecutors.newDiscardOldestVirtualExecutor("discard-oldest", + "discard-oldest-vt-", 1, 1, 1, (thread, throwable) -> { + }); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch thirdStarted = new CountDownLatch(1); + AtomicBoolean secondExecuted = new AtomicBoolean(false); + AtomicBoolean thirdVirtual = new AtomicBoolean(false); + try { + executor.execute(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + executor.execute(() -> secondExecuted.set(true)); + executor.execute(() -> { + thirdVirtual.set(Thread.currentThread().isVirtual()); + thirdStarted.countDown(); + }); + + releaseFirst.countDown(); + assertTrue(thirdStarted.await(5, TimeUnit.SECONDS)); + assertFalse(secondExecuted.get()); + assertTrue(thirdVirtual.get()); + } finally { + releaseFirst.countDown(); + executor.close(); + } + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java index 7eb3c680731..7adea1c05ac 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/CommonConfig.java @@ -31,6 +31,6 @@ @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG + SignConstants.DOT + ConfigConstants.FunctionModuleConstants.COMMON) -@EnableConfigurationProperties(CommonProperties.class) +@EnableConfigurationProperties({CommonProperties.class, VirtualThreadProperties.class}) public class CommonConfig { } diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java new file mode 100644 index 00000000000..f8590dee3ac --- /dev/null +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.config; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Virtual-thread related configuration. + */ +@Getter +@Setter +@ConfigurationProperties(prefix = "hertzbeat.vthreads") +public class VirtualThreadProperties { + + private boolean enabled = true; + + private PoolProperties collector = PoolProperties.collectorDefaults(); + + private PoolProperties common = PoolProperties.commonDefaults(); + + private PoolProperties manager = PoolProperties.managerDefaults(); + + private AlerterProperties alerter = new AlerterProperties(); + + private PoolProperties warehouse = PoolProperties.warehouseDefaults(); + + private AsyncProperties async = new AsyncProperties(); + + /** + * Create a detached properties instance with runtime defaults. + * + * @return defaults instance + */ + public static VirtualThreadProperties defaults() { + return new VirtualThreadProperties(); + } + + /** + * Pool-level configuration. + */ + @Getter + @Setter + public static class PoolProperties { + + private AdmissionMode mode = AdmissionMode.UNBOUNDED_VT; + + private int maxConcurrentJobs; + + private static PoolProperties collectorDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(defaultCollectorConcurrency()); + return properties; + } + + private static PoolProperties warehouseDefaults() { + return new PoolProperties(); + } + + private static PoolProperties commonDefaults() { + return new PoolProperties(); + } + + private static PoolProperties managerDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(10); + return properties; + } + + private static PoolProperties alerterNotifyDefaults() { + PoolProperties properties = new PoolProperties(); + properties.setMode(AdmissionMode.LIMIT_AND_REJECT); + properties.setMaxConcurrentJobs(64); + return properties; + } + + private static int defaultCollectorConcurrency() { + int historicalMax = Runtime.getRuntime().availableProcessors() * 16; + return Math.max(1, historicalMax - 1); + } + } + + /** + * Alerter-specific executor configuration. + */ + @Getter + @Setter + public static class AlerterProperties { + + private PoolProperties notify = PoolProperties.alerterNotifyDefaults(); + + private int periodicMaxConcurrentJobs = 10; + + private QueueProperties logWorker = QueueProperties.logWorkerDefaults(); + + private QueueProperties reduce = QueueProperties.reduceDefaults(); + + private QueueProperties windowEvaluator = QueueProperties.windowEvaluatorDefaults(); + + private int notifyMaxConcurrentPerChannel = 4; + } + + /** + * Queue-preserving executor configuration. + */ + @Getter + @Setter + public static class QueueProperties { + + private int maxConcurrentJobs; + + private int queueCapacity; + + private static QueueProperties reduceDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(2); + return properties; + } + + private static QueueProperties logWorkerDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(10); + properties.setQueueCapacity(1000); + return properties; + } + + private static QueueProperties windowEvaluatorDefaults() { + QueueProperties properties = new QueueProperties(); + properties.setMaxConcurrentJobs(2); + return properties; + } + } + + /** + * Async executor configuration. + */ + @Getter + @Setter + public static class AsyncProperties { + + private boolean enabled = true; + + private int concurrencyLimit = 256; + + private boolean rejectWhenLimitReached = true; + + private long taskTerminationTimeout = 5000L; + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java index 4a231230f4f..a753dccb2d5 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java @@ -18,13 +18,18 @@ package org.apache.hertzbeat.common.support; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -34,22 +39,53 @@ @Slf4j public class CommonThreadPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public CommonThreadPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("common executor has uncaughtException."); + @Autowired + public CommonThreadPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("common executor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getCommon(); + return ManagedExecutors.newVirtualExecutor("common-worker", "common-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("common-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("common-long-running", "common-long-running-", + (thread, throwable) -> { + log.error("common longRunningExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("common-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(1, + return new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, @@ -67,10 +103,20 @@ public void execute(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + /** + * Run a long-lived task outside of the short-task execution lane. + * + * @param runnable task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + @Override public void destroy() throws Exception { - if (workerExecutor != null) { - workerExecutor.shutdownNow(); + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); } } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java index daa09493a88..063fdf73225 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java @@ -17,92 +17,89 @@ package org.apache.hertzbeat.common.support; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import java.lang.reflect.Field; + +import java.util.concurrent.CountDownLatch; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.BeforeEach; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** - * test for {@link CommonThreadPool} + * Test for {@link CommonThreadPool}. */ - class CommonThreadPoolTest { private CommonThreadPool commonThreadPool; - private ThreadPoolExecutor executorMock; - - @BeforeEach - public void setUp() throws Exception { - - commonThreadPool = new CommonThreadPool(); - - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - executorMock = mock(ThreadPoolExecutor.class); - workerExecutorField.set(commonThreadPool, executorMock); - } - - @Test - public void testExecuteTask() { - - Runnable task = mock(Runnable.class); - commonThreadPool.execute(task); - verify(executorMock).execute(task); + @AfterEach + void tearDown() throws Exception { + if (commonThreadPool != null) { + commonThreadPool.destroy(); + } } @Test - public void testExecuteTaskThrowsEx() { - - Runnable task = mock(Runnable.class); - doThrow(RejectedExecutionException.class).when(executorMock).execute(task); - - assertThrows( - RejectedExecutionException.class, - () -> commonThreadPool.execute(task) - ); - } + void testExecuteRunsOnVirtualThread() throws Exception { + commonThreadPool = new CommonThreadPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); - @Test - public void testDestroy() throws Exception { + commonThreadPool.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - commonThreadPool.destroy(); - verify(executorMock).shutdownNow(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); } @Test - public void testDestroyWithNull() throws Exception { + void testExecuteLongRunningRunsOnPlatformThread() throws Exception { + commonThreadPool = new CommonThreadPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(true); - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - workerExecutorField.set(commonThreadPool, null); + commonThreadPool.executeLongRunning(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - commonThreadPool.destroy(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertFalse(virtualThread.get()); } @Test - public void testInitialization() throws Exception { - CommonThreadPool pool = new CommonThreadPool(); - - Field workerExecutorField = CommonThreadPool.class.getDeclaredField("workerExecutor"); - workerExecutorField.setAccessible(true); - ThreadPoolExecutor workerExecutor = (ThreadPoolExecutor) workerExecutorField.get(pool); - - assertNotNull(workerExecutor); - assertEquals(1, workerExecutor.getCorePoolSize()); - assertEquals(Integer.MAX_VALUE, workerExecutor.getMaximumPoolSize()); - assertEquals(10, workerExecutor.getKeepAliveTime(TimeUnit.SECONDS)); - assertTrue(workerExecutor.getQueue() instanceof SynchronousQueue); + void testExecuteRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties commonProperties = new VirtualThreadProperties.PoolProperties(); + commonProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + commonProperties.setMaxConcurrentJobs(1); + properties.setCommon(commonProperties); + commonThreadPool = new CommonThreadPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + commonThreadPool.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> commonThreadPool.execute(() -> { + })); + } finally { + release.countDown(); + } } - } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java index 31a537c6f2d..d45015a909d 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java @@ -61,11 +61,11 @@ public class LogSseManager { t.setDaemon(true); return t; }); - private final ExecutorService senderPool = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "sse-sender"); - t.setDaemon(true); - return t; - }); + private final ExecutorService senderPool = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("sse-sender-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("SSE sender has uncaughtException.", throwable)) + .factory()); private final AtomicLong queueSize = new AtomicLong(0); public LogSseManager() { diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java index 3f2c39ebeba..9ddc3b78477 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java +++ b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java @@ -26,7 +26,9 @@ import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; import java.io.IOException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -88,6 +91,24 @@ void shouldBroadcastLogWhenFilterMatches() throws IOException { ); } + @Test + void shouldSendBatchOnVirtualThread() throws IOException, InterruptedException { + SseEmitter mockEmitter = mock(SseEmitter.class); + AtomicBoolean virtualThread = new AtomicBoolean(false); + CountDownLatch latch = new CountDownLatch(1); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(mockEmitter).send(any(SseEmitter.SseEventBuilder.class)); + subscribeClient(CLIENT_ID, null, mockEmitter); + + logSseManager.broadcast(createLogEntry("INFO", "virtual-thread-send")); + + assertTrue(latch.await(1, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + @Test void shouldNotBroadcastLogWhenFilterDoesNotMatch() throws IOException, InterruptedException { // Given: A client with a filter for "ERROR" logs @@ -172,6 +193,16 @@ void shouldRemoveEmitterWhenBroadcastFails() throws IOException { }); } + @Test + void shouldDropLogsWhenQueueSizeLimitReached() { + for (int i = 0; i < 10_001; i++) { + logSseManager.broadcast(createLogEntry("INFO", "log-" + i)); + } + + assertEquals(10_000, logSseManager.getQueueSize()); + assertEquals(10_000, logSseManager.getLogQueue().size()); + } + /** * Helper method to create a subscriber and inject a mock emitter for testing */ @@ -190,4 +221,4 @@ private LogEntry createLogEntry(String severityText, String body) { .body(body) .build(); } -} \ No newline at end of file +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java index 3cbf7e22692..2ca8c48d05a 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/sd/ServiceDiscoveryWorker.java @@ -78,7 +78,7 @@ public ServiceDiscoveryWorker(MonitorService monitorService, ParamDao paramDao, @Override public void afterPropertiesSet() { - workerPool.executeJob(new SdUpdateTask()); + workerPool.executeLongRunning(new SdUpdateTask()); } private class SdUpdateTask implements Runnable { diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 94a0d7ca0cf..6639e5674c7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -30,11 +30,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.StatusPageComponent; @@ -45,6 +47,8 @@ import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Component; @@ -53,7 +57,7 @@ */ @Component @Slf4j -public class CalculateStatus { +public class CalculateStatus implements DisposableBean { private static final int DEFAULT_CALCULATE_INTERVAL_TIME = 300; @@ -67,180 +71,318 @@ public class CalculateStatus { private final int intervals; + private final ScheduledExecutorService calculateScheduler; + + private final ScheduledExecutorService combineHistoryScheduler; + + private final ExecutorService calculateExecutor; + + private final ExecutorService combineHistoryExecutor; + + private final ScheduledDispatchTask calculateTask; + + private final ScheduledDispatchTask combineHistoryTask; + public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, MonitorDao monitorDao) { + this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, + VirtualThreadProperties.defaults(), true); + } + + @Autowired + public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, + StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, + MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties) { + this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao, + virtualThreadProperties, true); + } + + CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao, + StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao, + MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.statusPageOrgDao = statusPageOrgDao; this.monitorDao = monitorDao; this.statusPageComponentDao = statusPageComponentDao; this.statusPageHistoryDao = statusPageHistoryDao; intervals = statusProperties.getCalculate() == null ? DEFAULT_CALCULATE_INTERVAL_TIME : statusProperties.getCalculate().getInterval(); - startCalculate(); - startCombineHistory(); + this.calculateScheduler = createScheduler("status-page-calculate-%d", "Status calculate has uncaughtException."); + this.combineHistoryScheduler = createScheduler("status-page-history-%d", "History combine has uncaughtException."); + this.calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-", + "Status calculate worker has uncaughtException."); + this.combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-", + "History combine worker has uncaughtException."); + this.calculateTask = new ScheduledDispatchTask(calculateExecutor, this::runCalculate); + this.combineHistoryTask = new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory); + if (autoStart) { + startCalculate(); + startCombineHistory(); + } } private void startCalculate() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Status calculate has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("status-page-calculate-%d") - .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - scheduledExecutor.scheduleAtFixedRate(() -> { - log.info("start to calculate status page state"); - try { - // calculate component state from tag bind monitors status - List statusPageOrgList = statusPageOrgDao.findAll(); - for (StatusPageOrg statusPageOrg : statusPageOrgList) { - long orgId = statusPageOrg.getId(); - List pageComponentList = statusPageComponentDao.findByOrgId(orgId); - Set stateSet = new HashSet<>(8); - for (StatusPageComponent component : pageComponentList) { - byte state; - if (component.getMethod() == CommonConstants.STATUS_PAGE_CALCULATE_METHOD_MANUAL) { - state = component.getConfigState(); - } else { - Map labels = component.getLabels(); - if (labels == null || labels.isEmpty()) { - continue; - } - Specification specification = (root, query, criteriaBuilder) -> { - List predicates = new ArrayList<>(); - // create every label condition - labels.forEach((key, value) -> { - String pattern = String.format("%%\"%s\":\"%s\"%%", key, value); - predicates.add(criteriaBuilder.like(root.get("labels"), pattern)); - }); - - // use or connect them - return criteriaBuilder.or(predicates.toArray(new Predicate[0])); - }; - List monitorList = monitorDao.findAll(specification); - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN; - for (Monitor monitor : monitorList) { - if (monitor.getStatus() == CommonConstants.MONITOR_DOWN_CODE) { - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL; - break; - } else if (monitor.getStatus() == CommonConstants.MONITOR_UP_CODE) { - state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL; - } + calculateScheduler.scheduleAtFixedRate(this::dispatchCalculate, 5, intervals, TimeUnit.SECONDS); + } + + private void startCombineHistory() { + // combine history every day at 1:00 AM + LocalDateTime now = LocalDateTime.now(); + LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); + if (now.isAfter(nextRun)) { + nextRun = nextRun.plusDays(1); + } + long delay = Duration.between(now, nextRun).toMillis(); + combineHistoryScheduler.scheduleAtFixedRate(this::dispatchCombineHistory, delay, + TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS); + } + + /** + * get calculate status intervals + * @return intervals + */ + public int getCalculateStatusIntervals() { + return intervals; + } + + void dispatchCalculate() { + calculateTask.dispatch(); + } + + void dispatchCombineHistory() { + combineHistoryTask.dispatch(); + } + + @Override + public void destroy() { + calculateScheduler.shutdownNow(); + combineHistoryScheduler.shutdownNow(); + if (calculateExecutor != null) { + calculateExecutor.shutdownNow(); + } + if (combineHistoryExecutor != null) { + combineHistoryExecutor.shutdownNow(); + } + } + + private void runCalculate() { + log.info("start to calculate status page state"); + try { + // calculate component state from tag bind monitors status + List statusPageOrgList = statusPageOrgDao.findAll(); + for (StatusPageOrg statusPageOrg : statusPageOrgList) { + long orgId = statusPageOrg.getId(); + List pageComponentList = statusPageComponentDao.findByOrgId(orgId); + Set stateSet = new HashSet<>(8); + for (StatusPageComponent component : pageComponentList) { + byte state; + if (component.getMethod() == CommonConstants.STATUS_PAGE_CALCULATE_METHOD_MANUAL) { + state = component.getConfigState(); + } else { + Map labels = component.getLabels(); + if (labels == null || labels.isEmpty()) { + continue; + } + Specification specification = (root, query, criteriaBuilder) -> { + List predicates = new ArrayList<>(); + // create every label condition + labels.forEach((key, value) -> { + String pattern = String.format("%%\"%s\":\"%s\"%%", key, value); + predicates.add(criteriaBuilder.like(root.get("labels"), pattern)); + }); + + // use or connect them + return criteriaBuilder.or(predicates.toArray(new Predicate[0])); + }; + List monitorList = monitorDao.findAll(specification); + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN; + for (Monitor monitor : monitorList) { + if (monitor.getStatus() == CommonConstants.MONITOR_DOWN_CODE) { + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL; + break; + } else if (monitor.getStatus() == CommonConstants.MONITOR_UP_CODE) { + state = CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL; } } - stateSet.add(state); - component.setState(state); - statusPageComponentDao.save(component); - // insert component state history - StatusPageHistory statusPageHistory = StatusPageHistory.builder() - .componentId(component.getId()) - .state(state) - .timestamp(System.currentTimeMillis()) - .build(); - statusPageHistoryDao.save(statusPageHistory); } - stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); - if (stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL)) { - if (stateSet.contains(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL)) { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_SOME_ABNORMAL); - } else { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_ABNORMAL); - } + stateSet.add(state); + component.setState(state); + statusPageComponentDao.save(component); + // insert component state history + StatusPageHistory statusPageHistory = StatusPageHistory.builder() + .componentId(component.getId()) + .state(state) + .timestamp(System.currentTimeMillis()) + .build(); + statusPageHistoryDao.save(statusPageHistory); + } + stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + if (stateSet.remove(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL)) { + if (stateSet.contains(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL)) { + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_SOME_ABNORMAL); } else { - statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_NORMAL); + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_ABNORMAL); } - statusPageOrg.setGmtUpdate(LocalDateTime.now()); - statusPageOrgDao.save(statusPageOrg); + } else { + statusPageOrg.setState(CommonConstants.STATUS_PAGE_ORG_STATE_ALL_NORMAL); } - } catch (Exception e) { - log.error("status page calculate component state error: {}", e.getMessage(), e); + statusPageOrg.setGmtUpdate(LocalDateTime.now()); + statusPageOrgDao.save(statusPageOrg); } - }, 5, intervals, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("status page calculate component state error: {}", e.getMessage(), e); + } } - private void startCombineHistory() { + private void runCombineHistory() { + try { + // combine pre day status history to one record + LocalDateTime nowTime = LocalDateTime.now(); + ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now()); + LocalDateTime midnight = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0); + LocalDateTime preNight = midnight.minusDays(1); + long midnightTimestamp = midnight.toInstant(zoneOffset).toEpochMilli(); + long preNightTimestamp = preNight.toInstant(zoneOffset).toEpochMilli(); + List statusPageHistoryList = statusPageHistoryDao + .findStatusPageHistoriesByTimestampBetween(preNightTimestamp, midnightTimestamp); + Map statusPageHistoryMap = new HashMap<>(8); + for (StatusPageHistory statusPageHistory : statusPageHistoryList) { + statusPageHistory.setNormal(0); + statusPageHistory.setAbnormal(0); + statusPageHistory.setUnknowing(0); + if (statusPageHistoryMap.containsKey(statusPageHistory.getComponentId())) { + StatusPageHistory history = statusPageHistoryMap.get(statusPageHistory.getComponentId()); + if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { + history.setAbnormal(history.getAbnormal() + intervals); + } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { + history.setUnknowing(history.getUnknowing() + intervals); + } else { + history.setNormal(history.getNormal() + intervals); + } + statusPageHistoryMap.put(statusPageHistory.getComponentId(), history); + } else { + if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { + statusPageHistory.setAbnormal(intervals); + } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { + statusPageHistory.setUnknowing(intervals); + } else { + statusPageHistory.setNormal(intervals); + } + statusPageHistoryMap.put(statusPageHistory.getComponentId(), statusPageHistory); + } + } + statusPageHistoryDao.deleteAll(statusPageHistoryList); + for (StatusPageHistory history : statusPageHistoryMap.values()) { + double total = history.getNormal() + history.getAbnormal() + history.getUnknowing(); + double uptime = 0; + if (total > 0) { + uptime = (double) history.getNormal() / total; + } + history.setUptime(uptime); + if (history.getAbnormal() > 0) { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL); + } else if (history.getNormal() > 0) { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL); + } else { + history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + } + statusPageHistoryDao.save(history); + } + } catch (Exception e) { + log.error("status page combine history error: {}", e.getMessage(), e); + } + } + + private ScheduledExecutorService createScheduler(String threadNameFormat, String errorMessage) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("History combine has uncaughtException."); + log.error(errorMessage); log.error(throwable.getMessage(), throwable); }) .setDaemon(true) - .setNameFormat("status-page-calculate-%d") + .setNameFormat(threadNameFormat) .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - // combine history every day at 1:00 AM - LocalDateTime now = LocalDateTime.now(); - LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0); - if (now.isAfter(nextRun)) { - nextRun = nextRun.plusDays(1); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createVirtualExecutor(VirtualThreadProperties virtualThreadProperties, String threadPrefix, + String errorMessage) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; } - long delay = Duration.between(now, nextRun).toMillis(); - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - // combine pre day status history to one record - LocalDateTime nowTime = LocalDateTime.now(); - ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(Instant.now()); - LocalDateTime midnight = nowTime.withHour(0).withMinute(0).withSecond(0).withNano(0); - LocalDateTime preNight = midnight.minusDays(1); - long midnightTimestamp = midnight.toInstant(zoneOffset).toEpochMilli(); - long preNightTimestamp = preNight.toInstant(zoneOffset).toEpochMilli(); - List statusPageHistoryList = statusPageHistoryDao - .findStatusPageHistoriesByTimestampBetween(preNightTimestamp, midnightTimestamp); - Map statusPageHistoryMap = new HashMap<>(8); - for (StatusPageHistory statusPageHistory : statusPageHistoryList) { - statusPageHistory.setNormal(0); - statusPageHistory.setAbnormal(0); - statusPageHistory.setUnknowing(0); - if (statusPageHistoryMap.containsKey(statusPageHistory.getComponentId())) { - StatusPageHistory history = statusPageHistoryMap.get(statusPageHistory.getComponentId()); - if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { - history.setAbnormal(history.getAbnormal() + intervals); - } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { - history.setUnknowing(history.getUnknowing() + intervals); - } else { - history.setNormal(history.getNormal() + intervals); - } - statusPageHistoryMap.put(statusPageHistory.getComponentId(), history); - } else { - if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL) { - statusPageHistory.setAbnormal(intervals); - } else if (statusPageHistory.getState() == CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN) { - statusPageHistory.setUnknowing(intervals); - } else { - statusPageHistory.setNormal(intervals); - } - statusPageHistoryMap.put(statusPageHistory.getComponentId(), statusPageHistory); - } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name(threadPrefix, 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error(errorMessage); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executorService; + private final Runnable task; + private final Object lock = new Object(); + private boolean running; + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { + this.executorService = executorService; + this.task = task; + } + + private void dispatch() { + if (executorService == null) { + task.run(); + return; + } + synchronized (lock) { + if (running) { + pendingRuns++; + return; } - statusPageHistoryDao.deleteAll(statusPageHistoryList); - for (StatusPageHistory history : statusPageHistoryMap.values()) { - double total = history.getNormal() + history.getAbnormal() + history.getUnknowing(); - double uptime = 0; - if (total > 0) { - uptime = (double) history.getNormal() / total; + running = true; + } + submit(); + } + + private void submit() { + boolean submitted = false; + try { + executorService.execute(() -> { + try { + task.run(); + } finally { + onComplete(); } - history.setUptime(uptime); - if (history.getAbnormal() > 0) { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_ABNORMAL); - } else if (history.getNormal() > 0) { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_NORMAL); - } else { - history.setState(CommonConstants.STATUS_PAGE_COMPONENT_STATE_UNKNOWN); + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + running = false; + pendingRuns = 0; } - statusPageHistoryDao.save(history); } - } catch (Exception e) { - log.error("status page combine history error: {}", e.getMessage(), e); } - }, delay, TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS); - } + } - /** - * get calculate status intervals - * @return intervals - */ - public int getCalculateStatusIntervals() { - return intervals; + private void onComplete() { + boolean shouldRunAgain; + synchronized (lock) { + if (pendingRuns > 0) { + pendingRuns--; + shouldRunAgain = true; + } else { + running = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submit(); + } + } } } diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java index 1bc4a72b984..baefdcdbeb7 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java @@ -18,12 +18,18 @@ package org.apache.hertzbeat.manager.scheduler; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,23 +37,54 @@ */ @Slf4j @Component -public class ManagerWorkerPool { - private ThreadPoolExecutor workerExecutor; +public class ManagerWorkerPool implements DisposableBean { + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public ManagerWorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("workerExecutor has uncaughtException."); + @Autowired + public ManagerWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("workerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getManager(); + return ManagedExecutors.newVirtualExecutor("manager-worker", "manager-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("manager-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("manager-long-running", "manager-long-running-", + (thread, throwable) -> { + log.error("manager longRunningExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("manager-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(6, + return new ThreadPoolExecutor(6, 10, 10, TimeUnit.SECONDS, @@ -59,4 +96,21 @@ private void initWorkExecutor() { public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } -} \ No newline at end of file + + /** + * Run a long-lived task outside of the short-task execution lane. + * + * @param runnable task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + + @Override + public void destroy() throws Exception { + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); + } + } +} diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 352415b65f0..7fb26a974d6 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -20,11 +20,13 @@ import io.netty.channel.Channel; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; @@ -39,6 +41,7 @@ import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; import org.apache.hertzbeat.remoting.netty.NettyServerConfig; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.Ordered; @@ -61,6 +64,14 @@ public class ManageServer implements CommandLineRunner { private ScheduledExecutorService channelSchedule; + private final ExecutorService channelCheckExecutor; + + private final Object channelCheckLock = new Object(); + + private boolean channelCheckRunning; + + private boolean channelCheckPending; + private RemotingServer remotingServer; private final Map clientChannelTable = new ConcurrentHashMap<>(16); @@ -69,9 +80,20 @@ public ManageServer(final SchedulerProperties schedulerProperties, final CollectorJobScheduler collectorJobScheduler, final CommonThreadPool threadPool, final CollectorAlertHandler collectorAlertHandler) { + this(schedulerProperties, collectorJobScheduler, threadPool, collectorAlertHandler, + VirtualThreadProperties.defaults()); + } + + @Autowired + public ManageServer(final SchedulerProperties schedulerProperties, + final CollectorJobScheduler collectorJobScheduler, + final CommonThreadPool threadPool, + final CollectorAlertHandler collectorAlertHandler, + final VirtualThreadProperties virtualThreadProperties) { this.collectorJobScheduler = collectorJobScheduler; this.collectorJobScheduler.setManageServer(this); this.collectorAlertHandler = collectorAlertHandler; + this.channelCheckExecutor = createChannelCheckExecutor(virtualThreadProperties); this.init(schedulerProperties, threadPool); } @@ -96,26 +118,18 @@ private void init(final SchedulerProperties schedulerProperties, final CommonThr public void start() { this.remotingServer.start(); - this.channelSchedule.scheduleAtFixedRate(() -> { - try { - this.clientChannelTable.forEach((collector, channel) -> { - if (!channel.isActive()) { - channel.closeFuture(); - this.clientChannelTable.remove(collector); - this.collectorJobScheduler.collectorGoOffline(collector); - this.collectorAlertHandler.offline(collector); - } - }); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - }, 10, 3, TimeUnit.SECONDS); + this.channelSchedule.scheduleAtFixedRate(this::dispatchChannelHealthCheck, 10, 3, TimeUnit.SECONDS); } public void shutdown() { this.remotingServer.shutdown(); - this.channelSchedule.shutdownNow(); + if (this.channelSchedule != null) { + this.channelSchedule.shutdownNow(); + } + if (this.channelCheckExecutor != null) { + this.channelCheckExecutor.shutdownNow(); + } } public CollectorJobScheduler getCollectorAndJobScheduler() { @@ -173,6 +187,21 @@ public ClusterMsg.Message sendMsgSync(final String identityId, final ClusterMsg. return null; } + void dispatchChannelHealthCheck() { + if (channelCheckExecutor == null) { + runChannelHealthCheck(); + return; + } + synchronized (channelCheckLock) { + if (channelCheckRunning) { + channelCheckPending = true; + return; + } + channelCheckRunning = true; + } + submitChannelHealthCheck(); + } + @Override public void run(String... args) throws Exception { this.start(); @@ -200,4 +229,68 @@ public void onChannelIdle(Channel channel) { } } } + + private ExecutorService createChannelCheckExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("manager-channel-check-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> log.error("Channel checker has uncaughtException.", throwable)) + .factory()); + } + + private void submitChannelHealthCheck() { + boolean submitted = false; + try { + channelCheckExecutor.execute(() -> { + try { + runChannelHealthCheck(); + } finally { + onChannelHealthCheckComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (channelCheckLock) { + channelCheckRunning = false; + channelCheckPending = false; + } + } + } + } + + private void onChannelHealthCheckComplete() { + boolean shouldRunAgain; + synchronized (channelCheckLock) { + if (channelCheckPending) { + channelCheckPending = false; + shouldRunAgain = true; + } else { + channelCheckRunning = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submitChannelHealthCheck(); + } + } + + private void runChannelHealthCheck() { + try { + this.clientChannelTable.forEach((collector, channel) -> { + if (!channel.isActive()) { + channel.closeFuture(); + this.clientChannelTable.remove(collector); + this.collectorJobScheduler.collectorGoOffline(collector); + this.collectorAlertHandler.offline(collector); + } + }); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java new file mode 100644 index 00000000000..0ec7833213f --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/component/status/CalculateStatusTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.component.status; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.manager.config.StatusProperties; +import org.apache.hertzbeat.manager.dao.MonitorDao; +import org.apache.hertzbeat.manager.dao.StatusPageComponentDao; +import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao; +import org.apache.hertzbeat.manager.dao.StatusPageOrgDao; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Test case for {@link CalculateStatus}. + */ +@ExtendWith(MockitoExtension.class) +class CalculateStatusTest { + + @Mock + private StatusPageOrgDao statusPageOrgDao; + + @Mock + private StatusPageComponentDao statusPageComponentDao; + + @Mock + private StatusPageHistoryDao statusPageHistoryDao; + + @Mock + private MonitorDao monitorDao; + + private CalculateStatus calculateStatus; + + @BeforeEach + void setUp() { + calculateStatus = new CalculateStatus(statusPageOrgDao, statusPageComponentDao, statusProperties(), + statusPageHistoryDao, monitorDao, new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (calculateStatus != null) { + calculateStatus.destroy(); + } + } + + @Test + void dispatchCalculateRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCombineHistoryRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return Collections.emptyList(); + }).when(statusPageHistoryDao).findStatusPageHistoriesByTimestampBetween(anyLong(), anyLong()); + + calculateStatus.dispatchCombineHistory(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCalculateDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return Collections.emptyList(); + }).when(statusPageOrgDao).findAll(); + + calculateStatus.dispatchCalculate(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + calculateStatus.dispatchCalculate(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private StatusProperties statusProperties() { + StatusProperties statusProperties = new StatusProperties(); + StatusProperties.CalculateProperties calculateProperties = new StatusProperties.CalculateProperties(); + calculateProperties.setInterval(300); + statusProperties.setCalculate(calculateProperties); + return statusProperties; + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java new file mode 100644 index 00000000000..35757bf5afc --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.scheduler; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Test for {@link ManagerWorkerPool}. + */ +class ManagerWorkerPoolTest { + + private ManagerWorkerPool managerWorkerPool; + + @AfterEach + void tearDown() throws Exception { + if (managerWorkerPool != null) { + managerWorkerPool.destroy(); + } + } + + @Test + void testExecuteJobRunsOnVirtualThread() throws Exception { + managerWorkerPool = new ManagerWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + managerWorkerPool.executeJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties managerProperties = new VirtualThreadProperties.PoolProperties(); + managerProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + managerProperties.setMaxConcurrentJobs(1); + properties.setManager(managerProperties); + managerWorkerPool = new ManagerWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + managerWorkerPool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> managerWorkerPool.executeJob(() -> { + })); + } finally { + release.countDown(); + } + } + + @Test + void testExecuteLongRunningRunsOnPlatformThread() throws Exception { + managerWorkerPool = new ManagerWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(true); + + managerWorkerPool.executeLongRunning(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertFalse(virtualThread.get()); + } +} diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java new file mode 100644 index 00000000000..eef43410c0a --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServerTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.scheduler.netty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.support.CommonThreadPool; +import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler; +import org.apache.hertzbeat.manager.scheduler.SchedulerProperties; +import org.apache.hertzbeat.remoting.RemotingServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +/** + * Test case for {@link ManageServer}. + */ +@ExtendWith(MockitoExtension.class) +class ManageServerTest { + + @Mock + private CollectorJobScheduler collectorJobScheduler; + + @Mock + private CommonThreadPool commonThreadPool; + + @Mock + private CollectorAlertHandler collectorAlertHandler; + + private ManageServer manageServer; + + @BeforeEach + void setUp() { + manageServer = new ManageServer(schedulerProperties(), collectorJobScheduler, commonThreadPool, + collectorAlertHandler, new VirtualThreadProperties()); + ReflectionTestUtils.setField(manageServer, "remotingServer", mock(RemotingServer.class)); + } + + @AfterEach + void tearDown() { + if (manageServer != null) { + manageServer.shutdown(); + } + } + + @Test + void dispatchChannelHealthCheckRunsOnVirtualThread() throws Exception { + Channel channel = mock(Channel.class); + when(channel.isActive()).thenReturn(false); + when(channel.closeFuture()).thenReturn(mock(ChannelFuture.class)); + clientChannelTable().put("collector-1", channel); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(collectorJobScheduler).collectorGoOffline(anyString()); + + manageServer.dispatchChannelHealthCheck(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchChannelHealthCheckDoesNotRunConcurrently() throws Exception { + Channel channel = mock(Channel.class); + clientChannelTable().put("collector-1", channel); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return true; + }).when(channel).isActive(); + + manageServer.dispatchChannelHealthCheck(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + manageServer.dispatchChannelHealthCheck(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @SuppressWarnings("unchecked") + private Map clientChannelTable() { + return (Map) ReflectionTestUtils.getField(manageServer, "clientChannelTable"); + } + + private SchedulerProperties schedulerProperties() { + SchedulerProperties schedulerProperties = new SchedulerProperties(); + SchedulerProperties.ServerProperties serverProperties = new SchedulerProperties.ServerProperties(); + serverProperties.setPort(1158); + serverProperties.setIdleStateEventTriggerTime(100); + schedulerProperties.setServer(serverProperties); + return schedulerProperties; + } +} diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java index a01c7fab804..7863e830e31 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java @@ -72,7 +72,7 @@ public NettyRemotingClient(final NettyClientConfig nettyClientConfig, @Override public void start() { - this.threadPool.execute(() -> { + this.threadPool.executeLongRunning(() -> { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("NettyClientWorker has uncaughtException."); diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java index 48062fadb9b..169f38220f3 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java @@ -77,7 +77,7 @@ public NettyRemotingServer(final NettyServerConfig nettyServerConfig, @Override public void start() { - this.threadPool.execute(() -> { + this.threadPool.executeLongRunning(() -> { int port = this.nettyServerConfig.getPort(); ThreadFactory bossThreadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java new file mode 100644 index 00000000000..28b0074c1d4 --- /dev/null +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SimpleAsyncTaskExecutor; + +/** + * Dedicated {@code @Async} executor configuration. + */ +@Configuration(proxyBeanMethods = false) +public class AsyncConfig { + + @Bean(name = "taskExecutor", destroyMethod = "close") + public SimpleAsyncTaskExecutor taskExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.getAsync(); + SimpleAsyncTaskExecutor executor = new SimpleAsyncTaskExecutor("async-worker-"); + executor.setVirtualThreads(virtualThreadProperties.isEnabled() && asyncProperties.isEnabled()); + executor.setConcurrencyLimit(asyncProperties.getConcurrencyLimit()); + executor.setRejectTasksWhenLimitReached(asyncProperties.isRejectWhenLimitReached()); + executor.setTaskTerminationTimeout(asyncProperties.getTaskTerminationTimeout()); + return executor; + } +} diff --git a/hertzbeat-startup/src/main/resources/application-test.yml b/hertzbeat-startup/src/main/resources/application-test.yml index 5a869dae1c2..eaf3e299d24 100644 --- a/hertzbeat-startup/src/main/resources/application-test.yml +++ b/hertzbeat-startup/src/main/resources/application-test.yml @@ -190,3 +190,34 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index 1203a679b16..ba90e282afb 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -334,3 +334,34 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java new file mode 100644 index 00000000000..00d6e2d1052 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.Test; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.core.task.TaskRejectedException; + +/** + * Tests for {@link AsyncConfig}. + */ +class AsyncConfigTest { + + private final AsyncConfig asyncConfig = new AsyncConfig(); + + @Test + void taskExecutorRunsAsyncTasksOnVirtualThreads() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + executor.execute(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + } + + @Test + void taskExecutorRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + properties.getAsync().setConcurrencyLimit(1); + properties.getAsync().setRejectWhenLimitReached(true); + + try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + + executor.execute(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(TaskRejectedException.class, () -> executor.execute(() -> { + })); + } finally { + release.countDown(); + } + } + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java index 14b77f9ba03..af4eb5172ee 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java @@ -18,12 +18,18 @@ package org.apache.hertzbeat.warehouse; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,24 +37,55 @@ */ @Component @Slf4j -public class WarehouseWorkerPool { +public class WarehouseWorkerPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public WarehouseWorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); + } + + @Autowired + public WarehouseWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Warehouse workerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.isEnabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.getWarehouse(); + return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-", + poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.isEnabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-", + (thread, throwable) -> { + log.error("Warehouse longRunningExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }); } - private void initWorkExecutor() { - // Thread factory + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("workerExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("warehouse-worker-%d") .build(); - workerExecutor = new ThreadPoolExecutor(2, + return new ThreadPoolExecutor(2, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, @@ -65,4 +102,21 @@ private void initWorkExecutor() { public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + + /** + * Run a long-lived warehouse consumer outside of the per-task executor. + * + * @param runnable consumer task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + + @Override + public void destroy() throws Exception { + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); + } + } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java index 177b337344f..aee2a33ac2a 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/DataStorageDispatch.java @@ -100,7 +100,7 @@ protected void startPersistentDataStorage() { } } }; - workerPool.executeJob(runnable); + workerPool.executeLongRunning(runnable); } protected void startLogDataStorage() { @@ -132,7 +132,7 @@ protected void startLogDataStorage() { } } }; - workerPool.executeJob(runnable); + workerPool.executeLongRunning(runnable); } protected void calculateMonitorStatus(CollectRep.MetricsData metricsData) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java index 22bcd468272..900c1fdc7a7 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/doris/DorisDataStorage.java @@ -334,7 +334,7 @@ private void initStreamLoadWriter() { */ private void startFlushThread() { try { - warehouseWorkerPool.executeJob(() -> { + warehouseWorkerPool.executeLongRunning(() -> { flushTaskStarted = true; try { while (flushThreadRunning || !metricsBufferQueue.isEmpty()) { diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java index d98c2ee7fb9..17ec3b60bfd 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java @@ -21,6 +21,7 @@ import com.zaxxer.hikari.HikariDataSource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.constants.MetricDataConstants; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; @@ -29,6 +30,7 @@ import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -46,6 +48,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -71,13 +74,35 @@ public class DuckdbDatabaseDataStorage extends AbstractHistoryDataStorage { private final String expireTimeStr; private final String dbPath; + private final ScheduledExecutorService cleanerScheduler; + private final ExecutorService cleanerExecutor; + private final ScheduledDispatchTask cleanerTask; private HikariDataSource dataSource; public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties) { + this(duckdbProperties, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + VirtualThreadProperties virtualThreadProperties) { + this(duckdbProperties, virtualThreadProperties, true); + } + + DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + VirtualThreadProperties virtualThreadProperties, + boolean autoStartCleaner) { this.dbPath = duckdbProperties.storePath(); this.expireTimeStr = duckdbProperties.expireTime(); + this.cleanerScheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "duckdb-cleaner"); + thread.setDaemon(true); + return thread; + }); + this.cleanerExecutor = createCleanerExecutor(virtualThreadProperties); + this.cleanerTask = new ScheduledDispatchTask(cleanerExecutor, this::runExpiredDataCleaner); this.serverAvailable = initDuckDb(); - if (this.serverAvailable) { + if (this.serverAvailable && autoStartCleaner) { startExpiredDataCleaner(); } } @@ -131,60 +156,8 @@ CREATE TABLE IF NOT EXISTS hzb_history ( } private void startExpiredDataCleaner() { - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(r -> { - Thread thread = new Thread(r, "duckdb-cleaner"); - thread.setDaemon(true); - return thread; - }); // Run every 1 hour - scheduledExecutor.scheduleAtFixedRate(() -> { - log.info("[duckdb] start data cleaner and checkpoint..."); - long expireTime; - try { - // Ensure no whitespace issues - String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim(); - Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr); - - if (NumberUtils.isParsable(cleanExpireStr)) { - expireTime = NumberUtils.toLong(cleanExpireStr); - expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L; - } else if (dayMatcher.matches()) { - // Strictly matched "90d" or "90D" format - long days = Long.parseLong(dayMatcher.group(1)); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days)); - expireTime = dateTime.toEpochSecond() * 1000L; - } else { - // Fallback to existing utility for other units (h, m, s, etc.) - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - expireTime = dateTime.toEpochSecond() * 1000L; - } - } catch (Exception e) { - log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30)); - expireTime = dateTime.toEpochSecond() * 1000L; - } - - try (Connection connection = this.dataSource.getConnection()) { - // 1. Delete expired data - try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) { - statement.setLong(1, expireTime); - int rows = statement.executeUpdate(); - if (rows > 0) { - log.info("[duckdb] delete {} expired records.", rows); - } - } - - // 2. Force Checkpoint to compress data and flush WAL - // This is crucial for keeping file size small and moving data from WAL to column store - try (Statement statement = connection.createStatement()) { - statement.execute("CHECKPOINT"); - } - - } catch (Exception e) { - log.error("[duckdb] clean expired data error: {}", e.getMessage(), e); - } - }, 5, 60, TimeUnit.MINUTES); + cleanerScheduler.scheduleAtFixedRate(this::dispatchExpiredDataCleaner, 5, 60, TimeUnit.MINUTES); } @Override @@ -444,8 +417,151 @@ private String formatValue(int type, ResultSet resultSet) throws SQLException { @Override public void destroy() throws Exception { + cleanerScheduler.shutdownNow(); + if (cleanerExecutor != null) { + cleanerExecutor.shutdownNow(); + } if (this.dataSource != null) { this.dataSource.close(); } } + + void dispatchExpiredDataCleaner() { + cleanerTask.dispatch(); + } + + void beforeExpiredDataCleanerRun() { + // Test hook. + } + + private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.isEnabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("duckdb-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("[duckdb] cleaner worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runExpiredDataCleaner() { + log.info("[duckdb] start data cleaner and checkpoint..."); + beforeExpiredDataCleanerRun(); + long expireTime; + try { + // Ensure no whitespace issues + String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim(); + Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr); + + if (NumberUtils.isParsable(cleanExpireStr)) { + expireTime = NumberUtils.toLong(cleanExpireStr); + expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L; + } else if (dayMatcher.matches()) { + // Strictly matched "90d" or "90D" format + long days = Long.parseLong(dayMatcher.group(1)); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days)); + expireTime = dateTime.toEpochSecond() * 1000L; + } else { + // Fallback to existing utility for other units (h, m, s, etc.) + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000L; + } + } catch (Exception e) { + log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30)); + expireTime = dateTime.toEpochSecond() * 1000L; + } + + try (Connection connection = this.dataSource.getConnection()) { + // 1. Delete expired data + try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) { + statement.setLong(1, expireTime); + int rows = statement.executeUpdate(); + if (rows > 0) { + log.info("[duckdb] delete {} expired records.", rows); + } + } + + // 2. Force Checkpoint to compress data and flush WAL + // This is crucial for keeping file size small and moving data from WAL to column store + try (Statement statement = connection.createStatement()) { + statement.execute("CHECKPOINT"); + } + + } catch (Exception e) { + log.error("[duckdb] clean expired data error: {}", e.getMessage(), e); + } + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executorService; + private final Runnable task; + private final Object lock = new Object(); + private boolean running; + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executorService, Runnable task) { + this.executorService = executorService; + this.task = task; + } + + private void dispatch() { + if (executorService == null) { + task.run(); + return; + } + synchronized (lock) { + if (running) { + pendingRuns++; + return; + } + running = true; + } + submit(); + } + + private void submit() { + boolean submitted = false; + try { + executorService.execute(() -> { + try { + task.run(); + } finally { + onComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + running = false; + pendingRuns = 0; + } + } + } + } + + private void onComplete() { + boolean shouldRunAgain; + synchronized (lock) { + if (pendingRuns > 0) { + pendingRuns--; + shouldRunAgain = true; + } else { + running = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submit(); + } + } + } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java index 028ae722cb1..c014ae24c57 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java @@ -18,9 +18,16 @@ package org.apache.hertzbeat.warehouse; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.BeforeEach; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -30,18 +37,19 @@ class WarehouseWorkerPoolTest { private static final int NUMBER_OF_THREADS = 10; private WarehouseWorkerPool pool; - private AtomicInteger counter; - private CountDownLatch latch; - @BeforeEach - void setUp() { - pool = new WarehouseWorkerPool(); - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); + @AfterEach + void tearDown() throws Exception { + if (pool != null) { + pool.destroy(); + } } @Test void executeJob() throws InterruptedException { + pool = new WarehouseWorkerPool(); + AtomicInteger counter = new AtomicInteger(); + CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS); for (int i = 0; i < NUMBER_OF_THREADS; i++) { pool.executeJob(() -> { counter.incrementAndGet(); @@ -53,4 +61,47 @@ void executeJob() throws InterruptedException { assertEquals(NUMBER_OF_THREADS, counter.get()); } + @Test + void executeJobRunsOnVirtualThread() throws InterruptedException { + pool = new WarehouseWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException { + VirtualThreadProperties properties = new VirtualThreadProperties(); + VirtualThreadProperties.PoolProperties warehouseProperties = new VirtualThreadProperties.PoolProperties(); + warehouseProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); + warehouseProperties.setMaxConcurrentJobs(1); + properties.setWarehouse(warehouseProperties); + pool = new WarehouseWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeJob(() -> { + })); + } finally { + release.countDown(); + } + } } diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java new file mode 100644 index 00000000000..4a796ec99ed --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorageTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Test case for {@link DuckdbDatabaseDataStorage}. + */ +class DuckdbDatabaseDataStorageTest { + + private TestDuckdbDatabaseDataStorage dataStorage; + + @TempDir + Path tempDir; + + @AfterEach + void tearDown() throws Exception { + if (dataStorage != null) { + dataStorage.destroy(); + } + } + + @Test + void dispatchExpiredDataCleanerRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + dataStorage = new TestDuckdbDatabaseDataStorage(properties(), latch, virtualThread, + null, null, null, null, null, null); + + dataStorage.dispatchExpiredDataCleaner(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchExpiredDataCleanerDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + dataStorage = new TestDuckdbDatabaseDataStorage(properties(), null, null, + firstStarted, releaseFirst, secondStarted, concurrent, maxConcurrent, invocations); + + dataStorage.dispatchExpiredDataCleaner(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + dataStorage.dispatchExpiredDataCleaner(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private DuckdbProperties properties() { + return new DuckdbProperties(true, "1d", tempDir.resolve("history.duckdb").toString()); + } + + private static final class TestDuckdbDatabaseDataStorage extends DuckdbDatabaseDataStorage { + + private final CountDownLatch virtualThreadLatch; + private final AtomicBoolean virtualThread; + private final CountDownLatch firstStarted; + private final CountDownLatch releaseFirst; + private final CountDownLatch secondStarted; + private final AtomicInteger concurrent; + private final AtomicInteger maxConcurrent; + private final AtomicInteger invocations; + + private TestDuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties, + CountDownLatch virtualThreadLatch, + AtomicBoolean virtualThread, + CountDownLatch firstStarted, + CountDownLatch releaseFirst, + CountDownLatch secondStarted, + AtomicInteger concurrent, + AtomicInteger maxConcurrent, + AtomicInteger invocations) { + super(duckdbProperties, new VirtualThreadProperties(), false); + this.virtualThreadLatch = virtualThreadLatch; + this.virtualThread = virtualThread; + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.concurrent = concurrent; + this.maxConcurrent = maxConcurrent; + this.invocations = invocations; + } + + @Override + void beforeExpiredDataCleanerRun() { + if (virtualThreadLatch != null && virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + virtualThreadLatch.countDown(); + } + if (firstStarted == null || releaseFirst == null || secondStarted == null + || concurrent == null || maxConcurrent == null || invocations == null) { + return; + } + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } +} From 6f389d012ea03b49c1be64d05f012e3a294adc4a Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:32:06 +0800 Subject: [PATCH 02/12] chore: document virtual thread configuration defaults --- .../src/main/resources/application.yml | 1 + .../config/VirtualThreadPropertiesTest.java | 59 +++++++++ .../src/main/resources/application.yml | 1 + home/docs/start/custom-config.md | 6 + home/docs/start/virtual-thread.md | 115 ++++++++++++++++++ .../current/start/custom-config.md | 6 + .../current/start/virtual-thread.md | 115 ++++++++++++++++++ script/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 31 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ .../conf/application.yml | 32 +++++ 13 files changed, 494 insertions(+) create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java create mode 100644 home/docs/start/virtual-thread.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 43057c241bf..9615e547b94 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -77,6 +77,7 @@ common: type: netty hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true common: diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java new file mode 100644 index 00000000000..7433ae3944c --- /dev/null +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.junit.jupiter.api.Test; + +class VirtualThreadPropertiesTest { + + @Test + void defaultsRemainSafeWithoutExternalConfiguration() { + VirtualThreadProperties properties = VirtualThreadProperties.defaults(); + + assertTrue(properties.isEnabled()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); + assertTrue(properties.getCollector().getMaxConcurrentJobs() >= 1); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); + assertEquals(10, properties.getManager().getMaxConcurrentJobs()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getAlerter().getNotify().getMode()); + assertEquals(64, properties.getAlerter().getNotify().getMaxConcurrentJobs()); + assertEquals(10, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + assertEquals(10, properties.getAlerter().getLogWorker().getMaxConcurrentJobs()); + assertEquals(1000, properties.getAlerter().getLogWorker().getQueueCapacity()); + assertEquals(2, properties.getAlerter().getReduce().getMaxConcurrentJobs()); + assertEquals(0, properties.getAlerter().getReduce().getQueueCapacity()); + assertEquals(2, properties.getAlerter().getWindowEvaluator().getMaxConcurrentJobs()); + assertEquals(0, properties.getAlerter().getWindowEvaluator().getQueueCapacity()); + assertEquals(4, properties.getAlerter().getNotifyMaxConcurrentPerChannel()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getWarehouse().getMode()); + + assertTrue(properties.getAsync().isEnabled()); + assertEquals(256, properties.getAsync().getConcurrencyLimit()); + assertTrue(properties.getAsync().isRejectWhenLimitReached()); + assertEquals(5000L, properties.getAsync().getTaskTerminationTimeout()); + } +} diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index ba90e282afb..ecebc66681f 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -336,6 +336,7 @@ grafana: password: admin hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true common: diff --git a/home/docs/start/custom-config.md b/home/docs/start/custom-config.md index 59df329d6aa..4d6953a0972 100644 --- a/home/docs/start/custom-config.md +++ b/home/docs/start/custom-config.md @@ -14,6 +14,12 @@ Configuring the HertzBeat configuration file: - **Docker Deployment:** ⚠️ When using a Docker container, the `application.yml` file must be mounted to the host machine - **Installation Package Deployment:** Extract the package and modify the configuration file located at `hertzbeat/config/application.yml` +## 0. Virtual Thread Configuration + +Virtual-thread related defaults, tuning guidance, rollback switches, and Docker/package config locations are documented on a dedicated page: + +- [Virtual Thread Configuration](./virtual-thread) + ## 1. Configuring Custom Alert Parameters ```yaml diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md new file mode 100644 index 00000000000..d28da0f3ca0 --- /dev/null +++ b/home/docs/start/virtual-thread.md @@ -0,0 +1,115 @@ +--- +id: virtual-thread +title: Virtual Thread Configuration +sidebar_label: Virtual Threads +description: Configure HertzBeat virtual-thread executors, defaults, rollback switches, and tuning guidance. +--- + +HertzBeat runs on JDK 21 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. + +## 1. Where to Configure It + +Choose the config file that matches your deployment mode: + +- Package deployment: `hertzbeat/config/application.yml` +- Docker single-node deployment: mount your local `application.yml` to `/opt/hertzbeat/config/application.yml` +- Docker Compose deployment: edit `script/docker-compose/*/conf/application.yml` +- Standalone collector deployment: edit `hertzbeat-collector/config/application.yml` + +## 2. No Configuration Required + +You can leave out the entire `hertzbeat.vthreads` block. + +```yaml +# No virtual-thread override is required. +``` + +HertzBeat will apply runtime defaults automatically. + +## 3. Full Optional Configuration Template + +Use this only when you want to override the defaults: + +```yaml +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 +``` + +## 4. Built-In Defaults + +| Key | Default | Notes | +| --- | --- | --- | +| `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | +| `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | +| `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | Computed at runtime; no YAML update required | +| `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | +| `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | +| `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | +| `hertzbeat.vthreads.alerter.notify.max-concurrent-jobs` | `64` | Global notify concurrency | +| `hertzbeat.vthreads.alerter.notify-max-concurrent-per-channel` | `4` | Per notification channel/type | +| `hertzbeat.vthreads.alerter.periodic-max-concurrent-jobs` | `10` | Global periodic alert concurrency | +| `hertzbeat.vthreads.alerter.log-worker.max-concurrent-jobs` | `10` | Log alert short-task concurrency | +| `hertzbeat.vthreads.alerter.log-worker.queue-capacity` | `1000` | Bounded queue to preserve backlog semantics | +| `hertzbeat.vthreads.alerter.reduce.max-concurrent-jobs` | `2` | Alarm reduce concurrency | +| `hertzbeat.vthreads.alerter.reduce.queue-capacity` | unbounded | Leave unset to keep the legacy unbounded queue behavior | +| `hertzbeat.vthreads.alerter.window-evaluator.max-concurrent-jobs` | `2` | Window evaluator concurrency | +| `hertzbeat.vthreads.alerter.window-evaluator.queue-capacity` | unbounded | Leave unset to keep the legacy unbounded queue behavior | +| `hertzbeat.vthreads.warehouse.mode` | `UNBOUNDED_VT` | Storage short tasks; downstream pools still limit real resources | +| `hertzbeat.vthreads.async.enabled` | `true` | Dedicated `@Async` executor switch | +| `hertzbeat.vthreads.async.concurrency-limit` | `256` | `@Async` concurrency guard | +| `hertzbeat.vthreads.async.reject-when-limit-reached` | `true` | Reject extra `@Async` tasks at the limit | +| `hertzbeat.vthreads.async.task-termination-timeout` | `5000` | Milliseconds | + +## 5. Tuning Guidance + +- Start with the defaults unless you already know a downstream dependency is weak. +- Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. +- Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. +- Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. +- `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. + +## 6. Rollback + +Disable HertzBeat virtual-thread executors with: + +```yaml +hertzbeat: + vthreads: + enabled: false +``` + +This rolls the affected executors back to their legacy platform-thread implementations. + +## 7. Notes + +- You do not need to add `spring.threads.virtual.enabled` for this feature set. HertzBeat uses a dedicated `@Async` executor configuration. +- Shipping YAML files now include the optional `hertzbeat.vthreads` block as an example, but the block is still optional. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md index 03c9538218a..ed18b3af3f3 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/custom-config.md @@ -14,6 +14,12 @@ sidebar_label: 常见参数配置 - **Docker部署：** ⚠️docker容器方式需要将 `application.yml` 文件挂载到主机本地 - **安装包方式：** 解压修改位于 `hertzbeat/config/application.yml` 的配置文件即可 +## 0. 虚拟线程配置 + +虚拟线程的默认值、调优建议、回滚方式，以及 Docker/安装包对应的配置文件位置，已经单独整理到文档页： + +- [虚拟线程配置说明](./virtual-thread) + ## 1. 配置告警自定义参数 ```yaml diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md new file mode 100644 index 00000000000..a8c4c1ba4da --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -0,0 +1,115 @@ +--- +id: virtual-thread +title: 虚拟线程配置说明 +sidebar_label: 虚拟线程 +description: 说明 HertzBeat 虚拟线程执行器的默认值、回滚开关和调优方式。 +--- + +HertzBeat 基于 JDK 21 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 + +## 1. 到哪里配置 + +根据你的部署方式修改对应配置文件： + +- 安装包部署：`hertzbeat/config/application.yml` +- Docker 单机部署：把本地 `application.yml` 挂载到容器内 `/opt/hertzbeat/config/application.yml` +- Docker Compose 部署：修改 `script/docker-compose/*/conf/application.yml` +- 独立 collector 部署：修改 `hertzbeat-collector/config/application.yml` + +## 2. 不配置也可以 + +你可以完全不写 `hertzbeat.vthreads` 这一段： + +```yaml +# 虚拟线程配置可以整体省略。 +``` + +HertzBeat 会自动使用运行时默认值。 + +## 3. 完整可选配置模板 + +只有在你需要覆盖默认值时，才需要显式写出下面这段： + +```yaml +hertzbeat: + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 +``` + +## 4. 内置默认值 + +| 配置项 | 默认值 | 说明 | +| --- | --- | --- | +| `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | +| `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | +| `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | 运行时动态计算，不依赖 YAML | +| `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | +| `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | +| `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | +| `hertzbeat.vthreads.alerter.notify.max-concurrent-jobs` | `64` | 通知全局并发 | +| `hertzbeat.vthreads.alerter.notify-max-concurrent-per-channel` | `4` | 单通知通道/类型并发 | +| `hertzbeat.vthreads.alerter.periodic-max-concurrent-jobs` | `10` | 周期告警全局并发 | +| `hertzbeat.vthreads.alerter.log-worker.max-concurrent-jobs` | `10` | 日志告警短任务并发 | +| `hertzbeat.vthreads.alerter.log-worker.queue-capacity` | `1000` | 有界队列，保留 backlog 语义 | +| `hertzbeat.vthreads.alerter.reduce.max-concurrent-jobs` | `2` | 告警 reduce 并发 | +| `hertzbeat.vthreads.alerter.reduce.queue-capacity` | 无界 | 默认不填，保持旧版无界队列语义 | +| `hertzbeat.vthreads.alerter.window-evaluator.max-concurrent-jobs` | `2` | 窗口 evaluator 并发 | +| `hertzbeat.vthreads.alerter.window-evaluator.queue-capacity` | 无界 | 默认不填，保持旧版无界队列语义 | +| `hertzbeat.vthreads.warehouse.mode` | `UNBOUNDED_VT` | 仓储短任务执行；真实资源仍由下游连接池限制 | +| `hertzbeat.vthreads.async.enabled` | `true` | 专用 `@Async` 执行器开关 | +| `hertzbeat.vthreads.async.concurrency-limit` | `256` | `@Async` 并发保护阈值 | +| `hertzbeat.vthreads.async.reject-when-limit-reached` | `true` | 达到上限后拒绝额外 `@Async` 任务 | +| `hertzbeat.vthreads.async.task-termination-timeout` | `5000` | 单位毫秒 | + +## 5. 调优建议 + +- 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 +- 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 +- 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 +- `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 +- `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 + +## 6. 回滚方式 + +如需关闭 HertzBeat 的虚拟线程执行器，可配置： + +```yaml +hertzbeat: + vthreads: + enabled: false +``` + +这样相关执行器会回退到原来的平台线程实现。 + +## 7. 补充说明 + +- 这套改造不要求额外配置 `spring.threads.virtual.enabled`。HertzBeat 当前使用的是单独的 `@Async` 执行器配置。 +- 交付给用户的 YAML 示例文件现在已经带上了可选的 `hertzbeat.vthreads` 配置块，但这段配置依然可以整体删除。 diff --git a/script/application.yml b/script/application.yml index 1203a679b16..ecebc66681f 100644 --- a/script/application.yml +++ b/script/application.yml @@ -334,3 +334,35 @@ grafana: expose-url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index 2ee71362555..c4c9d80797e 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -234,3 +234,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index b4c195c22fd..8a6dfe1528f 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -232,3 +232,34 @@ grafana: username: admin password: admin +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index c81a69107f0..d9118c6c560 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -234,3 +234,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index a479216209c..0a1feb682dc 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -231,3 +231,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index d071e395be4..ed4c59bcbb4 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -233,3 +233,35 @@ grafana: url: http://127.0.0.1:3000 username: admin password: admin + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT + manager: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 10 + alerter: + notify: + mode: LIMIT_AND_REJECT + max-concurrent-jobs: 64 + periodic-max-concurrent-jobs: 10 + log-worker: + max-concurrent-jobs: 10 + queue-capacity: 1000 + reduce: + max-concurrent-jobs: 2 + window-evaluator: + max-concurrent-jobs: 2 + notify-max-concurrent-per-channel: 4 + warehouse: + mode: UNBOUNDED_VT + async: + enabled: true + concurrency-limit: 256 + reject-when-limit-reached: true + task-termination-timeout: 5000 From b9a60e444f24af2a059320906c0e929bee05f738 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:34:57 +0800 Subject: [PATCH 03/12] docs: rewrite virtual thread notes for users --- home/docs/start/virtual-thread.md | 4 ++-- .../current/start/virtual-thread.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index d28da0f3ca0..89fb0094585 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -111,5 +111,5 @@ This rolls the affected executors back to their legacy platform-thread implement ## 7. Notes -- You do not need to add `spring.threads.virtual.enabled` for this feature set. HertzBeat uses a dedicated `@Async` executor configuration. -- Shipping YAML files now include the optional `hertzbeat.vthreads` block as an example, but the block is still optional. +- If your current deployment is stable, you can keep your existing `application.yml` unchanged. +- Add the `hertzbeat.vthreads` block only when you want to tune concurrency limits or explicitly disable the feature. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index a8c4c1ba4da..bcdc21d55c9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -111,5 +111,5 @@ hertzbeat: ## 7. 补充说明 -- 这套改造不要求额外配置 `spring.threads.virtual.enabled`。HertzBeat 当前使用的是单独的 `@Async` 执行器配置。 -- 交付给用户的 YAML 示例文件现在已经带上了可选的 `hertzbeat.vthreads` 配置块，但这段配置依然可以整体删除。 +- 如果你当前的部署运行稳定，可以继续保持现有 `application.yml` 不变。 +- 只有在你希望调节并发阈值，或者显式关闭该能力时，才需要增加 `hertzbeat.vthreads` 配置块。 From 0404c2bff6b9e07fccea0e975cf9a4473f1e22f1 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 12:41:22 +0800 Subject: [PATCH 04/12] feat: raise collector virtual thread concurrency default --- .../common/config/VirtualThreadProperties.java | 5 +++-- .../common/config/VirtualThreadPropertiesTest.java | 2 +- home/docs/help/issue.md | 11 +++++------ home/docs/start/virtual-thread.md | 4 +++- .../current/help/issue.md | 11 +++++------ .../current/start/virtual-thread.md | 4 +++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index f8590dee3ac..fde25f1e9fe 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -30,6 +30,8 @@ @ConfigurationProperties(prefix = "hertzbeat.vthreads") public class VirtualThreadProperties { + private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; + private boolean enabled = true; private PoolProperties collector = PoolProperties.collectorDefaults(); @@ -94,8 +96,7 @@ private static PoolProperties alerterNotifyDefaults() { } private static int defaultCollectorConcurrency() { - int historicalMax = Runtime.getRuntime().availableProcessors() * 16; - return Math.max(1, historicalMax - 1); + return DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS; } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index 7433ae3944c..a6b32e41da9 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -32,7 +32,7 @@ void defaultsRemainSafeWithoutExternalConfiguration() { assertTrue(properties.isEnabled()); assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); - assertTrue(properties.getCollector().getMaxConcurrentJobs() >= 1); + assertEquals(512, properties.getCollector().getMaxConcurrentJobs()); assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); diff --git a/home/docs/help/issue.md b/home/docs/help/issue.md index 0eae3fca8df..5c6713e23de 100644 --- a/home/docs/help/issue.md +++ b/home/docs/help/issue.md @@ -32,12 +32,11 @@ sidebar_label: Common issues 6. What is the task limit for a single collector? - > Specific limit parameters: - Core thread count: Math.max(2, Runtime.getRuntime().availableProcessors()) – at least 2 threads, or equal to the number of CPU cores. - Maximum thread count: Runtime.getRuntime().availableProcessors() * 16 – 16 times the number of CPU cores. - > The limit depends entirely on the server's CPU core count. For example, on an 8-core CPU server, a maximum of 8 × 16 = 128 collection tasks can be processed simultaneously. Exceeding this number triggers the error message. This is a dynamic configuration that adjusts automatically based on the hardware specifications of the runtime environment. - > If the runtime exceeds the maximum thread count, an error will appear: "the worker pool is full, reject this metrics task, put in queue again". - > In such cases, it is recommended to configure a new collector in public mode. HertzBeat will automatically distribute tasks to other collectors, avoiding errors due to the task limit of a single collector. + > In current versions, the default collector concurrency limit is `512` concurrent collection tasks when virtual threads are enabled. + > This default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. + > If the runtime exceeds the configured collector limit, an error will appear: "the worker pool is full, reject this metrics task, put in queue again". + > You can tune this limit through `hertzbeat.vthreads.collector.max-concurrent-jobs` in `application.yml`. + > If a single node still cannot absorb the workload, configure additional collectors in public mode so HertzBeat can distribute tasks across them. ### Docker Deployment common issues diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index 89fb0094585..6de95e1dbcb 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | Computed at runtime; no YAML update required | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Default single-node collector concurrency target | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | @@ -92,7 +92,9 @@ hertzbeat: ## 5. Tuning Guidance - Start with the defaults unless you already know a downstream dependency is weak. +- The collector default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. - Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. +- Raise `collector.max-concurrent-jobs` above `512` on dedicated collector nodes when downstream services, network bandwidth, and timeouts are already understood and controlled. - Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. - Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. - `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md index 792264ed0f6..c92f3279808 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/issue.md @@ -32,12 +32,11 @@ sidebar_label: 常见问题 6. 单个采集器的任务上限是多少? - > 具体上限参数 - 核心线程数: Math.max(2, Runtime.getRuntime().availableProcessors()) - 至少2个线程，或等于CPU核心数。 - 最大线程数: Runtime.getRuntime().availableProcessors() * 16 - CPU核心数的16倍。 - > 上限完全取决于服务器的CPU核心数。例如，在8核CPU的服务器上，最大可同时处理 8 × 16 = 128 个采集任务。当超过这个数量时就会触发该错误消息。这是一个动态配置，会根据运行环境的硬件规格自动调整。 - > 当运行时超出最大线程数会报错提示"the worker pool is full, reject this metrics task，put in queue again"。 - > 此时建议配置新的采集器,并设置为public模式，hertzbeat会自动将任务分配给其他采集器，不会因为单个采集器任务上限而报错。 + > 在当前版本中，启用虚拟线程后，单个 collector 默认可并发执行 `512` 个采集任务。 + > 这个默认值刻意高于旧版按 CPU 推导出来的线程池上限，目的是让单独部署的 HertzBeat 节点先尽量承载更多阻塞型采集任务，再决定是否扩容额外 collector。 + > 当运行时超出已配置的 collector 并发上限时，会报错提示 "the worker pool is full, reject this metrics task，put in queue again"。 + > 你可以在 `application.yml` 里通过 `hertzbeat.vthreads.collector.max-concurrent-jobs` 调整这个限制。 + > 如果单机仍然无法承载当前任务量，再建议增加新的 collector，并设置为 public 模式，让 HertzBeat 自动做任务分发。 ### Docker部署常见问题 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index bcdc21d55c9..7731c105740 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `availableProcessors() * 16 - 1` | 运行时动态计算，不依赖 YAML | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 单机默认采集并发目标值 | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | @@ -92,7 +92,9 @@ hertzbeat: ## 5. 调优建议 - 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 +- collector 默认值刻意高于旧版按 CPU 推导的线程池上限，这样单独部署 HertzBeat 主程序时可以承载更多阻塞型采集任务，减少对额外 collector 的依赖。 - 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 +- 对于专门部署的 collector 节点，如果你已经清楚下游容量、网络带宽和超时设置，也可以把 `collector.max-concurrent-jobs` 提高到 `512` 以上。 - 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 - `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 - `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 From 61c9b00aea35fe26e40e5a4538d3d960a837937e Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 15:23:03 +0800 Subject: [PATCH 05/12] docs: add virtual thread tuning guidance --- home/docs/start/virtual-thread.md | 9 +++++++-- .../current/start/virtual-thread.md | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index 6de95e1dbcb..f9d36190316 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | Global switch for the HertzBeat virtual-thread executors | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | Common short-running tasks | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | Keeps collector fast-fail admission | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Default single-node collector concurrency target | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | Balanced default for mixed HTTP and JDBC collection workloads on a single node | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | Keeps manager admission behavior | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | Same as the legacy limit | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | Notification executor admission | @@ -93,11 +93,16 @@ hertzbeat: - Start with the defaults unless you already know a downstream dependency is weak. - The collector default is intentionally higher than the legacy CPU-based pool size so a single HertzBeat node can carry more blocking collection work before you need extra collectors. +- `512` is the default because it is a good mixed-workload starting point. In local verification, HTTP-heavy collection continued scaling beyond `512`, while JDBC-style collection peaked around `512` and dropped when concurrency was pushed higher. +- Virtual threads remove platform-thread pressure, but they do not remove database limits, HTTP connection limits, network bandwidth limits, file descriptor limits, or downstream rate limits. Raising concurrency too far just moves the bottleneck. +- If most of your workload is HTTP collection across many different targets, try `768` first and then `1024` if timeouts, error rates, and connection usage remain stable. +- If most of your workload is JDBC or other database-backed collection, keep `collector.max-concurrent-jobs` around `256` to `512`. In this type of workload, raising concurrency above `512` can reduce total throughput instead of improving it. +- If you are not sure about the workload mix, keep `512` as the starting point. It is a safer default than `768+` for mixed environments. - Lower `collector.max-concurrent-jobs` when the collector talks to a small database, a low-capacity HTTP endpoint, or fragile network devices. -- Raise `collector.max-concurrent-jobs` above `512` on dedicated collector nodes when downstream services, network bandwidth, and timeouts are already understood and controlled. - Raise `alerter.notify.max-concurrent-jobs` or `notify-max-concurrent-per-channel` only if your notification providers and HTTP connection pools can absorb the increase. - Keep `warehouse.mode` unbounded unless you have a clear bottleneck model. Database and TSDB client pools should remain the main limiters. - `reduce.queue-capacity` and `window-evaluator.queue-capacity` are intentionally left unset by default so existing queueing semantics remain compatible. +- Change concurrency in steps and observe timeout rate, downstream `429` or `5xx`, database pool wait time, and memory or file descriptor usage before raising it again. ## 6. Rollback diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index 7731c105740..ae69b63e0a9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -70,7 +70,7 @@ hertzbeat: | `hertzbeat.vthreads.enabled` | `true` | HertzBeat 虚拟线程执行器总开关 | | `hertzbeat.vthreads.common.mode` | `UNBOUNDED_VT` | 通用短任务执行器 | | `hertzbeat.vthreads.collector.mode` | `LIMIT_AND_REJECT` | 保持采集入口快速拒绝语义 | -| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 单机默认采集并发目标值 | +| `hertzbeat.vthreads.collector.max-concurrent-jobs` | `512` | 面向单机混合 HTTP/JDBC 采集场景的折中默认值 | | `hertzbeat.vthreads.manager.mode` | `LIMIT_AND_REJECT` | 保持 manager 入口语义 | | `hertzbeat.vthreads.manager.max-concurrent-jobs` | `10` | 与原来的限制一致 | | `hertzbeat.vthreads.alerter.notify.mode` | `LIMIT_AND_REJECT` | 通知执行器入口控制 | @@ -93,11 +93,16 @@ hertzbeat: - 除非你已经明确知道某个下游资源比较脆弱，否则先使用默认值。 - collector 默认值刻意高于旧版按 CPU 推导的线程池上限，这样单独部署 HertzBeat 主程序时可以承载更多阻塞型采集任务，减少对额外 collector 的依赖。 +- 默认值之所以设为 `512`，是因为它更适合作为混合负载起点。我们本地验证时，HTTP 型采集在 `512` 以上还能继续扩展，而 JDBC 型采集在接近 `512` 时已经接近甜点，继续提高并发反而会掉总吞吐。 +- 虚拟线程解决的是平台线程成本问题，不会消除数据库连接上限、HTTP 连接上限、网络带宽、文件描述符或下游服务限流。并发调得过高，只是把瓶颈转移到这些资源上。 +- 如果你的采集任务大多是 HTTP，而且目标分散在很多不同主机上，可以先尝试把 `collector.max-concurrent-jobs` 提高到 `768`，稳定后再考虑 `1024`。 +- 如果你的采集任务大多是 JDBC 或其他数据库型采集，建议把 `collector.max-concurrent-jobs` 控制在 `256` 到 `512`。这类场景下，并发超过 `512` 后不一定更快。 +- 如果你暂时不确定负载结构，就先保持 `512`。对于混合场景，它通常比直接上 `768+` 更稳。 - 当 collector 连接的是小规格数据库、低容量 HTTP 服务或脆弱网络设备时，再下调 `collector.max-concurrent-jobs`。 -- 对于专门部署的 collector 节点，如果你已经清楚下游容量、网络带宽和超时设置，也可以把 `collector.max-concurrent-jobs` 提高到 `512` 以上。 - 只有当通知通道供应商和 HTTP 连接池都能承受更高吞吐时，才上调 `alerter.notify.max-concurrent-jobs` 或 `notify-max-concurrent-per-channel`。 - `warehouse.mode` 建议保持 `UNBOUNDED_VT`，真正的资源限制仍应交给数据库/TSDB 客户端连接池。 - `reduce.queue-capacity` 和 `window-evaluator.queue-capacity` 默认故意不写，这样才能兼容旧版队列语义。 +- 每次调高并发时都建议按小步递增，并同时观察超时率、下游 `429/5xx`、数据库连接池等待时间、内存和文件描述符使用情况。 ## 6. 回滚方式 From 9a62f596ab94ede38f5ab223945214a5cf820173 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 15:43:32 +0800 Subject: [PATCH 06/12] refactor: convert virtual thread properties to records --- .../hertzbeat/alert/AlerterWorkerPool.java | 16 +- .../periodic/PeriodicAlertRuleScheduler.java | 4 +- .../realtime/window/AlarmEvaluator.java | 6 +- .../alert/reduce/AlarmCommonReduce.java | 6 +- .../alert/reduce/AlarmGroupReduce.java | 2 +- .../alert/reduce/AlarmInhibitReduce.java | 2 +- .../alert/AlerterWorkerPoolTest.java | 65 ++++--- .../PeriodicAlertRuleSchedulerTest.java | 17 +- .../realtime/window/AlarmEvaluatorTest.java | 18 +- .../alert/reduce/AlarmCommonReduceTest.java | 18 +- .../collector/dispatch/WorkerPoolTest.java | 13 +- .../collector/dispatch/WorkerPool.java | 8 +- .../dispatch/entrance/CollectServer.java | 2 +- .../config/VirtualThreadProperties.java | 167 +++++++++--------- .../common/support/CommonThreadPool.java | 8 +- .../config/VirtualThreadPropertiesTest.java | 52 +++--- .../common/support/CommonThreadPoolTest.java | 13 +- .../component/status/CalculateStatus.java | 2 +- .../manager/scheduler/ManagerWorkerPool.java | 8 +- .../manager/scheduler/netty/ManageServer.java | 2 +- .../scheduler/ManagerWorkerPoolTest.java | 13 +- .../apache/hertzbeat/startup/AsyncConfig.java | 10 +- .../hertzbeat/startup/AsyncConfigTest.java | 11 +- .../warehouse/WarehouseWorkerPool.java | 8 +- .../duckdb/DuckdbDatabaseDataStorage.java | 2 +- .../warehouse/WarehouseWorkerPoolTest.java | 13 +- 26 files changed, 279 insertions(+), 207 deletions(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java index b6b743b8a2e..b8abee9edf7 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java @@ -83,13 +83,13 @@ private void initNotifyExecutor(VirtualThreadProperties properties) { log.error("Alerter notifyExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.AlerterProperties alerterProperties = properties.getAlerter(); - VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.getNotify(); - notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.getNotifyMaxConcurrentPerChannel()); + if (properties.enabled()) { + VirtualThreadProperties.AlerterProperties alerterProperties = properties.alerter(); + VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.notifyPool(); + notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.notifyMaxConcurrentPerChannel()); notifyChannelPermits = new ConcurrentHashMap<>(8); notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-", - notifyProperties.getMode(), notifyProperties.getMaxConcurrentJobs(), handler); + notifyProperties.mode(), notifyProperties.maxConcurrentJobs(), handler); return; } notifyMaxConcurrentPerChannel = 0; @@ -117,10 +117,10 @@ private void initLogWorkerExecutor(VirtualThreadProperties properties) { log.error("Alerter logWorkerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties logWorkerProperties = properties.getAlerter().getLogWorker(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties logWorkerProperties = properties.alerter().logWorker(); logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-", - logWorkerProperties.getMaxConcurrentJobs(), logWorkerProperties.getQueueCapacity(), handler); + logWorkerProperties.maxConcurrentJobs(), logWorkerProperties.queueCapacity(), handler); return; } logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler)); diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 9ad94c4d4dd..49e9004e77d 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -85,8 +85,8 @@ public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculat this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - this.virtualThreadsEnabled = properties.isEnabled(); - int maxConcurrentPeriodicTasks = Math.max(1, properties.getAlerter().getPeriodicMaxConcurrentJobs()); + this.virtualThreadsEnabled = properties.enabled(); + int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs()); this.periodicExecutor = virtualThreadsEnabled ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() .name("periodic-alert-task-", 0) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java index 1d210ac47c8..e85b145355f 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java @@ -74,10 +74,10 @@ public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) { log.error("alerter-reduce-worker has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getWindowEvaluator(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().windowEvaluator(); return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-", - queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); } return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2, 10, diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 4d6b2edb0e5..75f18f9ee37 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -58,10 +58,10 @@ private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { log.error("alerter-reduce-worker has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.QueueProperties queueProperties = properties.getAlerter().getReduce(); + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().reduce(); return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-", - queueProperties.getMaxConcurrentJobs(), queueProperties.getQueueCapacity(), handler); + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); } return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2, 2, diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index e43b2e43bcb..597ef9a58a6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -159,7 +159,7 @@ private ScheduledExecutorService createScheduler() { } private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index c8a51392692..4ebfdd22bf6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -151,7 +151,7 @@ private ScheduledExecutorService createCleanupScheduler() { } private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java index 434a999eb65..ba83880959d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java @@ -82,14 +82,20 @@ void executeNotifyRunsOnVirtualThread() throws Exception { @Test void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); - notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - notifyProperties.setMaxConcurrentJobs(1); - alerterProperties.setNotify(notifyProperties); - alerterProperties.setNotifyMaxConcurrentPerChannel(8); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 8), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); @@ -114,14 +120,20 @@ void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { @Test void executeNotifyRejectsWhenChannelLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.PoolProperties notifyProperties = new VirtualThreadProperties.PoolProperties(); - notifyProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - notifyProperties.setMaxConcurrentJobs(8); - alerterProperties.setNotify(notifyProperties); - alerterProperties.setNotifyMaxConcurrentPerChannel(1); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 8), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 1), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); @@ -160,13 +172,20 @@ void executeLogJob() throws InterruptedException { @Test void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.AlerterProperties alerterProperties = new VirtualThreadProperties.AlerterProperties(); - VirtualThreadProperties.QueueProperties logWorkerProperties = new VirtualThreadProperties.QueueProperties(); - logWorkerProperties.setMaxConcurrentJobs(1); - logWorkerProperties.setQueueCapacity(1); - alerterProperties.setLogWorker(logWorkerProperties); - properties.setAlerter(alerterProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + new VirtualThreadProperties.QueueProperties(1, 1), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new AlerterWorkerPool(properties); CountDownLatch firstStarted = new CountDownLatch(1); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java index a21f66fc2a8..a18b8113e7d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -195,8 +195,19 @@ private AlertDefine metricRule(Long id) { } private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { - VirtualThreadProperties properties = VirtualThreadProperties.defaults(); - properties.getAlerter().setPeriodicMaxConcurrentJobs(maxConcurrentJobs); - return properties; + return new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + maxConcurrentJobs, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java index 394b9b77d02..d520821bec7 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java @@ -378,10 +378,20 @@ void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception { @Test void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); - queueProperties.setMaxConcurrentJobs(1); - properties.getAlerter().setWindowEvaluator(queueProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); alarmEvaluator.destroy(); alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties); diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index 8dae8999511..02081885293 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -87,10 +87,20 @@ void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception { @Test void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.QueueProperties queueProperties = new VirtualThreadProperties.QueueProperties(); - queueProperties.setMaxConcurrentJobs(1); - properties.getAlerter().setReduce(queueProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); alarmCommonReduce.destroy(); alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties); diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java index e4d2aac786d..2348dc1cf34 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/WorkerPoolTest.java @@ -59,11 +59,14 @@ void testExecuteJobRunsOnVirtualThread() throws Exception { @Test void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties collectorProperties = new VirtualThreadProperties.PoolProperties(); - collectorProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - collectorProperties.setMaxConcurrentJobs(1); - properties.setCollector(collectorProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); workerPool = new WorkerPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java index 7ba179ce40d..a42cc9a4283 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getCollector(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.collector(); return ManagedExecutors.newVirtualExecutor("collector-worker", "collect-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("collector-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("collector-long-running", "collect-long-running-", diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 112209c8b76..1e77da0e211 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -217,7 +217,7 @@ public void onChannelIdle(Channel channel) { private ExecutorService createHeartbeatExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index fde25f1e9fe..8fdbc9c51d1 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -17,34 +17,40 @@ package org.apache.hertzbeat.common.config; -import lombok.Getter; -import lombok.Setter; import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; +import org.springframework.boot.context.properties.bind.Name; /** * Virtual-thread related configuration. */ -@Getter -@Setter @ConfigurationProperties(prefix = "hertzbeat.vthreads") -public class VirtualThreadProperties { +public record VirtualThreadProperties( + @DefaultValue("true") boolean enabled, + @DefaultValue PoolProperties collector, + @DefaultValue PoolProperties common, + @DefaultValue PoolProperties manager, + @DefaultValue AlerterProperties alerter, + @DefaultValue PoolProperties warehouse, + @DefaultValue AsyncProperties async) { private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; - private boolean enabled = true; - - private PoolProperties collector = PoolProperties.collectorDefaults(); - - private PoolProperties common = PoolProperties.commonDefaults(); - - private PoolProperties manager = PoolProperties.managerDefaults(); - - private AlerterProperties alerter = new AlerterProperties(); - - private PoolProperties warehouse = PoolProperties.warehouseDefaults(); + public VirtualThreadProperties { + collector = collector == null ? PoolProperties.collectorDefaults() : collector; + common = common == null ? PoolProperties.commonDefaults() : common; + manager = manager == null ? PoolProperties.managerDefaults() : manager; + alerter = alerter == null ? AlerterProperties.defaults() : alerter; + warehouse = warehouse == null ? PoolProperties.warehouseDefaults() : warehouse; + async = async == null ? AsyncProperties.defaults() : async; + } - private AsyncProperties async = new AsyncProperties(); + public VirtualThreadProperties() { + this(true, PoolProperties.collectorDefaults(), PoolProperties.commonDefaults(), + PoolProperties.managerDefaults(), AlerterProperties.defaults(), + PoolProperties.warehouseDefaults(), AsyncProperties.defaults()); + } /** * Create a detached properties instance with runtime defaults. @@ -58,41 +64,36 @@ public static VirtualThreadProperties defaults() { /** * Pool-level configuration. */ - @Getter - @Setter - public static class PoolProperties { + public record PoolProperties( + @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, + @DefaultValue("0") int maxConcurrentJobs) { - private AdmissionMode mode = AdmissionMode.UNBOUNDED_VT; + public PoolProperties { + mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; + } - private int maxConcurrentJobs; + public PoolProperties() { + this(AdmissionMode.UNBOUNDED_VT, 0); + } - private static PoolProperties collectorDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(defaultCollectorConcurrency()); - return properties; + public static PoolProperties collectorDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, defaultCollectorConcurrency()); } - private static PoolProperties warehouseDefaults() { + public static PoolProperties warehouseDefaults() { return new PoolProperties(); } - private static PoolProperties commonDefaults() { + public static PoolProperties commonDefaults() { return new PoolProperties(); } - private static PoolProperties managerDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(10); - return properties; + public static PoolProperties managerDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 10); } - private static PoolProperties alerterNotifyDefaults() { - PoolProperties properties = new PoolProperties(); - properties.setMode(AdmissionMode.LIMIT_AND_REJECT); - properties.setMaxConcurrentJobs(64); - return properties; + public static PoolProperties alerterNotifyDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 64); } private static int defaultCollectorConcurrency() { @@ -103,67 +104,71 @@ private static int defaultCollectorConcurrency() { /** * Alerter-specific executor configuration. */ - @Getter - @Setter - public static class AlerterProperties { - - private PoolProperties notify = PoolProperties.alerterNotifyDefaults(); - - private int periodicMaxConcurrentJobs = 10; - - private QueueProperties logWorker = QueueProperties.logWorkerDefaults(); - - private QueueProperties reduce = QueueProperties.reduceDefaults(); + public record AlerterProperties( + @Name("notify") @DefaultValue PoolProperties notifyPool, + @DefaultValue("10") int periodicMaxConcurrentJobs, + @DefaultValue QueueProperties logWorker, + @DefaultValue QueueProperties reduce, + @DefaultValue QueueProperties windowEvaluator, + @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + + public AlerterProperties { + notifyPool = notifyPool == null ? PoolProperties.alerterNotifyDefaults() : notifyPool; + logWorker = logWorker == null ? QueueProperties.logWorkerDefaults() : logWorker; + reduce = reduce == null ? QueueProperties.reduceDefaults() : reduce; + windowEvaluator = windowEvaluator == null ? QueueProperties.windowEvaluatorDefaults() : windowEvaluator; + } - private QueueProperties windowEvaluator = QueueProperties.windowEvaluatorDefaults(); + public AlerterProperties() { + this(PoolProperties.alerterNotifyDefaults(), 10, + QueueProperties.logWorkerDefaults(), QueueProperties.reduceDefaults(), + QueueProperties.windowEvaluatorDefaults(), 4); + } - private int notifyMaxConcurrentPerChannel = 4; + public static AlerterProperties defaults() { + return new AlerterProperties(); + } } /** * Queue-preserving executor configuration. */ - @Getter - @Setter - public static class QueueProperties { - - private int maxConcurrentJobs; + public record QueueProperties( + @DefaultValue("0") int maxConcurrentJobs, + @DefaultValue("0") int queueCapacity) { - private int queueCapacity; + public QueueProperties() { + this(0, 0); + } - private static QueueProperties reduceDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(2); - return properties; + public static QueueProperties reduceDefaults() { + return new QueueProperties(2, 0); } - private static QueueProperties logWorkerDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(10); - properties.setQueueCapacity(1000); - return properties; + public static QueueProperties logWorkerDefaults() { + return new QueueProperties(10, 1000); } - private static QueueProperties windowEvaluatorDefaults() { - QueueProperties properties = new QueueProperties(); - properties.setMaxConcurrentJobs(2); - return properties; + public static QueueProperties windowEvaluatorDefaults() { + return new QueueProperties(2, 0); } } /** * Async executor configuration. */ - @Getter - @Setter - public static class AsyncProperties { - - private boolean enabled = true; - - private int concurrencyLimit = 256; - - private boolean rejectWhenLimitReached = true; + public record AsyncProperties( + @DefaultValue("true") boolean enabled, + @DefaultValue("256") int concurrencyLimit, + @DefaultValue("true") boolean rejectWhenLimitReached, + @DefaultValue("5000") long taskTerminationTimeout) { + + public AsyncProperties() { + this(true, 256, true, 5000L); + } - private long taskTerminationTimeout = 5000L; + public static AsyncProperties defaults() { + return new AsyncProperties(); + } } } diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java index a753dccb2d5..5ae508740d1 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/support/CommonThreadPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("common executor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getCommon(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.common(); return ManagedExecutors.newVirtualExecutor("common-worker", "common-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("common-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("common-long-running", "common-long-running-", diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index a6b32e41da9..e468bfbd192 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -29,31 +29,31 @@ class VirtualThreadPropertiesTest { void defaultsRemainSafeWithoutExternalConfiguration() { VirtualThreadProperties properties = VirtualThreadProperties.defaults(); - assertTrue(properties.isEnabled()); - - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getCollector().getMode()); - assertEquals(512, properties.getCollector().getMaxConcurrentJobs()); - - assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getCommon().getMode()); - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getManager().getMode()); - assertEquals(10, properties.getManager().getMaxConcurrentJobs()); - - assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.getAlerter().getNotify().getMode()); - assertEquals(64, properties.getAlerter().getNotify().getMaxConcurrentJobs()); - assertEquals(10, properties.getAlerter().getPeriodicMaxConcurrentJobs()); - assertEquals(10, properties.getAlerter().getLogWorker().getMaxConcurrentJobs()); - assertEquals(1000, properties.getAlerter().getLogWorker().getQueueCapacity()); - assertEquals(2, properties.getAlerter().getReduce().getMaxConcurrentJobs()); - assertEquals(0, properties.getAlerter().getReduce().getQueueCapacity()); - assertEquals(2, properties.getAlerter().getWindowEvaluator().getMaxConcurrentJobs()); - assertEquals(0, properties.getAlerter().getWindowEvaluator().getQueueCapacity()); - assertEquals(4, properties.getAlerter().getNotifyMaxConcurrentPerChannel()); - - assertEquals(AdmissionMode.UNBOUNDED_VT, properties.getWarehouse().getMode()); - - assertTrue(properties.getAsync().isEnabled()); - assertEquals(256, properties.getAsync().getConcurrencyLimit()); - assertTrue(properties.getAsync().isRejectWhenLimitReached()); - assertEquals(5000L, properties.getAsync().getTaskTerminationTimeout()); + assertTrue(properties.enabled()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.common().mode()); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.manager().mode()); + assertEquals(10, properties.manager().maxConcurrentJobs()); + + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.alerter().notifyPool().mode()); + assertEquals(64, properties.alerter().notifyPool().maxConcurrentJobs()); + assertEquals(10, properties.alerter().periodicMaxConcurrentJobs()); + assertEquals(10, properties.alerter().logWorker().maxConcurrentJobs()); + assertEquals(1000, properties.alerter().logWorker().queueCapacity()); + assertEquals(2, properties.alerter().reduce().maxConcurrentJobs()); + assertEquals(0, properties.alerter().reduce().queueCapacity()); + assertEquals(2, properties.alerter().windowEvaluator().maxConcurrentJobs()); + assertEquals(0, properties.alerter().windowEvaluator().queueCapacity()); + assertEquals(4, properties.alerter().notifyMaxConcurrentPerChannel()); + + assertEquals(AdmissionMode.UNBOUNDED_VT, properties.warehouse().mode()); + + assertTrue(properties.async().enabled()); + assertEquals(256, properties.async().concurrencyLimit()); + assertTrue(properties.async().rejectWhenLimitReached()); + assertEquals(5000L, properties.async().taskTerminationTimeout()); } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java index 063fdf73225..160eab430d5 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/support/CommonThreadPoolTest.java @@ -76,11 +76,14 @@ void testExecuteLongRunningRunsOnPlatformThread() throws Exception { @Test void testExecuteRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties commonProperties = new VirtualThreadProperties.PoolProperties(); - commonProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - commonProperties.setMaxConcurrentJobs(1); - properties.setCommon(commonProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); commonThreadPool = new CommonThreadPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java index 6639e5674c7..80590944272 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/component/status/CalculateStatus.java @@ -308,7 +308,7 @@ private ExecutorService createVirtualExecutor(VirtualThreadProperties virtualThr String errorMessage) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java index baefdcdbeb7..0c516377089 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPool.java @@ -59,16 +59,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("workerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getManager(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.manager(); return ManagedExecutors.newVirtualExecutor("manager-worker", "manager-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("manager-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("manager-long-running", "manager-long-running-", diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java index 7fb26a974d6..275eb4ebf1d 100644 --- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java +++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/scheduler/netty/ManageServer.java @@ -233,7 +233,7 @@ public void onChannelIdle(Channel channel) { private ExecutorService createChannelCheckExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java index 35757bf5afc..b15d192414c 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/scheduler/ManagerWorkerPoolTest.java @@ -61,11 +61,14 @@ void testExecuteJobRunsOnVirtualThread() throws Exception { @Test void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties managerProperties = new VirtualThreadProperties.PoolProperties(); - managerProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - managerProperties.setMaxConcurrentJobs(1); - properties.setManager(managerProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); managerWorkerPool = new ManagerWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java index 28b0074c1d4..5fdfedbb132 100644 --- a/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java +++ b/hertzbeat-startup/src/main/java/org/apache/hertzbeat/startup/AsyncConfig.java @@ -30,12 +30,12 @@ public class AsyncConfig { @Bean(name = "taskExecutor", destroyMethod = "close") public SimpleAsyncTaskExecutor taskExecutor(VirtualThreadProperties virtualThreadProperties) { - VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.getAsync(); + VirtualThreadProperties.AsyncProperties asyncProperties = virtualThreadProperties.async(); SimpleAsyncTaskExecutor executor = new SimpleAsyncTaskExecutor("async-worker-"); - executor.setVirtualThreads(virtualThreadProperties.isEnabled() && asyncProperties.isEnabled()); - executor.setConcurrencyLimit(asyncProperties.getConcurrencyLimit()); - executor.setRejectTasksWhenLimitReached(asyncProperties.isRejectWhenLimitReached()); - executor.setTaskTerminationTimeout(asyncProperties.getTaskTerminationTimeout()); + executor.setVirtualThreads(virtualThreadProperties.enabled() && asyncProperties.enabled()); + executor.setConcurrencyLimit(asyncProperties.concurrencyLimit()); + executor.setRejectTasksWhenLimitReached(asyncProperties.rejectWhenLimitReached()); + executor.setTaskTerminationTimeout(asyncProperties.taskTerminationTimeout()); return executor; } } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java index 00d6e2d1052..da0eaae1105 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/AsyncConfigTest.java @@ -54,9 +54,14 @@ void taskExecutorRunsAsyncTasksOnVirtualThreads() throws Exception { @Test void taskExecutorRejectsWhenConcurrencyLimitReached() throws Exception { - VirtualThreadProperties properties = new VirtualThreadProperties(); - properties.getAsync().setConcurrencyLimit(1); - properties.getAsync().setRejectWhenLimitReached(true); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + new VirtualThreadProperties.AsyncProperties(true, 1, true, 5000L)); try (SimpleAsyncTaskExecutor executor = asyncConfig.taskExecutor(properties)) { CountDownLatch started = new CountDownLatch(1); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java index af4eb5172ee..43c6fbe2a74 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPool.java @@ -60,16 +60,16 @@ private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) log.error("Warehouse workerExecutor has uncaughtException."); log.error(throwable.getMessage(), throwable); }; - if (properties.isEnabled()) { - VirtualThreadProperties.PoolProperties poolProperties = properties.getWarehouse(); + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.warehouse(); return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-", - poolProperties.getMode(), poolProperties.getMaxConcurrentJobs(), handler); + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); } return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler)); } private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { - if (!properties.isEnabled()) { + if (!properties.enabled()) { return fallback; } return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-", diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java index 17ec3b60bfd..01cb7c04291 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/duckdb/DuckdbDatabaseDataStorage.java @@ -437,7 +437,7 @@ void beforeExpiredDataCleanerRun() { private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) { VirtualThreadProperties properties = virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; - if (!properties.isEnabled()) { + if (!properties.enabled()) { return null; } return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java index c014ae24c57..ddcb449bdfe 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/WarehouseWorkerPoolTest.java @@ -78,11 +78,14 @@ void executeJobRunsOnVirtualThread() throws InterruptedException { @Test void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException { - VirtualThreadProperties properties = new VirtualThreadProperties(); - VirtualThreadProperties.PoolProperties warehouseProperties = new VirtualThreadProperties.PoolProperties(); - warehouseProperties.setMode(AdmissionMode.LIMIT_AND_REJECT); - warehouseProperties.setMaxConcurrentJobs(1); - properties.setWarehouse(warehouseProperties); + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.AsyncProperties.defaults()); pool = new WarehouseWorkerPool(properties); CountDownLatch started = new CountDownLatch(1); From f7d0bb3d3d74fca5828ca7d1e043224d7b2cdef1 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 17:16:10 +0800 Subject: [PATCH 07/12] fix: restore record-based virtual thread binding --- .../config/VirtualThreadProperties.java | 83 ++++++++++++++----- .../config/VirtualThreadPropertiesTest.java | 49 +++++++++++ 2 files changed, 112 insertions(+), 20 deletions(-) diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index 8fdbc9c51d1..0962e608ad3 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -19,6 +19,7 @@ import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.ConstructorBinding; import org.springframework.boot.context.properties.bind.DefaultValue; import org.springframework.boot.context.properties.bind.Name; @@ -28,19 +29,24 @@ @ConfigurationProperties(prefix = "hertzbeat.vthreads") public record VirtualThreadProperties( @DefaultValue("true") boolean enabled, - @DefaultValue PoolProperties collector, - @DefaultValue PoolProperties common, - @DefaultValue PoolProperties manager, - @DefaultValue AlerterProperties alerter, - @DefaultValue PoolProperties warehouse, - @DefaultValue AsyncProperties async) { + PoolProperties collector, + PoolProperties common, + PoolProperties manager, + AlerterProperties alerter, + PoolProperties warehouse, + AsyncProperties async) { private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; + private static final int DEFAULT_MANAGER_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS = 64; + private static final int DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL = 4; + @ConstructorBinding public VirtualThreadProperties { - collector = collector == null ? PoolProperties.collectorDefaults() : collector; + collector = normalizePool(collector, PoolProperties.collectorDefaults()); common = common == null ? PoolProperties.commonDefaults() : common; - manager = manager == null ? PoolProperties.managerDefaults() : manager; + manager = normalizePool(manager, PoolProperties.managerDefaults()); alerter = alerter == null ? AlerterProperties.defaults() : alerter; warehouse = warehouse == null ? PoolProperties.warehouseDefaults() : warehouse; async = async == null ? AsyncProperties.defaults() : async; @@ -68,6 +74,7 @@ public record PoolProperties( @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, @DefaultValue("0") int maxConcurrentJobs) { + @ConstructorBinding public PoolProperties { mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; } @@ -89,11 +96,11 @@ public static PoolProperties commonDefaults() { } public static PoolProperties managerDefaults() { - return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 10); + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_MANAGER_MAX_CONCURRENT_JOBS); } public static PoolProperties alerterNotifyDefaults() { - return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 64); + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS); } private static int defaultCollectorConcurrency() { @@ -105,24 +112,29 @@ private static int defaultCollectorConcurrency() { * Alerter-specific executor configuration. */ public record AlerterProperties( - @Name("notify") @DefaultValue PoolProperties notifyPool, + @Name("notify") PoolProperties notifyPool, @DefaultValue("10") int periodicMaxConcurrentJobs, - @DefaultValue QueueProperties logWorker, - @DefaultValue QueueProperties reduce, - @DefaultValue QueueProperties windowEvaluator, + QueueProperties logWorker, + QueueProperties reduce, + QueueProperties windowEvaluator, @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + @ConstructorBinding public AlerterProperties { - notifyPool = notifyPool == null ? PoolProperties.alerterNotifyDefaults() : notifyPool; - logWorker = logWorker == null ? QueueProperties.logWorkerDefaults() : logWorker; - reduce = reduce == null ? QueueProperties.reduceDefaults() : reduce; - windowEvaluator = windowEvaluator == null ? QueueProperties.windowEvaluatorDefaults() : windowEvaluator; + notifyPool = normalizePool(notifyPool, PoolProperties.alerterNotifyDefaults()); + periodicMaxConcurrentJobs = periodicMaxConcurrentJobs <= 0 + ? DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS : periodicMaxConcurrentJobs; + logWorker = normalizeQueue(logWorker, QueueProperties.logWorkerDefaults()); + reduce = normalizeQueue(reduce, QueueProperties.reduceDefaults()); + windowEvaluator = normalizeQueue(windowEvaluator, QueueProperties.windowEvaluatorDefaults()); + notifyMaxConcurrentPerChannel = notifyMaxConcurrentPerChannel <= 0 + ? DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL : notifyMaxConcurrentPerChannel; } public AlerterProperties() { - this(PoolProperties.alerterNotifyDefaults(), 10, + this(PoolProperties.alerterNotifyDefaults(), DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS, QueueProperties.logWorkerDefaults(), QueueProperties.reduceDefaults(), - QueueProperties.windowEvaluatorDefaults(), 4); + QueueProperties.windowEvaluatorDefaults(), DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL); } public static AlerterProperties defaults() { @@ -137,6 +149,10 @@ public record QueueProperties( @DefaultValue("0") int maxConcurrentJobs, @DefaultValue("0") int queueCapacity) { + @ConstructorBinding + public QueueProperties { + } + public QueueProperties() { this(0, 0); } @@ -163,6 +179,10 @@ public record AsyncProperties( @DefaultValue("true") boolean rejectWhenLimitReached, @DefaultValue("5000") long taskTerminationTimeout) { + @ConstructorBinding + public AsyncProperties { + } + public AsyncProperties() { this(true, 256, true, 5000L); } @@ -171,4 +191,27 @@ public static AsyncProperties defaults() { return new AsyncProperties(); } } + + private static PoolProperties normalizePool(PoolProperties configured, PoolProperties defaults) { + if (configured == null) { + return defaults; + } + if (configured.mode() != AdmissionMode.UNBOUNDED_VT && configured.maxConcurrentJobs() <= 0) { + return new PoolProperties(configured.mode(), defaults.maxConcurrentJobs()); + } + return configured; + } + + private static QueueProperties normalizeQueue(QueueProperties configured, QueueProperties defaults) { + if (configured == null) { + return defaults; + } + int maxConcurrentJobs = configured.maxConcurrentJobs() <= 0 + ? defaults.maxConcurrentJobs() : configured.maxConcurrentJobs(); + int queueCapacity = configured.queueCapacity(); + if (queueCapacity <= 0 && defaults.queueCapacity() > 0) { + queueCapacity = defaults.queueCapacity(); + } + return new QueueProperties(maxConcurrentJobs, queueCapacity); + } } diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java index e468bfbd192..3ead4ddca16 100644 --- a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java +++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesTest.java @@ -22,9 +22,14 @@ import org.apache.hertzbeat.common.concurrent.AdmissionMode; import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; class VirtualThreadPropertiesTest { + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(BindingConfig.class); + @Test void defaultsRemainSafeWithoutExternalConfiguration() { VirtualThreadProperties properties = VirtualThreadProperties.defaults(); @@ -56,4 +61,48 @@ void defaultsRemainSafeWithoutExternalConfiguration() { assertTrue(properties.async().rejectWhenLimitReached()); assertEquals(5000L, properties.async().taskTerminationTimeout()); } + + @Test + void collectorModeOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.collector.mode=LIMIT_AND_REJECT") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_REJECT, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + }); + } + + @Test + void collectorModeOverrideUsesConfiguredMode() { + contextRunner.withPropertyValues("hertzbeat.vthreads.collector.mode=LIMIT_AND_BLOCK") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_BLOCK, properties.collector().mode()); + assertEquals(512, properties.collector().maxConcurrentJobs()); + }); + } + + @Test + void notifyModeOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.alerter.notify.mode=LIMIT_AND_BLOCK") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(AdmissionMode.LIMIT_AND_BLOCK, properties.alerter().notifyPool().mode()); + assertEquals(64, properties.alerter().notifyPool().maxConcurrentJobs()); + }); + } + + @Test + void logWorkerQueueOnlyBindingRetainsDefaultConcurrency() { + contextRunner.withPropertyValues("hertzbeat.vthreads.alerter.log-worker.queue-capacity=32") + .run(context -> { + VirtualThreadProperties properties = context.getBean(VirtualThreadProperties.class); + assertEquals(10, properties.alerter().logWorker().maxConcurrentJobs()); + assertEquals(32, properties.alerter().logWorker().queueCapacity()); + }); + } + + @EnableConfigurationProperties(VirtualThreadProperties.class) + static class BindingConfig { + } } From d0306d8b4888728bb962d7b39fc5e256a47f96a0 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 22:08:02 +0800 Subject: [PATCH 08/12] refactor: update parameter handling and imports across multiple files --- .../ai/tools/impl/MonitorToolsImpl.java | 10 +- .../dispatch/entrance/CollectServerTest.java | 4 +- .../hertzbeat-collector-common/pom.xml | 4 - .../dispatch/entrance/CollectServer.java | 45 ++++- .../entrance/processor/GoCloseProcessor.java | 17 +- .../processor/GoOfflineProcessor.java | 12 +- .../entrance/processor/GoOnlineProcessor.java | 10 +- .../collector/timer/TimerDispatcher.java | 29 ++- .../collector/timer/WheelTimerTask.java | 14 +- .../processor/GoOnlineProcessorTest.java | 43 ++--- .../collect/kafka/KafkaCollectImpl.java | 19 +- .../mongodb/MongodbSingleCollectImpl.java | 19 +- .../collect/nebulagraph/NgqlCollectImpl.java | 25 +-- .../rocketmq/RocketmqSingleCollectImpl.java | 21 ++- hertzbeat-common-core/pom.xml | 1 + .../concurrent/BackgroundTaskExecutor.java | 48 +++++ .../common/config/BaseKafkaProperties.java | 0 .../config/VirtualThreadProperties.java | 42 ++--- .../common/entity/dto/sms/SmsConfig.java | 6 +- .../hertzbeat/common/entity/job/Job.java | 63 +++---- .../hertzbeat/common/entity/job/Metrics.java | 0 .../common/entity/job/RuntimeParamDefine.java | 60 ++++-- .../common/entity/job/SshTunnel.java | 4 +- .../common/entity/plugin/PluginConfig.java | 9 +- .../common/entity/push/PushMetricsDto.java | 0 .../support/ResourceBundleUtf8Control.java | 0 .../exception/AlertExpressionException.java | 2 +- .../CommonDataQueueUnknownException.java | 2 +- .../support/exception/CommonException.java | 0 .../exception/ExpressionVisitorException.java | 2 +- .../support/exception/IgnoreException.java | 0 .../exception/SendMessageException.java | 0 .../support/valid/SqlSecurityException.java | 0 .../common/util/ResourceBundleUtil.java | 0 hertzbeat-common-spring/pom.xml | 1 + .../hertzbeat/common/config/CommonConfig.java | 8 +- .../common/config/SmsConfigBinding.java | 28 +++ .../VirtualThreadPropertiesBinding.java | 177 ++++++++++++++++++ .../common/entity/job/SshTunnel.java | 87 --------- .../common/support/CommonThreadPool.java | 3 +- .../hertzbeat/common/util/IpDomainUtil.java | 169 ----------------- .../common/config/SmsConfigBindingTest.java | 50 +++++ .../config/VirtualThreadPropertiesTest.java | 8 +- .../component/validator/ParamValidator.java | 6 +- .../validator/ParamValidatorManager.java | 6 +- .../validator/impl/ArrayParamValidator.java | 6 +- .../validator/impl/BooleanParamValidator.java | 6 +- .../impl/HostParamValidatorAdapter.java | 6 +- .../validator/impl/JsonParamValidator.java | 6 +- .../validator/impl/NumberParamValidator.java | 6 +- .../validator/impl/OptionParamValidator.java | 10 +- .../impl/PasswordParamValidator.java | 6 +- .../validator/impl/TextParamValidator.java | 6 +- .../manager/controller/AppController.java | 4 +- .../controller/CollectorController.java | 2 +- .../controller/MonitorsController.java | 14 +- .../manager/controller/PluginController.java | 2 +- .../controller/StatusPageController.java | 41 ++-- .../StatusPagePublicController.java | 12 +- .../manager/pojo/dto/CollectorInfo.java | 82 ++++++++ .../manager/pojo}/dto/CollectorSummary.java | 13 +- .../manager/pojo/dto/ComponentStatus.java | 63 +++++-- .../manager/pojo/dto/MonitorDto.java | 93 +++++++-- .../manager/pojo/dto/MonitorInfo.java | 134 +++++++++++++ .../manager/pojo/dto/MonitorParam.java | 81 ++++++++ .../manager/pojo/dto/ParamDefineDto.java | 3 +- .../manager/pojo/dto/ParamDefineInfo.java | 144 ++++++++++++++ .../manager/pojo/dto/PluginParametersVO.java | 5 +- .../manager/pojo}/dto/PluginUpload.java | 5 +- .../pojo/dto/StatusPageComponentInfo.java | 95 ++++++++++ .../pojo/dto/StatusPageHistoryInfo.java | 94 ++++++++++ .../dto/StatusPageIncidentContentInfo.java | 84 +++++++++ .../pojo/dto/StatusPageIncidentInfo.java | 115 ++++++++++++ .../manager/pojo/dto/StatusPageOrgInfo.java | 99 ++++++++++ .../scheduler/CollectorJobScheduler.java | 4 +- .../manager/scheduler/SchedulerInit.java | 4 +- .../manager/scheduler/netty/ManageServer.java | 24 ++- .../CollectCyclicDataResponseProcessor.java | 9 +- ...ServiceDiscoveryDataResponseProcessor.java | 9 +- .../hertzbeat/manager/service/AppService.java | 4 +- .../manager/service/CollectorService.java | 2 +- .../manager/service/PluginService.java | 2 +- .../manager/service/StatusPageService.java | 26 +-- .../manager/service/impl/AppServiceImpl.java | 47 +++-- .../service/impl/CollectorServiceImpl.java | 6 +- .../service/impl/MonitorServiceImpl.java | 26 +-- .../service/impl/PluginServiceImpl.java | 6 +- .../service/impl/StatusPageServiceImpl.java | 66 ++++--- .../validator/ParamValidatorManagerTest.java | 12 +- .../impl/ArrayParamValidatorTest.java | 16 +- .../impl/BooleanParamValidatorTest.java | 20 +- .../impl/HostParamValidatorAdapterTest.java | 24 +-- .../impl/JsonParamValidatorTest.java | 12 +- .../impl/NumberParamValidatorTest.java | 20 +- .../impl/OptionParamValidatorTest.java | 20 +- .../impl/PasswordParamValidatorTest.java | 12 +- .../impl/TextParamValidatorTest.java | 12 +- .../manager/controller/AppControllerTest.java | 6 +- .../controller/MonitorsControllerTest.java | 25 ++- .../controller/StatusPageControllerTest.java | 45 ++--- .../StatusPagePublicControllerTest.java | 4 +- .../scheduler/netty/ManageServerTest.java | 10 +- .../manager/service/AppServiceTest.java | 11 +- .../manager/service/MonitorServiceTest.java | 101 ++++------ .../manager/service/PluginServiceTest.java | 2 +- .../service/StatusPageServiceTest.java | 29 +-- hertzbeat-push/pom.xml | 5 + hertzbeat-remoting/pom.xml | 2 +- .../remoting/netty/NettyRemotingClient.java | 8 +- .../remoting/netty/NettyRemotingServer.java | 6 +- .../remoting/RemotingServiceTest.java | 23 ++- 111 files changed, 2102 insertions(+), 849 deletions(-) create mode 100644 hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java (84%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java (89%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java (93%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java (100%) rename hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java => hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java (56%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/plugin/PluginConfig.java (84%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/entity/push/PushMetricsDto.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/ResourceBundleUtf8Control.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/AlertExpressionException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/CommonDataQueueUnknownException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/CommonException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/ExpressionVisitorException.java (99%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/IgnoreException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/exception/SendMessageException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/support/valid/SqlSecurityException.java (100%) rename {hertzbeat-common-spring => hertzbeat-common-core}/src/main/java/org/apache/hertzbeat/common/util/ResourceBundleUtil.java (100%) create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/SmsConfigBinding.java create mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadPropertiesBinding.java delete mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/SshTunnel.java delete mode 100644 hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/util/IpDomainUtil.java create mode 100644 hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/config/SmsConfigBindingTest.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/CollectorInfo.java rename {hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity => hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo}/dto/CollectorSummary.java (88%) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/MonitorInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/MonitorParam.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ParamDefineInfo.java rename {hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity => hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo}/dto/PluginUpload.java (93%) create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageComponentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageHistoryInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageIncidentContentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageIncidentInfo.java create mode 100644 hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/StatusPageOrgInfo.java diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java index 7495cf83eb9..41080c79851 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.ai.config.McpContextHolder; import org.apache.hertzbeat.manager.pojo.dto.MonitorDto; +import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo; import org.apache.hertzbeat.manager.service.MonitorService; import org.apache.hertzbeat.manager.service.AppService; import org.apache.hertzbeat.ai.utils.UtilityClass; @@ -32,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.Param; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import java.util.ArrayList; import java.util.List; @@ -280,7 +280,9 @@ public String addMonitor( // Validate that all required parameters for this monitor type are provided try { - MonitorDto monitorDto = MonitorDto.builder().monitor(monitor).params(paramList).build(); + MonitorDto monitorDto = new MonitorDto(); + monitorDto.setMonitor(monitor); + monitorDto.setParams(paramList); monitorService.validate(monitorDto, false); } catch (IllegalArgumentException argumentException) { if (argumentException.getMessage().contains("required")) { @@ -456,7 +458,7 @@ public String getMonitorParams( } // Get parameter definitions from app service - List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); + List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); if (paramDefines == null || paramDefines.isEmpty()) { return String.format("No parameter definitions found for monitor type '%s'. " @@ -468,7 +470,7 @@ public String getMonitorParams( response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n", app, paramDefines.size())); - for (ParamDefine paramDefine : paramDefines) { + for (ParamDefineInfo paramDefine : paramDefines) { response.append("• Field: ").append(paramDefine.getField()).append("\n"); // Add display name if available diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java index c56d53e948f..03a6977da63 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java @@ -36,9 +36,9 @@ import org.apache.hertzbeat.collector.dispatch.DispatchProperties; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -70,7 +70,7 @@ class CollectServerTest { private DispatchProperties.EntranceProperties.NettyProperties nettyProperties; @Mock - private CommonThreadPool threadPool; + private BackgroundTaskExecutor threadPool; @Mock private CollectorInfoProperties infoProperties; diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index b9a80815d3a..d4cffe23fb7 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -51,10 +51,6 @@ org.apache.hertzbeat hertzbeat-common-core - - org.apache.hertzbeat - hertzbeat-common-spring - org.apache.sshd diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 1e77da0e211..a5bf494750f 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -32,16 +32,18 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProcessor; import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.RemotingClient; import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; @@ -81,20 +83,33 @@ public class CollectServer implements CommandLineRunner { private boolean heartbeatPending; + private final Runnable closeApplicationAction; + public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, - final CommonThreadPool threadPool, + final BackgroundTaskExecutor threadPool, final CollectorInfoProperties infoProperties) { - this(collectJobService, timerDispatch, properties, threadPool, infoProperties, VirtualThreadProperties.defaults()); + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, + VirtualThreadProperties.defaults()); + } + + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final BackgroundTaskExecutor threadPool, + final CollectorInfoProperties infoProperties, + final VirtualThreadProperties virtualThreadProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, virtualThreadProperties); } @Autowired public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, - final CommonThreadPool threadPool, + final BackgroundTaskExecutor threadPool, final CollectorInfoProperties infoProperties, + final ConfigurableApplicationContext applicationContext, final VirtualThreadProperties virtualThreadProperties) { if (properties == null || properties.getEntrance() == null || properties.getEntrance().getNetty() == null) { log.error("init error, please config dispatch entrance netty props in application.yml"); @@ -109,10 +124,11 @@ public CollectServer(final CollectJobService collectJobService, this.collectJobService.setCollectServer(this); this.infoProperties = infoProperties; this.heartbeatExecutor = createHeartbeatExecutor(virtualThreadProperties); + this.closeApplicationAction = createCloseApplicationAction(applicationContext); this.init(properties, threadPool); } - private void init(final DispatchProperties properties, final CommonThreadPool threadPool) { + private void init(final DispatchProperties properties, final BackgroundTaskExecutor threadPool) { NettyClientConfig nettyClientConfig = new NettyClientConfig(); DispatchProperties.EntranceProperties.NettyProperties nettyProperties = properties.getEntrance().getNetty(); nettyClientConfig.setServerHost(nettyProperties.getManagerHost()); @@ -123,9 +139,10 @@ private void init(final DispatchProperties properties, final CommonThreadPool th this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this)); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, + new GoCloseProcessor(this, timerDispatch, closeApplicationAction)); } public void shutdown() { @@ -279,4 +296,16 @@ private void sendHeartbeat(String identity) { log.error("schedule send heartbeat to server error.{}", e.getMessage()); } } + + private Runnable createCloseApplicationAction(ConfigurableApplicationContext applicationContext) { + if (applicationContext == null) { + return () -> {}; + } + return () -> { + SpringApplication.exit(applicationContext, () -> 0); + if (applicationContext.isActive()) { + applicationContext.close(); + } + }; + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java index 4b068f1e45f..b259af80313 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java @@ -23,9 +23,7 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; -import org.springframework.boot.SpringApplication; /** * handle collector close message @@ -34,24 +32,25 @@ @Slf4j public class GoCloseProcessor implements NettyRemotingProcessor { private final CollectServer collectServer; - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + private final Runnable closeApplicationAction; - public GoCloseProcessor(final CollectServer collectServer) { + public GoCloseProcessor(final CollectServer collectServer, + final TimerDispatch timerDispatch, + final Runnable closeApplicationAction) { this.collectServer = collectServer; + this.timerDispatch = timerDispatch; + this.closeApplicationAction = closeApplicationAction; } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg()); } this.timerDispatch.goOffline(); this.collectServer.shutdown(); - SpringApplication.exit(SpringContextHolder.getApplicationContext(), () -> 0); - SpringContextHolder.shutdown(); + closeApplicationAction.run(); log.info("receive offline message and close success"); return null; } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java index aad85194ee4..ccacf943e53 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java @@ -23,7 +23,6 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; /** @@ -32,14 +31,15 @@ */ @Slf4j public class GoOfflineProcessor implements NettyRemotingProcessor { - - private TimerDispatch timerDispatch; + + private final TimerDispatch timerDispatch; + + public GoOfflineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } timerDispatch.goOffline(); log.info("receive offline message and handle success"); if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java index acebb42e3ad..e35c66a151b 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java @@ -24,7 +24,6 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.dto.ServerInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.AesUtil; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; @@ -36,13 +35,14 @@ @Slf4j public class GoOnlineProcessor implements NettyRemotingProcessor { - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + + public GoOnlineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().isEmpty()) { log.warn("The message that server response to collector is empty, please upgrade server"); } else { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java index 4aa6e4c0250..a8d6a600021 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java @@ -25,9 +25,11 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.constants.ScheduleTypeEnum; +import org.apache.hertzbeat.collector.dispatch.MetricsTaskDispatch; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectResponseEventListener; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; @@ -35,6 +37,8 @@ import org.apache.hertzbeat.common.timer.HashedWheelTimer; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.Timer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.DisposableBean; import org.springframework.scheduling.support.CronExpression; import org.springframework.stereotype.Component; @@ -69,7 +73,19 @@ public class TimerDispatcher implements TimerDispatch, DisposableBean { */ private final AtomicBoolean started; + private final Supplier metricsTaskDispatchSupplier; + public TimerDispatcher() { + this(() -> timeout -> { + }); + } + + @Autowired + public TimerDispatcher(ObjectProvider metricsTaskDispatchProvider) { + this(resolveMetricsTaskDispatchSupplier(metricsTaskDispatchProvider)); + } + + private TimerDispatcher(Supplier metricsTaskDispatchSupplier) { this.wheelTimer = new HashedWheelTimer(r -> { Thread ret = new Thread(r, "wheelTimer"); ret.setDaemon(true); @@ -79,6 +95,16 @@ public TimerDispatcher() { this.currentTempTaskMap = new ConcurrentHashMap<>(8); this.eventListeners = new ConcurrentHashMap<>(8); this.started = new AtomicBoolean(true); + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; + } + + private static Supplier resolveMetricsTaskDispatchSupplier( + ObjectProvider metricsTaskDispatchProvider) { + if (metricsTaskDispatchProvider == null) { + return () -> timeout -> { + }; + } + return metricsTaskDispatchProvider::getObject; } @Override @@ -87,7 +113,8 @@ public void addJob(Job addJob, CollectResponseEventListener eventListener) { log.warn("Collector is offline, can not dispatch collect jobs."); return; } - WheelTimerTask timerJob = new WheelTimerTask(addJob); + // Delay dispatcher lookup to avoid a startup cycle with CommonDispatcher. + WheelTimerTask timerJob = new WheelTimerTask(addJob, metricsTaskDispatchSupplier); if (addJob.isCyclic()) { Long nextExecutionTime = getNextExecutionInterval(addJob); Timeout timeout = wheelTimer.newTimeout(timerJob, nextExecutionTime, TimeUnit.SECONDS); diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java index 5dfe1922a57..30d97ba2e11 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java @@ -27,7 +27,6 @@ import org.apache.hertzbeat.common.entity.job.Configmap; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.TimerTask; import org.apache.hertzbeat.common.util.AesUtil; @@ -35,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -44,11 +44,15 @@ public class WheelTimerTask implements TimerTask { private final Job job; - private final MetricsTaskDispatch metricsTaskDispatch; + private final Supplier metricsTaskDispatchSupplier; private static final Gson GSON = new Gson(); - public WheelTimerTask(Job job) { - this.metricsTaskDispatch = SpringContextHolder.getBean(MetricsTaskDispatch.class); + public WheelTimerTask(Job job, MetricsTaskDispatch metricsTaskDispatch) { + this(job, () -> metricsTaskDispatch); + } + + public WheelTimerTask(Job job, Supplier metricsTaskDispatchSupplier) { + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; this.job = job; // The initialization job will monitor the actual parameter value and replace the collection field initJobMetrics(job); @@ -93,7 +97,7 @@ private void initJobMetrics(Job job) { @Override public void run(Timeout timeout) throws Exception { job.setDispatchTime(System.currentTimeMillis()); - metricsTaskDispatch.dispatchMetricsTask(timeout); + metricsTaskDispatchSupplier.get().dispatchMetricsTask(timeout); } public Job getJob() { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java index d43de3e97d2..b544c8c39bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java @@ -20,19 +20,15 @@ import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import io.netty.channel.ChannelHandlerContext; -import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.collector.timer.TimerDispatcher; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.JsonUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.lang.reflect.Field; @@ -51,32 +47,27 @@ class GoOnlineProcessorTest { @Mock private ChannelHandlerContext channelHandlerContext; - private MockedStatic springContextHolderMockedStatic; - @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - goOnlineProcessor = new GoOnlineProcessor(); timerDispatcher = new TimerDispatcher(); - springContextHolderMockedStatic = Mockito.mockStatic(SpringContextHolder.class); - springContextHolderMockedStatic.when(() -> SpringContextHolder.getBean(TimerDispatch.class)).thenReturn(timerDispatcher); + goOnlineProcessor = new GoOnlineProcessor(timerDispatcher); } @AfterEach void tearDown() throws Exception { - springContextHolderMockedStatic.close(); timerDispatcher.destroy(); } @Test void verifyTaskMapPreservation() throws Exception { Job job = Job.builder() - .app("test") - .id(12345L) - .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) - .configmap(Lists.newArrayList()) - .isCyclic(true) - .build(); + .app("test") + .id(12345L) + .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) + .configmap(Lists.newArrayList()) + .isCyclic(true) + .build(); timerDispatcher.addJob(job, null); Field cyclicTaskMapField = TimerDispatcher.class.getDeclaredField("currentCyclicTaskMap"); @@ -85,20 +76,20 @@ void verifyTaskMapPreservation() throws Exception { assertEquals(1, currentCyclicTaskMap.size(), "Task map should have 1 job initially"); ClusterMsg.Message responseMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.RESPONSE) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.RESPONSE) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, responseMsg); assertEquals(1, currentCyclicTaskMap.size(), "Task map should still have 1 job after receiving RESPONSE"); ClusterMsg.Message requestMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.REQUEST) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.REQUEST) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, requestMsg); assertEquals(0, currentCyclicTaskMap.size(), "Task map should be empty after receiving REQUEST"); } diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java index fabdf973628..b4f91304f16 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java @@ -46,7 +46,6 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; -import org.springframework.util.Assert; import java.util.Collection; import java.util.Collections; @@ -223,14 +222,14 @@ private static boolean filterInternalTopics(String topic, Boolean monitorInterna @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null, "Metrics cannot be null"); + require(metrics != null, "Metrics cannot be null"); KafkaProtocol kafkaProtocol = metrics.getKclient(); // Ensure that metrics and kafkaProtocol are not null - Assert.isTrue(metrics != null && kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); + require(kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); // Ensure that host and port are not empty - Assert.hasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); - Assert.hasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); + requireHasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); + requireHasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); } @Override @@ -384,4 +383,14 @@ private long getLatestOffset(AdminClient adminClient, TopicPartition topicPartit public String supportProtocol() { return DispatchConstants.PROTOCOL_KAFKA; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java index b99304e60e4..0cf3107c580 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java @@ -43,7 +43,6 @@ import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.CommonUtil; import org.bson.Document; -import org.springframework.util.Assert; /** * Mongodb single collect @@ -87,11 +86,11 @@ public class MongodbSingleCollectImpl extends AbstractCollect { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException{ - Assert.isTrue(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); + require(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); MongodbProtocol mongodbProtocol = metrics.getMongodb(); - Assert.hasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); - Assert.hasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); - Assert.hasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); + requireHasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); + requireHasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); + requireHasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); } @Override @@ -226,4 +225,14 @@ private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) { connectionCommonCache.addCache(identifier, mongodbConnect, 3600 * 1000L); return mongoClient; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java index 33a361aa0bf..4034c1e15bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.collector.collect.AbstractCollect; @@ -33,8 +34,6 @@ import org.apache.hertzbeat.common.entity.job.protocol.NgqlProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.entity.message.CollectRep.MetricsData.Builder; -import org.springframework.util.Assert; -import org.springframework.util.StopWatch; /** * connect nebulaGraph and collect metrics use NGQL @@ -53,18 +52,17 @@ public class NgqlCollectImpl extends AbstractCollect { @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { NgqlProtocol ngql = metrics.getNgql(); - Assert.hasText(ngql.getHost(), "NGQL protocol host is required"); - Assert.hasText(ngql.getPort(), "Port protocol host is required"); - Assert.hasText(ngql.getParseType(), "NGQL protocol parseType is required"); - Assert.hasText(ngql.getUsername(), "NGQL protocol username is required"); - Assert.hasText(ngql.getPassword(), "NGQL protocol password is required"); + requireHasText(ngql.getHost(), "NGQL protocol host is required"); + requireHasText(ngql.getPort(), "Port protocol host is required"); + requireHasText(ngql.getParseType(), "NGQL protocol parseType is required"); + requireHasText(ngql.getUsername(), "NGQL protocol username is required"); + requireHasText(ngql.getPassword(), "NGQL protocol password is required"); } @Override public void collect(Builder builder, Metrics metrics) { NgqlProtocol ngql = metrics.getNgql(); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); + long startTimeNanos = System.nanoTime(); NebulaTemplate nebulaTemplate = new NebulaTemplate(); try { boolean initSuccess = nebulaTemplate.initSession(ngql); @@ -79,8 +77,7 @@ public void collect(Builder builder, Metrics metrics) { return; } - stopWatch.stop(); - long responseTime = stopWatch.getTotalTimeMillis(); + long responseTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); try { switch (ngql.getParseType()) { case PARSE_TYPE_FILTER_COUNT -> filterCount(nebulaTemplate, ngql, metrics.getAliasFields(), builder, responseTime); @@ -247,4 +244,10 @@ private Map showJobs(NebulaTemplate template, String protocolSpa result.put("running_jobs", String.valueOf(jobs.stream().filter(job -> Objects.equals(job.get("Status"), STATUS_RUNNING)).count())); return result; } + + private static void requireHasText(String value, String message) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException(message); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java index a8077ca6091..a98ff1ada44 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java @@ -54,14 +54,12 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.util.Assert; /** * rocketmq collect */ @Slf4j -public class RocketmqSingleCollectImpl extends AbstractCollect implements DisposableBean { +public class RocketmqSingleCollectImpl extends AbstractCollect { private static final int WAIT_TIMEOUT = 10; static final int QUEUE_CAPACITY = 5000; @@ -102,7 +100,6 @@ private static ManagedExecutor createExecutor() { corePoolSize, maximumPoolSize, QUEUE_CAPACITY, handler); } - @Override public void destroy() { this.executorService.close(); } @@ -113,10 +110,10 @@ public void destroy() { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); + require(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); RocketmqProtocol rocketmq = metrics.getRocketmq(); - Assert.hasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); - Assert.hasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); + requireHasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); + requireHasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); } @Override @@ -374,4 +371,14 @@ private void fillBuilder(RocketmqCollectData rocketmqCollectData, CollectRep.Met void executeConsumerTask(Runnable runnable) { executorService.execute(runnable); } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-common-core/pom.xml b/hertzbeat-common-core/pom.xml index 8cccffe102a..949cf19ed53 100644 --- a/hertzbeat-common-core/pom.xml +++ b/hertzbeat-common-core/pom.xml @@ -28,6 +28,7 @@ hertzbeat-common-core ${project.artifactId} + Framework-agnostic shared runtime models, utilities, and protocol support. diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java new file mode 100644 index 00000000000..030800faff5 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.RejectedExecutionException; + +/** + * Generic background task executor abstraction for runtime components. + */ +public interface BackgroundTaskExecutor { + + /** + * Execute a short-lived task. + * + * @param runnable task + * @throws RejectedExecutionException when execution is rejected + */ + void execute(Runnable runnable) throws RejectedExecutionException; + + /** + * Execute a long-lived background task. + * + * @param runnable task + */ + void executeLongRunning(Runnable runnable); + + /** + * Release executor resources. + * + * @throws Exception close exception + */ + void destroy() throws Exception; +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java similarity index 84% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java index 0962e608ad3..956b8a54140 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -18,17 +18,12 @@ package org.apache.hertzbeat.common.config; import org.apache.hertzbeat.common.concurrent.AdmissionMode; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.bind.ConstructorBinding; -import org.springframework.boot.context.properties.bind.DefaultValue; -import org.springframework.boot.context.properties.bind.Name; /** - * Virtual-thread related configuration. + * Framework-agnostic virtual-thread runtime configuration. */ -@ConfigurationProperties(prefix = "hertzbeat.vthreads") public record VirtualThreadProperties( - @DefaultValue("true") boolean enabled, + boolean enabled, PoolProperties collector, PoolProperties common, PoolProperties manager, @@ -42,7 +37,6 @@ public record VirtualThreadProperties( private static final int DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS = 10; private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL = 4; - @ConstructorBinding public VirtualThreadProperties { collector = normalizePool(collector, PoolProperties.collectorDefaults()); common = common == null ? PoolProperties.commonDefaults() : common; @@ -71,10 +65,9 @@ public static VirtualThreadProperties defaults() { * Pool-level configuration. */ public record PoolProperties( - @DefaultValue("UNBOUNDED_VT") AdmissionMode mode, - @DefaultValue("0") int maxConcurrentJobs) { + AdmissionMode mode, + int maxConcurrentJobs) { - @ConstructorBinding public PoolProperties { mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; } @@ -112,14 +105,13 @@ private static int defaultCollectorConcurrency() { * Alerter-specific executor configuration. */ public record AlerterProperties( - @Name("notify") PoolProperties notifyPool, - @DefaultValue("10") int periodicMaxConcurrentJobs, + PoolProperties notifyPool, + int periodicMaxConcurrentJobs, QueueProperties logWorker, QueueProperties reduce, QueueProperties windowEvaluator, - @DefaultValue("4") int notifyMaxConcurrentPerChannel) { + int notifyMaxConcurrentPerChannel) { - @ConstructorBinding public AlerterProperties { notifyPool = normalizePool(notifyPool, PoolProperties.alerterNotifyDefaults()); periodicMaxConcurrentJobs = periodicMaxConcurrentJobs <= 0 @@ -146,12 +138,8 @@ public static AlerterProperties defaults() { * Queue-preserving executor configuration. */ public record QueueProperties( - @DefaultValue("0") int maxConcurrentJobs, - @DefaultValue("0") int queueCapacity) { - - @ConstructorBinding - public QueueProperties { - } + int maxConcurrentJobs, + int queueCapacity) { public QueueProperties() { this(0, 0); @@ -174,14 +162,10 @@ public static QueueProperties windowEvaluatorDefaults() { * Async executor configuration. */ public record AsyncProperties( - @DefaultValue("true") boolean enabled, - @DefaultValue("256") int concurrencyLimit, - @DefaultValue("true") boolean rejectWhenLimitReached, - @DefaultValue("5000") long taskTerminationTimeout) { - - @ConstructorBinding - public AsyncProperties { - } + boolean enabled, + int concurrencyLimit, + boolean rejectWhenLimitReached, + long taskTerminationTimeout) { public AsyncProperties() { this(true, 256, true, 5000L); diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java similarity index 89% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java index 2b2c3a5f6ae..c689b913949 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java @@ -21,17 +21,13 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; /** - * SMS configuration + * Framework-agnostic SMS runtime configuration. */ @Data @AllArgsConstructor @NoArgsConstructor -@Component -@ConfigurationProperties(prefix = "alerter.sms") public class SmsConfig { /** diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java similarity index 93% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java index 4b4e1d53265..e7fb6e52c1f 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.common.entity.job; - import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Collections; import java.util.Comparator; @@ -35,13 +34,11 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; -import org.springframework.util.CollectionUtils; /** - * Collect task details + * Collect task details. */ @Data @AllArgsConstructor @@ -51,15 +48,15 @@ public class Job { /** - * Task Job id + * Task Job id. */ private long id; /** - * Tenant id + * Tenant id. */ private long tenantId = 0; /** - * Monitoring Task ID + * Monitoring Task ID. */ private long monitorId; /** @@ -68,11 +65,11 @@ public class Job { */ private Map metadata; /** - * bind labels + * bind labels. */ private Map labels; /** - * bind annotations + * bind annotations. */ private Map annotations; /** @@ -86,12 +83,12 @@ public class Job { */ private String category; /** - * Type of monitoring eg: linux | mysql | jvm + * Type of monitoring eg: linux | mysql | jvm. */ private String app; /** * The internationalized name of the monitoring type - * PING CONNECT + * PING CONNECT. */ private Map name; /** @@ -101,55 +98,54 @@ public class Job { */ private Map help; /** - * The monitor help link + * The monitor help link. */ private Map helpLink; /** - * Task dispatch start timestamp + * Task dispatch start timestamp. */ private long timestamp; /** - * Default task collection time interval (unit: second) eg: 30,60,600 + * Default task collection time interval (unit: second) eg: 30,60,600. */ private long defaultInterval = 600L; /** - * Refresh time list for one cycle of the job + * Refresh time list for one cycle of the job. */ private ConcurrentLinkedDeque intervals; /** - * Whether it is a recurring periodic task true is yes, false is no + * Whether it is a recurring periodic task true is yes, false is no. */ private boolean isCyclic = false; /** - * monitor input need params + * monitor input need params. */ - private List params; + private List params; /** - * Metrics configuration eg: cpu memory - * eg: cpu memory + * Metrics configuration eg: cpu memory. */ private List metrics; /** - * Monitoring configuration parameter properties and values eg: username password timeout host + * Monitoring configuration parameter properties and values eg: username password timeout host. */ private List configmap; /** - * Whether it is a service discovery job, true is yes, false is no + * Whether it is a service discovery job, true is yes, false is no. */ private boolean isSd = false; /** - * Whether to use the Prometheus proxy + * Whether to use the Prometheus proxy. */ private boolean prometheusProxyMode = false; /** - * Scheduling type: interval or cron + * Scheduling type: interval or cron. */ private String scheduleType = "interval"; /** - * Cron expression for scheduling, used when scheduleType is "cron" + * Cron expression for scheduling, used when scheduleType is "cron". */ private String cronExpression = null; @@ -160,7 +156,7 @@ public class Job { private Map envConfigmaps; /** - * collector use - timestamp when the task was scheduled by the time wheel + * collector use - timestamp when the task was scheduled by the time wheel. */ @JsonIgnore private transient long dispatchTime; @@ -179,13 +175,13 @@ public class Job { private transient LinkedList> priorMetrics; /** - * collector use - Temporarily store one-time task metrics response data + * collector use - Temporarily store one-time task metrics response data. */ @JsonIgnore private transient List responseDataTemp; /** - * collector use - construct to initialize metrics execution view + * collector use - construct to initialize metrics execution view. */ public synchronized void constructPriorMetrics() { long now = System.currentTimeMillis(); @@ -232,7 +228,7 @@ public synchronized void constructPriorMetrics() { } /** - * collector use - to get the next set of priority metric group tasks + * collector use - to get the next set of priority metric group tasks. * * @param metrics Current Metrics * @param first Is it the first time to get @@ -314,7 +310,7 @@ public void initIntervals() { } /** - * The greatest common divisor + * The greatest common divisor. */ public static long gcd(long a, long b) { while (b != 0) { @@ -326,7 +322,7 @@ public static long gcd(long a, long b) { } /** - * The least common multiple + * The least common multiple. */ public static long lcm(List array) { if (array != null && !array.isEmpty()) { @@ -340,9 +336,8 @@ public static long lcm(List array) { } /** - * * @param metricsIntervals A unique list composed of intervals for all metrics - * Generate a list of refresh intervals for metric collection + * Generate a list of refresh intervals for metric collection. */ public synchronized void generateMetricsIntervals(List metricsIntervals) { // 1. To find the least common multiple (LCM) of all metric refresh intervals @@ -368,7 +363,7 @@ public synchronized void generateMetricsIntervals(List metricsIntervals) { } public synchronized long getInterval() { - if (!CollectionUtils.isEmpty(this.intervals)) { + if (this.intervals != null && !this.intervals.isEmpty()) { Long interval = this.intervals.removeFirst(); if (interval != null) { this.intervals.addLast(interval); diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java similarity index 56% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java index f93093157d2..eae3e24e286 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java @@ -17,38 +17,60 @@ package org.apache.hertzbeat.common.entity.job; -import java.io.Serializable; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; /** - * Monitoring configuration parameter properties and values - * During the process, you need to replace the content with the identifier ^_^key^_^ - * in the protocol configuration parameter with the real value in the configuration parameter + * Framework-agnostic parameter definition used by runtime templates and jobs. */ @Data +@Builder @AllArgsConstructor @NoArgsConstructor -@Builder -public class Configmap implements Serializable { +public class RuntimeParamDefine { - /** - * Parameter key, replace the content with the identifier ^^_key_^^ in the protocol - * configuration parameter with the real value in the configuration parameter - */ - private String key; + private String app; - /** - * parameter value - */ - private Object value; + private Map name; + + private String field; + + private String type; + + private boolean required = false; + + private String defaultValue; + + private String placeholder; + + private String range; + + private Short limit; + + private List + + + org.apache.hertzbeat + hertzbeat-common-spring + org.springframework.boot diff --git a/hertzbeat-remoting/pom.xml b/hertzbeat-remoting/pom.xml index 1fddaf8853d..c9d4f53d37e 100644 --- a/hertzbeat-remoting/pom.xml +++ b/hertzbeat-remoting/pom.xml @@ -35,7 +35,7 @@ org.apache.hertzbeat - hertzbeat-common-spring + hertzbeat-common-core diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java index 7863e830e31..2001fd912fd 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingClient.java @@ -36,9 +36,9 @@ import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; import java.util.concurrent.ThreadFactory; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; import org.apache.hertzbeat.remoting.event.NettyEventListener; @@ -51,10 +51,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { private static final int DEFAULT_WORKER_THREAD_NUM = Math.min(4, Runtime.getRuntime().availableProcessors()); - + private final NettyClientConfig nettyClientConfig; - private final CommonThreadPool threadPool; + private final BackgroundTaskExecutor threadPool; private final Bootstrap bootstrap = new Bootstrap(); @@ -64,7 +64,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti public NettyRemotingClient(final NettyClientConfig nettyClientConfig, final NettyEventListener nettyEventListener, - final CommonThreadPool threadPool) { + final BackgroundTaskExecutor threadPool) { super(nettyEventListener); this.nettyClientConfig = nettyClientConfig; this.threadPool = threadPool; diff --git a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java index 169f38220f3..f6e743bda5e 100644 --- a/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java +++ b/hertzbeat-remoting/src/main/java/org/apache/hertzbeat/remoting/netty/NettyRemotingServer.java @@ -43,9 +43,9 @@ import io.netty.handler.timeout.IdleStateHandler; import java.util.List; import java.util.concurrent.ThreadFactory; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingServer; import org.apache.hertzbeat.remoting.event.NettyEventListener; @@ -59,7 +59,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti private final NettyServerConfig nettyServerConfig; - private final CommonThreadPool threadPool; + private final BackgroundTaskExecutor threadPool; private EventLoopGroup bossGroup; @@ -69,7 +69,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti public NettyRemotingServer(final NettyServerConfig nettyServerConfig, final NettyEventListener nettyEventListener, - final CommonThreadPool threadPool) { + final BackgroundTaskExecutor threadPool) { super(nettyEventListener); this.nettyServerConfig = nettyServerConfig; this.threadPool = threadPool; diff --git a/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java b/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java index 4a852d99d8f..5d98432572e 100644 --- a/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java +++ b/hertzbeat-remoting/src/test/java/org/apache/hertzbeat/remoting/RemotingServiceTest.java @@ -18,8 +18,10 @@ package org.apache.hertzbeat.remoting; import com.google.protobuf.ByteString; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; import org.apache.hertzbeat.remoting.netty.NettyRemotingServer; @@ -35,7 +37,24 @@ */ public class RemotingServiceTest { - private final CommonThreadPool threadPool = new CommonThreadPool(); + private final BackgroundTaskExecutor threadPool = new BackgroundTaskExecutor() { + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Override + public void execute(Runnable runnable) { + executor.execute(runnable); + } + + @Override + public void executeLongRunning(Runnable runnable) { + executor.execute(runnable); + } + + @Override + public void destroy() { + executor.shutdownNow(); + } + }; private RemotingServer remotingServer; From 494094886c7f7624b156359ec62a57d64e6cc310 Mon Sep 17 00:00:00 2001 From: Logic Date: Tue, 10 Mar 2026 22:13:26 +0800 Subject: [PATCH 09/12] refactor: replace MappingJackson2HttpMessageConverter with JacksonJsonHttpMessageConverter in test classes --- .../controller/MetricsFavoriteControllerTest.java | 4 ++-- .../manager/controller/MonitorsControllerTest.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java index 3be38dcb15e..f926a477d8a 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MetricsFavoriteControllerTest.java @@ -29,7 +29,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.http.MediaType; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.http.converter.json.JacksonJsonHttpMessageConverter; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -73,7 +73,7 @@ class MetricsFavoriteControllerTest { void setUp() { this.mockMvc = MockMvcBuilders.standaloneSetup(metricsFavoriteController) .setControllerAdvice(new GlobalExceptionHandler()) - .setMessageConverters(new MappingJackson2HttpMessageConverter()) + .setMessageConverters(new JacksonJsonHttpMessageConverter()) .build(); } diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java index 7cd4b68040f..f5026a2d928 100644 --- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/MonitorsControllerTest.java @@ -39,13 +39,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.data.web.config.EnableSpringDataWebSupport; import org.springframework.http.MediaType; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.http.converter.json.JacksonJsonHttpMessageConverter; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; -import org.springframework.data.web.config.SpringDataJacksonConfiguration; +import org.springframework.data.web.config.SpringDataJackson3Configuration; import org.springframework.data.web.config.SpringDataWebSettings; +import tools.jackson.databind.json.JsonMapper; /** * Test case for {@link MonitorsController} @@ -63,9 +63,9 @@ class MonitorsControllerTest { @BeforeEach void setUp() { - MappingJackson2HttpMessageConverter messageConverter = new MappingJackson2HttpMessageConverter( - Jackson2ObjectMapperBuilder.json() - .modules(new SpringDataJacksonConfiguration.PageModule( + JacksonJsonHttpMessageConverter messageConverter = new JacksonJsonHttpMessageConverter( + JsonMapper.builder() + .addModule(new SpringDataJackson3Configuration.PageModule( new SpringDataWebSettings(EnableSpringDataWebSupport.PageSerializationMode.DIRECT))) .build()); this.mockMvc = MockMvcBuilders.standaloneSetup(monitorsController) From 7a79d18356852b2d923b7d970a8e95b64e9343bf Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 22:22:31 +0800 Subject: [PATCH 10/12] Add native docs and config notes --- .github/workflows/collector-native-build.yml | 99 +++++++++ CONTRIBUTING.md | 4 +- README.md | 9 +- README_CN.md | 9 +- README_JP.md | 21 +- hertzbeat-ai/pom.xml | 2 +- .../hertzbeat-collector-basic/pom.xml | 6 +- .../hertzbeat-collector-collector/pom.xml | 205 +++++++++++++++++- .../apache/hertzbeat/collector/Collector.java | 12 +- .../CollectorRuntimeHintsRegistrar.java | 111 ++++++++++ .../nativex/NativeCollectorDefaults.java | 58 +++++ .../main/resources/META-INF/spring.factories | 2 + .../reflect-config.json | 54 +++++ ...ertzbeat.collector.collect.AbstractCollect | 37 ++++ .../nativex/NativeCollectorDefaultsTest.java | 39 ++++ .../hertzbeat-collector-common/pom.xml | 4 +- .../collect/common/ssh/SshTunnelHelper.java | 63 +++--- .../hertzbeat-collector-kafka/pom.xml | 6 +- .../hertzbeat-collector-mongodb/pom.xml | 6 +- .../hertzbeat-collector-nebulagraph/pom.xml | 6 +- hertzbeat-collector/pom.xml | 2 +- .../hertzbeat-collector-basic-e2e/pom.xml | 4 +- .../hertzbeat-collector-common-e2e/pom.xml | 6 +- .../hertzbeat-collector-kafka-e2e/pom.xml | 4 +- hertzbeat-e2e/hertzbeat-log-e2e/pom.xml | 4 +- hertzbeat-e2e/pom.xml | 4 +- home/docs/community/contribution.md | 2 +- home/docs/community/development.md | 17 +- home/docs/community/how-to-release.md | 15 +- home/docs/download.md | 17 +- home/docs/help/db2.md | 7 + home/docs/help/mysql.md | 7 + home/docs/help/oceanbase.md | 7 + home/docs/help/oracle.md | 7 + home/docs/help/risc-v.md | 8 +- home/docs/start/native-collector.md | 75 +++++++ home/docs/start/package-deploy.md | 55 ++++- home/docs/start/quickstart.md | 17 +- home/docs/start/virtual-thread.md | 2 +- .../current/community/contribution.md | 2 +- .../current/community/development.md | 17 +- .../current/community/how-to-release.md | 15 +- .../current/download.md | 17 +- .../current/help/db2.md | 7 + .../current/help/mysql.md | 7 + .../current/help/oceanbase.md | 7 + .../current/help/oracle.md | 7 + .../current/help/risc-v.md | 8 +- .../current/start/native-collector.md | 75 +++++++ .../current/start/package-deploy.md | 60 +++-- .../current/start/quickstart.md | 17 +- .../current/start/virtual-thread.md | 2 +- home/sidebars.json | 1 + home/src/components/StructuredData.js | 2 +- home/src/pages/faq.js | 2 +- home/src/pages/zh-cn/faq.js | 2 +- home/static/llms-zh.txt | 2 +- home/static/llms.txt | 2 +- pom.xml | 6 +- script/assembly/collector/assembly-native.xml | 82 +++++++ script/assembly/collector/assembly.xml | 5 +- .../collector/bin-native-win/restart.bat | 25 +++ .../collector/bin-native-win/shutdown.bat | 49 +++++ .../collector/bin-native-win/startup.bat | 102 +++++++++ .../assembly/collector/bin-native/restart.sh | 27 +++ .../assembly/collector/bin-native/shutdown.sh | 62 ++++++ .../assembly/collector/bin-native/startup.sh | 98 +++++++++ script/assembly/collector/bin/startup.sh | 2 +- script/assembly/server/bin/startup.sh | 2 +- .../ci/github-actions/setup-deps/action.yml | 6 +- script/docker/collector/Dockerfile | 4 +- script/docker/collector/build.sh | 11 +- script/docker/server/Dockerfile | 2 +- script/ext-lib/README | 8 +- 74 files changed, 1573 insertions(+), 183 deletions(-) create mode 100644 .github/workflows/collector-native-build.yml create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect create mode 100644 hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java create mode 100644 home/docs/start/native-collector.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md create mode 100644 script/assembly/collector/assembly-native.xml create mode 100644 script/assembly/collector/bin-native-win/restart.bat create mode 100644 script/assembly/collector/bin-native-win/shutdown.bat create mode 100644 script/assembly/collector/bin-native-win/startup.bat create mode 100644 script/assembly/collector/bin-native/restart.sh create mode 100644 script/assembly/collector/bin-native/shutdown.sh create mode 100644 script/assembly/collector/bin-native/startup.sh diff --git a/.github/workflows/collector-native-build.yml b/.github/workflows/collector-native-build.yml new file mode 100644 index 00000000000..97a14b3e8fd --- /dev/null +++ b/.github/workflows/collector-native-build.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Collector Native CI + +on: + workflow_dispatch: + push: + branches: [ action* ] + paths: + - '.github/workflows/collector-native-build.yml' + - 'pom.xml' + - 'hertzbeat-collector/**' + - 'script/assembly/collector/**' + pull_request: + branches: [ master, dev ] + paths: + - '.github/workflows/collector-native-build.yml' + - 'pom.xml' + - 'hertzbeat-collector/**' + - 'script/assembly/collector/**' + +jobs: + build-native-collector: + name: Native collector (${{ matrix.platform }}) + permissions: + contents: read + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: ubuntu-latest + archive_ext: tar.gz + - platform: linux-arm64 + runner: ubuntu-24.04-arm + archive_ext: tar.gz + - platform: macos-amd64 + runner: macos-13 + archive_ext: tar.gz + - platform: macos-arm64 + runner: macos-14 + archive_ext: tar.gz + - platform: windows-amd64 + runner: windows-latest + archive_ext: zip + + steps: + - uses: actions/checkout@v4 + + - name: Set up GraalVM JDK 25 + uses: graalvm/setup-graalvm@v1 + with: + distribution: graalvm-community + java-version: '25' + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: maven + native-image-job-reports: 'true' + + - name: Verify toolchain + shell: pwsh + run: | + java -version + native-image --version + mvn -version + + - name: Build native collector package + run: mvn -B -pl hertzbeat-collector/hertzbeat-collector-collector -am -Pnative -DskipTests package + + - name: Locate native collector package + id: package + shell: pwsh + run: | + $package = Get-ChildItem -Path "dist/apache-hertzbeat-collector-native-*-${{ matrix.platform }}-bin.${{ matrix.archive_ext }}" | Select-Object -First 1 + if (-not $package) { + throw "Native collector package not found for ${{ matrix.platform }}" + } + "archive=$($package.FullName)" >> $env:GITHUB_OUTPUT + + - name: Upload native collector package + uses: actions/upload-artifact@v4 + with: + name: apache-hertzbeat-collector-native-${{ matrix.platform }} + path: ${{ steps.package.outputs.archive }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a1738267c9..9f329883578 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` @@ -172,7 +172,7 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/README.md b/README.md index 0d8cff96a67..e6b5b7dbf7a 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,12 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh ` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters (Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download](https://hertzbeat.apache.org/docs/download) + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml collector: @@ -148,7 +148,8 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh ` or `bin/startup.bat` + - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access `http://localhost:1157` and you will see the registered new collector in dashboard Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -156,7 +157,7 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` diff --git a/README_CN.md b/README_CN.md index 7393ace7010..fda8d7ca640 100644 --- a/README_CN.md +++ b/README_CN.md @@ -127,12 +127,12 @@ ##### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包 `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml` (可选) 3. 部署启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群（可选） - - 下载采集器安装包 `hertzbeat-collector-xx.tar.gz` 到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) + - 下载采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或与你目标平台匹配的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主 HertzBeat 服务的对外 IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml collector: @@ -145,7 +145,8 @@ manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` + - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包。 + - JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -153,7 +154,7 @@ ##### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程 `hertzbeat-startup` 和前端工程 `web-app` -2. 后端：需要 `maven3+`, `java21` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 +2. 后端：需要 `maven3+`, `java25` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 3. 前端：需要 `nodejs npm angular-cli`环境，待本地后端启动后，在 `web-app` 目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` diff --git a/README_JP.md b/README_JP.md index 13c49de6a4f..f8993253319 100644 --- a/README_JP.md +++ b/README_JP.md @@ -108,7 +108,7 @@ docker run -d -p 1157:1157 -p 1158:1158 --name hertzbeat apache/hertzbeat ``` -2. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 +2. スタート：`http://localhost:1157`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 3. コレクタークラスタのデプロイメント（オプション） @@ -126,13 +126,13 @@ ##### 方式２：インストールパッケージ -1. リリースパッケージをダウンロード `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) -2. HertzBeatのymlファイルを設定 `hertzbeat/config/application.yml` （オプション） -3. コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行 -4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat` +1. リリースパッケージ `apache-hertzbeat-xx-bin.tar.gz` をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) +2. HertzBeat の設定ファイル `hertzbeat/config/application.yml` を編集します（任意） +3. コマンド `$ ./bin/startup.sh` または `bin/startup.bat` を実行します +4. ブラウザで `http://localhost:1157` にアクセスします。デフォルトのアカウントとパスワードは `admin/hertzbeat` です 5. コレクタークラスタのデプロイメント（オプション） - - コレクターパッケージを別のホストにダウンロード `hertzbeat-collector-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) - - コレクターのymlファイルを設定 `hertzbeat-collector/config/application.yml` + - 別ホストにコレクターのインストールパッケージ `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM コレクター）または対象プラットフォーム向けの Native コレクターパッケージ（例: `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`）をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) + - コレクターの設定ファイル `hertzbeat-collector/config/application.yml` を編集します ```yaml collector: dispatch: @@ -148,15 +148,16 @@ - `mode: ${MODE:public}`：実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。 - `manager-host: ${MANAGER_HOST:127.0.0.1}`：メインhertzbeatサーバーのIP。 - `manager-port: ${MANAGER_PORT:1158}`：メインhertzbeatサーバポート。 - - コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行。 - - `http://localhost:1157`にアクセスし、登録された新しいコレクターを見ることがでます。 + - `ext-lib` で MySQL、OceanBase、Oracle、DB2 などの外部 JDBC ドライバーを読み込む必要がある場合は、JVM コレクターのインストールパッケージを使用してください。 + - JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。 + - メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。 詳細ステップ [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) ##### 方式３：ローカルの実行 1. ローカルの実行には、バックエンドのプロジェクト`hertzbeat-startup`とフロントエンドのプロジェクト`web-app`を起動する必要があります。 -2. バックエンド：`maven3+`、`Java21`と`lombok`の環境は必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し、`hertzbeat-startup` を起動します。 +2. バックエンド：`maven3+`、`Java25`、`lombok` の環境が必要です。`YML` 設定を修正し、Java 仮想マシンパラメータに `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加して `hertzbeat-startup` を起動します。 3. フロントエンド：`nodejs npm angular-cli`の環境は必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `ng serve --open` というコマンドを実行します。 4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 diff --git a/hertzbeat-ai/pom.xml b/hertzbeat-ai/pom.xml index 2b9298b6b5e..235b2481dce 100644 --- a/hertzbeat-ai/pom.xml +++ b/hertzbeat-ai/pom.xml @@ -27,7 +27,7 @@ ${hertzbeat.version} 1.1.1 - 21 + 25 diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index c43972cda0a..de107950e41 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -30,8 +30,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 1.2.5 @@ -189,4 +189,4 @@ ${zookeeper.version} - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index 8a108c99de0..d40bdf60abb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -29,12 +29,18 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 + + org.apache.hertzbeat + hertzbeat-common-spring + ${hertzbeat.version} + + org.apache.hertzbeat @@ -142,8 +148,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -216,8 +222,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -344,5 +350,192 @@ + + native + + hertzbeat-collector-collector + + ../../script/assembly/collector/bin-native + tar.gz + unsupported + ${native.target.platform}-bin + apache-hertzbeat-collector-native-${hzb.version}-${native.target.platform}-bin + target/${native.image.name}${native.binary.extension} + ${project.build.finalName}${native.binary.extension} + + + apache-hertzbeat-collector-native-${hzb.version} + + + src/main/resources + + META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + full + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-supported-native-platform + validate + + enforce + + + + + native.target.platform + ^(linux-amd64|linux-arm64|macos-amd64|macos-arm64|windows-amd64)$ + Unsupported native collector platform. Build the native collector on a supported GitHub runner or matching local host: linux-amd64, linux-arm64, macos-amd64, macos-arm64, windows-amd64. + + + + + + + + org.graalvm.buildtools + native-maven-plugin + 0.11.4 + + ${native.image.name} + + -H:+UnlockExperimentalVMOptions + -H:-AddAllFileSystemProviders + -H:ServiceLoaderFeatureExcludeServiceProviders=org.apache.sshd.common.file.root.RootedFileSystemProvider,org.apache.sshd.sftp.client.fs.SftpFileSystemProvider + --initialize-at-build-time=net.i2p.crypto.eddsa.EdDSASecurityProvider,org.apache.arrow.memory.util.MemoryUtil + -J--add-opens=java.base/java.nio=ALL-UNNAMED + -J-Dorg.apache.sshd.security.registrars=org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + native-bin + package + + single + + + + ../../script/assembly/collector/assembly-native.xml + + ../../dist + + + + + + + + + native-platform-linux-amd64 + + + Linux + amd64 + + + + linux-amd64 + + + + native-platform-linux-x86_64 + + + Linux + x86_64 + + + + linux-amd64 + + + + native-platform-linux-arm64 + + + Linux + aarch64 + + + + linux-arm64 + + + + native-platform-macos-amd64 + + + mac + x86_64 + + + + macos-amd64 + + + + native-platform-macos-arm64 + + + mac + aarch64 + + + + macos-arm64 + + + + native-platform-windows-amd64 + + + windows + amd64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + + + native-platform-windows-x86_64 + + + windows + x86_64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java index 3e41f228010..0eca3ce9e35 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java @@ -18,10 +18,13 @@ package org.apache.hertzbeat.collector; import jakarta.annotation.PostConstruct; +import org.apache.hertzbeat.collector.nativex.CollectorRuntimeHintsRegistrar; +import org.apache.hertzbeat.collector.nativex.NativeCollectorDefaults; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ImportRuntimeHints; /** * collector startup @@ -29,13 +32,20 @@ @ComponentScan(basePackages = {"org.apache.hertzbeat"}) @ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"}) @SpringBootApplication +@ImportRuntimeHints(CollectorRuntimeHintsRegistrar.class) public class Collector { public static void main(String[] args) { - SpringApplication.run(Collector.class, args); + SpringApplication application = new SpringApplication(Collector.class); + NativeCollectorDefaults.applyTo(application); + application.run(args); } @PostConstruct public void init() { System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory"); + if (System.getProperty("arrow.allocation.manager.type") == null + && System.getenv("ARROW_ALLOCATION_MANAGER_TYPE") == null) { + System.setProperty("arrow.allocation.manager.type", "Netty"); + } } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java new file mode 100644 index 00000000000..3464caa1b1b --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.LinkedHashSet; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.arrow.memory.netty.NettyAllocationManager; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.IntervalUnit; +import org.apache.arrow.vector.types.MetadataVersion; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.UnionMode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.hertzbeat.common.entity.dto.ServerInfo; +import org.springframework.aot.hint.BindingReflectionHintsRegistrar; +import org.springframework.aot.hint.MemberCategory; +import org.springframework.aot.hint.RuntimeHints; +import org.springframework.aot.hint.RuntimeHintsRegistrar; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.lang.NonNull; +import org.springframework.util.ClassUtils; + +/** + * Registers native binding hints for collector-side message payloads. + */ +@Slf4j +public class CollectorRuntimeHintsRegistrar implements RuntimeHintsRegistrar { + + private static final String JOB_PACKAGE = "org.apache.hertzbeat.common.entity.job"; + private static final String JOB_PROTOCOL_PACKAGE = "org.apache.hertzbeat.common.entity.job.protocol"; + + @Override + public void registerHints(@NonNull RuntimeHints hints, ClassLoader classLoader) { + BindingReflectionHintsRegistrar bindingRegistrar = new BindingReflectionHintsRegistrar(); + registerType(bindingRegistrar, hints, ServerInfo.class); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PACKAGE); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PROTOCOL_PACKAGE); + hints.reflection().registerType(NettyAllocationManager.class, MemberCategory.DECLARED_FIELDS); + registerType(bindingRegistrar, hints, Schema.class); + registerType(bindingRegistrar, hints, Field.class); + registerType(bindingRegistrar, hints, FieldType.class); + registerType(bindingRegistrar, hints, DictionaryEncoding.class); + registerType(bindingRegistrar, hints, DateUnit.class); + registerType(bindingRegistrar, hints, FloatingPointPrecision.class); + registerType(bindingRegistrar, hints, IntervalUnit.class); + registerType(bindingRegistrar, hints, MetadataVersion.class); + registerType(bindingRegistrar, hints, TimeUnit.class); + registerType(bindingRegistrar, hints, UnionMode.class); + for (Class nestedClass : ArrowType.class.getDeclaredClasses()) { + if (!nestedClass.isAnnotation() && !nestedClass.isInterface()) { + registerType(bindingRegistrar, hints, nestedClass); + } + } + } + + private void scanBindingPackage(ClassLoader classLoader, BindingReflectionHintsRegistrar bindingRegistrar, + RuntimeHints hints, String basePackage) { + for (Class clazz : findBindingTypes(basePackage, classLoader)) { + registerType(bindingRegistrar, hints, clazz); + } + } + + private void registerType(BindingReflectionHintsRegistrar bindingRegistrar, RuntimeHints hints, Class clazz) { + bindingRegistrar.registerReflectionHints(hints.reflection(), clazz); + } + + private Set> findBindingTypes(String basePackage, ClassLoader classLoader) { + Set> bindingTypes = new LinkedHashSet<>(); + ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false); + TypeFilter includeAll = (metadataReader, metadataReaderFactory) -> true; + scanner.addIncludeFilter(includeAll); + for (BeanDefinition candidate : scanner.findCandidateComponents(basePackage)) { + String className = candidate.getBeanClassName(); + if (className == null) { + continue; + } + try { + Class clazz = ClassUtils.forName(className, classLoader); + if (!clazz.isAnnotation() && !clazz.isInterface()) { + bindingTypes.add(clazz); + } + } catch (Throwable ex) { + log.debug("Skip native binding hint registration for {}", className, ex); + } + } + return bindingTypes; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java new file mode 100644 index 00000000000..5f97c36ecbf --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.Map; +import org.springframework.boot.SpringApplication; +import org.springframework.core.NativeDetector; + +/** + * Applies the native-only collector defaults without forking {@code application.yml}. + */ +public final class NativeCollectorDefaults { + + static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude"; + static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + "org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration", + "org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration", + "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration", + "org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.health.DataSourceHealthContributorAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.metrics.DataSourcePoolMetricsAutoConfiguration", + "org.springframework.boot.tomcat.autoconfigure.metrics.TomcatMetricsAutoConfiguration"); + + private NativeCollectorDefaults() { + } + + public static void applyTo(SpringApplication application) { + Map defaultProperties = defaultProperties(NativeDetector.inNativeImage()); + if (!defaultProperties.isEmpty()) { + application.setDefaultProperties(defaultProperties); + } + } + + static Map defaultProperties(boolean nativeImage) { + if (!nativeImage) { + return Map.of(); + } + return Map.of(AUTOCONFIGURE_EXCLUDE_PROPERTY, NATIVE_AUTOCONFIGURE_EXCLUDES); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..16c40f8c6f3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +# Intentionally left blank. +# Native collector defaults are applied from Collector.main. diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json new file mode 100644 index 00000000000..227a978fe57 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json @@ -0,0 +1,54 @@ +[ + { + "name": "org.apache.hertzbeat.collector.Collector__ApplicationContextInitializer", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.collector.Collector__BeanFactoryRegistrations", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.common.entity.dto.ServerInfo", + "allDeclaredConstructors": true, + "allDeclaredFields": true, + "allDeclaredMethods": true + }, + { + "name": "io.netty.channel.kqueue.KQueueDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.nio.NioEventLoopGroup", + "allPublicConstructors": true + } +] diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect new file mode 100644 index 00000000000..81ed2990010 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect @@ -0,0 +1,37 @@ +org.apache.hertzbeat.collector.collect.http.HttpCollectImpl +org.apache.hertzbeat.collector.collect.http.SslCertificateCollectImpl +org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect +org.apache.hertzbeat.collector.collect.icmp.IcmpCollectImpl +org.apache.hertzbeat.collector.collect.jmx.JmxCollectImpl +org.apache.hertzbeat.collector.collect.redis.RedisCommonCollectImpl +org.apache.hertzbeat.collector.collect.mongodb.MongodbSingleCollectImpl +org.apache.hertzbeat.collector.collect.rocketmq.RocketmqSingleCollectImpl +org.apache.hertzbeat.collector.collect.snmp.SnmpCollectImpl +org.apache.hertzbeat.collector.collect.ssh.SshCollectImpl +org.apache.hertzbeat.collector.collect.telnet.TelnetCollectImpl +org.apache.hertzbeat.collector.collect.smtp.SmtpCollectImpl +org.apache.hertzbeat.collector.collect.ntp.NtpCollectImpl +org.apache.hertzbeat.collector.collect.websocket.WebsocketCollectImpl +org.apache.hertzbeat.collector.collect.ftp.FtpCollectImpl +org.apache.hertzbeat.collector.collect.udp.UdpCollectImpl +org.apache.hertzbeat.collector.collect.push.PushCollectImpl +org.apache.hertzbeat.collector.collect.dns.DnsCollectImpl +org.apache.hertzbeat.collector.collect.nginx.NginxCollectImpl +org.apache.hertzbeat.collector.collect.memcached.MemcachedCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl +org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl +org.apache.hertzbeat.collector.collect.registry.RegistryImpl +org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl +org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl +org.apache.hertzbeat.collector.collect.script.ScriptCollectImpl +org.apache.hertzbeat.collector.collect.mqtt.MqttCollectImpl +org.apache.hertzbeat.collector.collect.ipmi2.IpmiCollectImpl +org.apache.hertzbeat.collector.collect.kafka.KafkaCollectImpl +org.apache.hertzbeat.collector.collect.sd.HttpSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.NacosSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.DnsSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.EurekaSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.ConsulSdCollectImpl +org.apache.hertzbeat.collector.collect.modbus.ModbusCollectImpl +org.apache.hertzbeat.collector.collect.s7.S7CollectImpl diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java new file mode 100644 index 00000000000..2ffa4beeebc --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +class NativeCollectorDefaultsTest { + + @Test + void shouldProvideNativeAutoconfigureExcludesWhenNativeImage() { + Map properties = NativeCollectorDefaults.defaultProperties(true); + assertEquals(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void shouldNotProvideNativeSpecificPropertiesForJvmCollector() { + assertTrue(NativeCollectorDefaults.defaultProperties(false).isEmpty()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index d4cffe23fb7..7e0de3eda8a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java index 16b3087e895..de65033abef 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java @@ -17,10 +17,6 @@ package org.apache.hertzbeat.collector.collect.common.ssh; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.Scheduler; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -35,7 +31,6 @@ import java.io.IOException; import java.net.ServerSocket; import java.security.GeneralSecurityException; -import java.time.Duration; import java.util.Comparator; import java.util.ArrayList; import java.util.Iterator; @@ -53,34 +48,7 @@ public class SshTunnelHelper { private static final long DEFAULT_CACHE_TIMEOUT = 500 * 1000; - private static final Cache TRACKER_CACHE = - Caffeine.newBuilder() - .initialCapacity(1) - .maximumSize(1000) - .expireAfterAccess(Duration.ofMillis(DEFAULT_CACHE_TIMEOUT)) - .scheduler(Scheduler.systemScheduler()) - .removalListener((key, value, cause) -> { - if (cause == RemovalCause.REPLACED) { - return; - } - if (key != null && value != null) { - // 1. try close tunnel - SshClientSessionWrapper clientSessionWrapper = (SshClientSessionWrapper) key; - LocalPortForwardingWrapper wrapper = (LocalPortForwardingWrapper) value; - wrapper.remove(clientSessionWrapper.getClientSession()); - - // 2. try close session - if (!clientSessionWrapper.isShareConnection()) { - try { - clientSessionWrapper.close(); - log.info("[SSH Tunnel] close unshared ssh connection, {}", clientSessionWrapper); - } catch (IOException e) { - log.error("[SSH Tunnel] close unshared ssh connection error", e); - } - } - } - }) - .build(); + private static final Map TRACKER_CACHE = new ConcurrentHashMap<>(); /** @@ -122,15 +90,13 @@ public static int localPortForward(SshTunnel sshTunnel, String remoteHost, Strin // 2. get tunnel LocalPortForwardingWrapper forwardingWrapper = selectWrapper( - TRACKER_CACHE.getIfPresent(sessionWrapper), sessionWrapper, remoteHost, remotePort); + TRACKER_CACHE.get(sessionWrapper), sessionWrapper, remoteHost, remotePort); int localPort; if (forwardingWrapper == null) { localPort = getRandomPort(); LocalPortForwardingWrapper newForwardingWrapper = sessionWrapper .createLocalPortForwardingTracker(localPort, remoteHost, Integer.parseInt(remotePort)); - if (TRACKER_CACHE.getIfPresent(sessionWrapper) == null) { - TRACKER_CACHE.put(sessionWrapper, newForwardingWrapper); - } + TRACKER_CACHE.putIfAbsent(sessionWrapper, newForwardingWrapper); log.info("[SSH Tunnel] created ssh forwarding tracker ssh:{}, remote:{}, localPort:{}", sshTunnel.getHost() + ":" + sshTunnel.getPort(), remoteHost + ":" + remotePort, localPort); } else { @@ -184,6 +150,23 @@ private static int getRandomPort() throws IOException { } } + private static void removeSessionCacheEntry(ClientSession session) { + TRACKER_CACHE.entrySet().removeIf(entry -> { + if (!Objects.equals(entry.getKey().getClientSession(), session)) { + return false; + } + if (!entry.getKey().isShareConnection()) { + try { + entry.getKey().close(); + log.info("[SSH Tunnel] close unshared ssh connection, {}", entry.getKey()); + } catch (IOException e) { + log.error("[SSH Tunnel] close unshared ssh connection error", e); + } + } + return true; + }); + } + @Getter @Setter @EqualsAndHashCode @@ -267,6 +250,10 @@ public List select(ClientSession session, Predicate< list.add(wrapper); } } + if (trackerList.isEmpty()) { + map.remove(session); + removeSessionCacheEntry(session); + } return list; } @@ -290,6 +277,8 @@ public void remove(ClientSession session) { log.error("[SSH Tunnel] Remove ssh session local port forwarding error", e); } } + map.remove(session); + removeSessionCacheEntry(session); } public void close() throws IOException { diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml index 9acd5c98d94..7175050e818 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -47,4 +47,4 @@ kafka-clients - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml index 1cdd2070f38..472dc55c3b9 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -46,4 +46,4 @@ mongodb-driver-sync - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml index b21d4554633..9ea2e9d914e 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -57,4 +57,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index 48daddead48..bbb03e841d5 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -28,7 +28,7 @@ ${project.artifactId} pom - 21 + 25 ${java.version} ${java.version} diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml index af0d937df12..316661344be 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-basic-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 42.5.5 diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml index 323e3c57b7e..1e2952bdf28 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-common-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -67,4 +67,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml index d56825f6def..3c1824951d1 100644 --- a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-kafka-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml index c725c7bd35f..a5638e23a20 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-log-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 3.3.1 3.6.1 diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index dbcbad9c5df..445a8a50776 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -36,8 +36,8 @@ true - 17 - 17 + ${java.version} + ${java.version} UTF-8 2.0.3 4.13.2 diff --git a/home/docs/community/contribution.md b/home/docs/community/contribution.md index 72b823e716d..e1651205688 100644 --- a/home/docs/community/contribution.md +++ b/home/docs/community/contribution.md @@ -52,7 +52,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` diff --git a/home/docs/community/development.md b/home/docs/community/development.md index fd2c83327cf..c52e5c075e1 100644 --- a/home/docs/community/development.md +++ b/home/docs/community/development.md @@ -12,7 +12,7 @@ sidebar_label: Development ### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` @@ -34,7 +34,7 @@ sidebar_label: Development ## Build HertzBeat binary package -> Requires `maven3+`, `java21`, `node` and `pnpm` environments. +> Requires `maven3+`, `java25`, `node` and `pnpm` environments. ### Frontend build @@ -50,20 +50,23 @@ sidebar_label: Development ### Backend build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean package -Prelease` -The HertzBeat install package will at `dist/hertzbeat-{version}.tar.gz` +The HertzBeat install package will be generated at `dist/apache-hertzbeat-{version}-bin.tar.gz` ### Collector build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean install` 3. Cd to the `hertzbeat-collector` directory: `cd hertzbeat-collector` -4. Execute under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +4. Build the JVM collector package under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +5. Build the native collector package under `hertzbeat-collector` directory: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -The HertzBeat collector package will at `dist/hertzbeat-collector-{version}.tar.gz` +> Native collector packaging requires GraalVM for JDK 25 with the `native-image` tool available in `PATH`. + +The HertzBeat collector packages will be generated at `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` and a platform-specific native package such as `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` diff --git a/home/docs/community/how-to-release.md b/home/docs/community/how-to-release.md index 6deb97d01b2..c92feedccb6 100644 --- a/home/docs/community/how-to-release.md +++ b/home/docs/community/how-to-release.md @@ -10,7 +10,7 @@ This tutorial describes in detail how to release Apache HertzBeat™, take the r This release process is operated in the UbuntuOS(Windows,Mac), and the following tools are required: -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,23 @@ mvn clean install mvn clean package -Pcluster ``` +> Build the native collector binaries with GraalVM JDK 25 and `native-image` on matching GitHub-hosted runners or equivalent local hosts + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> The repository workflow `.github/workflows/collector-native-build.yml` can build and upload all supported native collector packages on GitHub-hosted runners. Release signing and final publishing remain a release manager responsibility. + The release package are here: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 Package the source code diff --git a/home/docs/download.md b/home/docs/download.md index fde53959f4b..b6d62fa5f06 100644 --- a/home/docs/download.md +++ b/home/docs/download.md @@ -22,9 +22,15 @@ Download the latest Apache HertzBeat™ release (v1.8.0) as server binary, colle |-------------|------|---------|----------| | **Server Binary** | ~200MB | Main monitoring server | Linux, macOS, Windows | | **Collector Binary** | ~50MB | Distributed collectors | Linux, macOS, Windows | -| **Source Code** | ~30MB | Build from source | Any with Java 21+ | +| **Source Code** | ~30MB | Build from source | Any with Java 25+ | | **Docker Compose** | ~5MB | Full stack deployment | Docker environments | +:::tip Native Collector Recommendation +If you do not need MySQL, OceanBase, Oracle, DB2, or other monitoring types that rely on external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. + +Trade-offs: native packages are platform-specific and do not support runtime `ext-lib` JDBC loading. See [Native Collector Guide](start/native-collector). +::: + :::tip Security Verification Verify downloads using GPG signatures and SHA512 checksums. See [Apache Verification Guide](https://www.apache.org/dyn/closer.cgi#verify) and [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS). ::: @@ -57,6 +63,9 @@ For older releases, please check the [archive](https://archive.apache.org/dist/i **Server Binary** - For most users. Includes the main HertzBeat monitoring server with web UI. **Collector Binary** - For distributed deployments. Deploy collectors in remote networks to report to the main server. +Native collector downloads are platform-specific, for example `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`. + +If you are deciding between JVM and native collector packages, start with [Native Collector Guide](start/native-collector). **Source Code** - For developers who want to build, modify, or contribute to HertzBeat. @@ -73,17 +82,19 @@ Import Apache HertzBeat KEYS first: `wget https://downloads.apache.org/hertzbeat ### What are the system requirements? **Server Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 4GB RAM minimum (8GB recommended) - 2 CPU cores minimum - 20GB disk space **Collector Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 2GB RAM minimum - 1 CPU core minimum - 5GB disk space +Native collector packages are published per target platform, while the JVM collector package remains cross-platform. + ### Can I use Docker instead of binary packages? Yes. Docker is the recommended installation method: diff --git a/home/docs/help/db2.md b/home/docs/help/db2.md index 5652094fa92..9258d261634 100644 --- a/home/docs/help/db2.md +++ b/home/docs/help/db2.md @@ -16,6 +16,13 @@ keywords: [ open source monitoring tool, open source database monitoring tool, m - Copy the JAR package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +DB2 monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for DB2 monitoring +- Do not use the native collector package for DB2 monitoring +::: + ### Configuration Parameters The following are the required configuration parameters for DB2 monitoring: diff --git a/home/docs/help/mysql.md b/home/docs/help/mysql.md index f21a1a1da87..7d62f7c0a46 100644 --- a/home/docs/help/mysql.md +++ b/home/docs/help/mysql.md @@ -14,6 +14,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +MySQL monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for MySQL monitoring +- Do not use the native collector package for MySQL monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/oceanbase.md b/home/docs/help/oceanbase.md index 510934771f5..d4fe52aeb18 100644 --- a/home/docs/help/oceanbase.md +++ b/home/docs/help/oceanbase.md @@ -13,6 +13,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +OceanBase monitoring depends on the external MySQL JDBC driver in `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for OceanBase monitoring +- Do not use the native collector package for OceanBase monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/oracle.md b/home/docs/help/oracle.md index 83d7854030e..562e68cfcfd 100644 --- a/home/docs/help/oracle.md +++ b/home/docs/help/oracle.md @@ -13,6 +13,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +Oracle monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for Oracle monitoring +- Do not use the native collector package for Oracle monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/risc-v.md b/home/docs/help/risc-v.md index 41d605afe84..a7d2ee5d467 100644 --- a/home/docs/help/risc-v.md +++ b/home/docs/help/risc-v.md @@ -48,7 +48,7 @@ Reference: [Ubuntu Official Documentation](https://canonical-ubuntu-boards.readt ## Install and Configure RISC-V-Compatible JDK -> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 21. +> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 25. **1. Download Temurin JDK** Download link: [https://adoptium.net/temurin/releases](https://adoptium.net/temurin/releases) @@ -66,11 +66,11 @@ sudo apt install -y tar wget sudo mkdir -p /usr/lib/jvm # Extract to system directory -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # Edit environment variables sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # Apply configuration @@ -91,6 +91,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > Notes: > -> 1. Replace `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` with your actual JDK filename. +> 1. Replace `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` with your actual JDK filename. > 2. Ensure the emulator has internet access to download HertzBeat. > 3. If issues arise, verify disk permissions and Java environment paths. diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md new file mode 100644 index 00000000000..9f16f217b15 --- /dev/null +++ b/home/docs/start/native-collector.md @@ -0,0 +1,75 @@ +--- +id: native-collector +title: Native Collector Guide +sidebar_label: Native Collector +description: When to choose the HertzBeat native collector package, its benefits, limitations, and deployment guidance. +--- + +## When should I choose the native collector? + +Choose the native collector package when your monitoring workload does not depend on loading external JDBC drivers from `ext-lib`. + +Typical native-friendly workloads include: + +- HTTP, HTTPS, website availability, and API checks +- Port, ping, SSL certificate, and other network probes +- Redis, Zookeeper, Kafka, and other non-JDBC monitoring types + +## Why use it? + +Compared with the JVM collector package, the native collector package is usually a better fit when you want: + +- Faster startup +- Lower baseline memory usage +- A simpler runtime without a bundled or preinstalled JDK + +## What are the trade-offs? + +The native collector package is not a drop-in replacement for every JVM collector scenario. + +- Native packages are platform-specific. You must choose the package that matches your OS and CPU architecture. +- The native collector does not support loading external JDBC driver JARs from `ext-lib` at runtime. +- If your deployment depends on JVM-style runtime classpath extension, keep using the JVM collector package. + +## When should I stay on the JVM collector? + +Use the JVM collector package if your monitoring depends on external JDBC drivers, especially: + +- MySQL, which requires `mysql-connector-j` +- OceanBase, which also depends on the MySQL JDBC driver +- Oracle, which requires `ojdbc8` and sometimes `orai18n` +- DB2, which requires `jcc` + +## Package naming + +The JVM collector package remains cross-platform: + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +The native collector package is platform-specific: + +- Linux or macOS: `apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows: `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +Examples: + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## Configuration consistency + +The native collector package uses the same `config/application.yml` layout as the JVM collector package. + +That means: + +- Collector connection settings are edited in the same place +- Virtual-thread related configuration is edited in the same place +- Native-only boot adjustments are applied by code at runtime instead of maintaining a second `application.yml` + +## Recommended decision + +- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. +- Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. + +For package deployment steps, refer to [Install HertzBeat via Package](package-deploy). diff --git a/home/docs/start/package-deploy.md b/home/docs/start/package-deploy.md index 0747d7821d9..f2e8981f7cd 100644 --- a/home/docs/start/package-deploy.md +++ b/home/docs/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: Install via Package :::tip You can install and run Apache HertzBeat™ on Linux Windows Mac system, and CPU supports X86/ARM64. -Since version 1.6.0 uses `Java 21` and the installation package no longer provides a built-in JDK version, use the new Hertzbeat according to the following situations: +The current branch uses `Java 25`, and the standard installation package no longer provides a built-in JDK. Use HertzBeat according to the following situations: -- When the default environment variable on your server is `Java 21`, you do not need to take any action for this step. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`, and if there are no other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and search the engine for how to set a new environment variable pointing to the new `Java 21`. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`,and you don't want to change the environment variable because if there are other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and rename the extracted folder to `java`, then copy it to the Hertzbeat extraction directory. +- When the default environment variable on your server is `Java 25`, you do not need to take any action for this step. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and if there are no other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, and set a new environment variable pointing to `Java 25`. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and you do not want to change the environment variable because there are other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, rename the extracted folder to `java`, and then copy it to the HertzBeat extraction directory. ::: @@ -64,11 +64,20 @@ HertzBeat Collector is a lightweight data collector used to collect and send dat Deploying multiple HertzBeat Collectors can achieve high availability, load balancing, and cloud-edge collaboration of data. ::: +:::tip Native Collector Recommendation +If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. + +Before choosing it, review the trade-offs in [Native Collector Guide](native-collector). +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. Download installation package - Download installation package `apache-hertzbeat-collector-xxx-bin.tar.gz` corresponding to your system environment + Download the collector package that matches your deployment mode: + - JVM collector package: `apache-hertzbeat-collector-xxx-bin.tar.gz` + - Native collector package for Linux or macOS: `apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Native collector package for Windows: `apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` - [Download Page](/docs/download) 2. Configure the collector configuration file @@ -77,6 +86,10 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # or + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # or + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` Configure the collector configuration yml file `config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` @@ -102,11 +115,31 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 3. Start the service - Run command `$ ./bin/startup.sh` or `bin/startup.bat` + Run `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. 4. Begin to explore HertzBeat Collector - Access `http://ip:1157` and you will see the registered new collector in dashboard + Open the HertzBeat server dashboard at `http://:1157` and confirm the new collector is registered. + +:::important Native Collector Limitations +The native collector package is suitable for monitoring types that do not rely on external JVM classpath extension. + +See [Native Collector Guide](native-collector) for package selection, package naming, and platform-specific trade-offs. + +`ext-lib`-based JDBC driver loading is a JVM collector capability. The native collector package does not support loading external JDBC driver JARs from `ext-lib` at runtime. + +If your monitoring depends on external JDBC drivers, use the JVM collector package instead of the native collector package. This currently includes: + +- MySQL, which requires `mysql-connector-j` +- OceanBase, which also relies on the MySQL JDBC driver +- Oracle, which requires `ojdbc8` and often `orai18n` +- DB2, which requires `jcc` + +Recommended deployment: + +- Use the native collector package for HTTP, website, port, ping, and similar non-JDBC monitoring types +- Use the JVM collector package when you need `ext-lib` driver extension +::: **HAVE FUN** @@ -117,15 +150,15 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 1. you need to prepare the JAVA environment in advance Install JAVA runtime environment-refer to [official website](https://www.oracle.com/java/technologies/downloads/) - requirement：JDK21 ENV + requirement：JDK25 ENV download JAVA installation package: [mirror website](https://mirrors.huaweicloud.com/openjdk/) After installation use command line to check whether you install it successfully. ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/docs/start/quickstart.md b/home/docs/start/quickstart.md index ddb24cab1e5..8e4830ec023 100644 --- a/home/docs/start/quickstart.md +++ b/home/docs/start/quickstart.md @@ -13,7 +13,7 @@ Install Apache HertzBeat™ in under 5 minutes using Docker with a single comman ## Installation Methods -HertzBeat provides four installation options: +HertzBeat provides multiple installation options: 1. **Docker** (Recommended) - Fastest setup, production-ready 2. **Binary Package** - Traditional deployment with manual configuration @@ -54,12 +54,13 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters(Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download Page](https://hertzbeat.apache.org/docs/download) + - If you do not need MySQL, OceanBase, Oracle, DB2, or other `ext-lib` JDBC drivers, prefer the native collector package for faster startup and lower memory usage. See [Native Collector Guide](native-collector). + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your target platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download Page](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml @@ -74,15 +75,17 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh` or `bin/startup.bat` - - Access `http://localhost:1157` and you will see the registered new collector in dashboard + - Native collector trade-offs: platform-specific packages, no runtime `ext-lib` JDBC loading, and less suitable for JVM-style runtime classpath extension. See [Native Collector Guide](native-collector). + - If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers from `ext-lib`, use the JVM collector package. + - Run command `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. + - Access the HertzBeat server dashboard at `http://localhost:1157` and confirm the new collector is registered. Detailed config refer to [Install HertzBeat via Package](package-deploy) ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `manager` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` @@ -108,7 +111,7 @@ Detailed steps refer to [Artifact Hub](https://artifacthub.io/packages/helm/hert - 2 CPU cores - 4GB RAM - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ **Operating Systems:** Linux, macOS, Windows (via Docker or WSL) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index f9d36190316..06ee9518a8e 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: Virtual Threads description: Configure HertzBeat virtual-thread executors, defaults, rollback switches, and tuning guidance. --- -HertzBeat runs on JDK 21 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. +HertzBeat runs on JDK 25 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. ## 1. Where to Configure It diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md index fe530b8bc01..3748879cf55 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md @@ -52,7 +52,7 @@ limitations under the License. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md index 76b20f0d2ac..3df906d2f8e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md @@ -11,7 +11,7 @@ sidebar_label: 运行编译 ### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` @@ -37,7 +37,7 @@ sidebar_label: 运行编译 ## 生成二进制包 -> 需要 `maven3+`, `java21`, `node` 和 `pnpm` 环境. +> 需要 `maven3+`, `java25`, `node` 和 `pnpm` 环境. ### 前端打包 @@ -53,20 +53,23 @@ sidebar_label: 运行编译 ### 后端打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean package -Prelease` -HertzBeat 包将生成为 `dist/hertzbeat-{version}.tar.gz` +HertzBeat 包将生成为 `dist/apache-hertzbeat-{version}-bin.tar.gz` ### 采样器打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean install` 3. 切换到 `hertzbeat-collector` 目录: `cd hertzbeat-collector` -4. 在 `hertzbeat-collector` 目录下执行: `mvn clean package -Pcluster` +4. 在 `hertzbeat-collector` 目录下打 JVM 采集器安装包: `mvn clean package -Pcluster` +5. 在 `hertzbeat-collector` 目录下打 Native 采集器安装包: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -HertzBeat 采样器包将生成为 `dist/hertzbeat-collector-{version}.tar.gz` +> Native 采集器打包需要带有 `native-image` 命令的 GraalVM JDK 25 环境。 + +HertzBeat 采集器安装包将生成为 `dist/apache-hertzbeat-collector-{version}-bin.tar.gz`，以及类似 `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 这样的 Native 平台安装包 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md index b0d5ae4239e..e5a8594d134 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md @@ -10,7 +10,7 @@ sidebar_position: 4 此发布过程在 UbuntuOS(可在 Windows Mac) 中进行操作，并需要以下环境： -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,23 @@ mvn clean install mvn clean package -Pcluster ``` +> 在匹配的平台 GitHub Hosted Runner 或对应本地主机上，使用带 `native-image` 的 GraalVM JDK 25 构建 Native 采集器安装包 + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> 仓库中的 `.github/workflows/collector-native-build.yml` 可以在 GitHub Hosted Runner 上构建并上传全部受支持平台的 Native 采集器安装包。正式发布时的签名与最终上传仍由 release manager 负责。 + 生成的二进制包在: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 打包项目源代码 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md index 2325660f32d..1b467661198 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md @@ -22,9 +22,15 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 |--------|------|------|------| | **服务器二进制** | ~200MB | 主监控服务器 | Linux、macOS、Windows | | **采集器二进制** | ~50MB | 分布式采集器 | Linux、macOS、Windows | -| **源码** | ~30MB | 从源码构建 | 任何支持 Java 21+ 的平台 | +| **源码** | ~30MB | 从源码构建 | 任何支持 Java 25+ 的平台 | | **Docker Compose** | ~5MB | 全栈部署 | Docker 环境 | +:::tip Native 采集器推荐 +如果你不需要 MySQL、OceanBase、Oracle、DB2，或其他依赖 `ext-lib` 外部 JDBC 驱动的监控类型，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。 + +它的代价是安装包按平台区分，且不支持运行时 `ext-lib` JDBC 加载。详见 [Native 采集器指南](start/native-collector)。 +::: + :::tip 安全验证 使用 GPG 签名和 SHA512 校验和验证下载。参见 [Apache 验证指南](https://www.apache.org/dyn/closer.cgi#verify) 和 [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS)。 ::: @@ -57,6 +63,9 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 **服务器二进制** - 大多数用户使用。包含主 HertzBeat 监控服务器和 Web UI。 **采集器二进制** - 分布式部署使用。在远程网络部署采集器向主服务器上报。 +Native 采集器下载包按目标平台区分，例如 `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 或 `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`。 + +如果你正在 JVM 采集器和 Native 采集器之间做选择，建议先阅读 [Native 采集器指南](start/native-collector)。 **源码** - 开发者想要构建、修改或贡献 HertzBeat 时使用。 @@ -73,17 +82,19 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 ### 系统要求是什么？ **服务器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 4GB RAM 最低（推荐 8GB） - 2 CPU 核心 最低 - 20GB 磁盘空间 **采集器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 2GB RAM 最低 - 1 CPU 核心 最低 - 5GB 磁盘空间 +Native 采集器安装包会按目标平台分别发布，JVM 采集器安装包仍然保持跨平台。 + ### 可以用 Docker 代替二进制包吗？ 可以。Docker 是推荐的安装方法： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md index 0c2f9bb7e32..0dcba300aee 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md @@ -15,6 +15,13 @@ keywords: [ 开源监控系统, 开源数据库监控, DB2数据库监控 ] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下 - 重启 HertzBeat 服务 +:::important 采集器包选择 +DB2 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- DB2 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 DB2 监控 +::: + ### 配置参数 以下是 DB2 监控所需的配置参数： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md index 82b3f36b564..bc7e723a558 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, Mysql数据库监控] - 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. - 重启 HertzBeat 服务。 +:::important 采集器包选择 +MySQL 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- MySQL 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 MySQL 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md index edeeaeaf611..5893d9c1957 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, OceanBase 数据库监控] - 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. - 重启 HertzBeat 服务。 +:::important 采集器包选择 +OceanBase 监控同样依赖 `ext-lib` 目录下的 MySQL JDBC 驱动。 + +- OceanBase 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 OceanBase 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md index f15d3dc62b5..153ef72fcb4 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下。 - 重启 HertzBeat 服务。 +:::important 采集器包选择 +Oracle 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- Oracle 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 Oracle 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md index f931b561024..50b7da7a250 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md @@ -45,7 +45,7 @@ qemu-system-riscv64 \ ## 安装并配置支持 RISC-V 架构的 JDK -> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 21。 +> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 25。 **1. 下载 Temurin JDK** @@ -63,11 +63,11 @@ sudo apt install -y tar wget # 创建安装目录 sudo mkdir -p /usr/lib/jvm # 解压到系统目录 -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # 编辑环境变量，添加以下内容 sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # 使配置生效 @@ -88,6 +88,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > 注意事项： > -> 1. 请将 `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` 替换为您实际下载的 JDK 文件名。 +> 1. 请将 `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` 替换为您实际下载的 JDK 文件名。 > 2. 确保模拟器具备网络访问能力，以下载 HertzBeat。 > 3. 若遇到问题，请检查磁盘权限和 Java 环境路径配置。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md new file mode 100644 index 00000000000..1d26af32a47 --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -0,0 +1,75 @@ +--- +id: native-collector +title: Native 采集器指南 +sidebar_label: Native 采集器 +description: 说明 HertzBeat Native 采集器安装包适合什么场景、优缺点、限制和部署建议。 +--- + +## 什么场景适合使用 Native 采集器？ + +当你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动时，优先考虑 Native 采集器安装包。 + +比较适合 Native 采集器的场景包括： + +- HTTP、HTTPS、网站可用性、API 检查 +- 端口可用性、Ping、SSL 证书等网络探测 +- Redis、Zookeeper、Kafka 等非 JDBC 监控类型 + +## 为什么选择它？ + +相较 JVM 采集器安装包，Native 采集器安装包通常更适合以下诉求： + +- 启动更快 +- 常驻内存更低 +- 运行时更轻，不需要额外准备 bundled 或预装 JDK + +## 它的缺点和限制是什么？ + +Native 采集器并不是所有 JVM 采集器场景的无损替代。 + +- Native 安装包是平台相关的，必须选择与你操作系统和 CPU 架构匹配的包。 +- Native 采集器不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 +- 如果你的部署依赖 JVM 风格的运行时 classpath 扩展能力，仍然应该使用 JVM 采集器安装包。 + +## 哪些场景应该继续使用 JVM 采集器？ + +如果你的监控依赖外部 JDBC 驱动，请继续使用 JVM 采集器安装包，尤其包括： + +- MySQL，需要 `mysql-connector-j` +- OceanBase，同样依赖 MySQL JDBC 驱动 +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` + +## 安装包命名规则 + +JVM 采集器安装包仍然保持跨平台： + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +Native 采集器安装包按平台区分： + +- Linux 或 macOS：`apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows：`apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +例如： + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## 配置文件是否和 JVM 采集器一致？ + +Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/application.yml` 结构。 + +这意味着： + +- 采集器连接参数仍然在同一个位置修改 +- 虚拟线程相关配置仍然在同一个位置修改 +- Native 专用的启动调整通过代码在运行时生效，而不是长期维护第二份 `application.yml` + +## 推荐选择 + +- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 +- 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 + +具体安装步骤可参考 [通过安装包安装 HertzBeat](package-deploy)。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md index b34768ac984..3091205add9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: 安装包方式安装 :::tip Apache HertzBeat™ 支持在Linux Windows Mac系统安装运行，CPU支持X86/ARM64。 -由于1.6.0及以后版本使用 `Java 21` ，且安装包不再提供内置jdk的版本，参考以下情况使用新版Hertzbeat。 +当前分支默认使用 `Java 25`，且标准安装包不再提供内置 JDK。可参考以下情况使用 HertzBeat： -- 当你的服务器中默认环境变量为 `Java 21` 时，这一步你无需任何操作。 -- 当你的服务器中默认环境变量不为 `Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**没有**其他应用需要低版本 `Java` ，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并在搜索引擎搜索如何设置新的环境变量指向新的`Java 21`。 -- 当你的服务器中默认环境变量不为`Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**有**其他应用需要低版本 `Java` ，你不想更改环境变量，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并将解压后的文件夹重命名为`java`，复制到Hertzbeat的解压目录下。 +- 当你的服务器中默认环境变量为 `Java 25` 时，这一步无需任何操作。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**没有**其他应用需要低版本 `Java`，根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将环境变量指向新的 `Java 25`。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**有**其他应用需要低版本 `Java`，不希望修改全局环境变量，可根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将解压后的文件夹重命名为 `java`，复制到 HertzBeat 的解压目录下。 ::: @@ -63,11 +63,21 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 通过部署多个 HertzBeat Collector 可以实现数据的高可用，负载均衡和云边协同。 ::: +:::tip Native 采集器推荐 +如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。 + +在选择前，建议先阅读 [Native 采集器指南](native-collector) 了解它的限制和取舍。 +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. 下载安装包 - 从 [下载页面](/docs/download) 下载您系统环境对应的安装包版本 `apache-hertzbeat-collector-xxx-bin.tar.gz` + 按部署形态选择对应的采集器安装包： + - JVM 采集器安装包：`apache-hertzbeat-collector-xxx-bin.tar.gz` + - Linux 或 macOS 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Windows 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` + - 从 [下载页面](/docs/download) 下载 2. 设置配置文件 @@ -75,6 +85,10 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # 或 + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # 或 + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` 配置采集器的配置文件 `config/application.yml` 里面的 HertzBeat Server 连接 IP, 端口, 采集器名称(需保证唯一性)等参数。 @@ -100,14 +114,30 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 3. 启动 - 执行位于安装目录 hertzbeat-collector/bin/ 下的启动脚本 startup.sh, windows 环境下为 startup.bat - - ```shell - ./startup.sh - ``` + JVM 采集器安装包执行位于安装目录 `hertzbeat-collector/bin/` 下的启动脚本 `startup.sh`，Windows 环境下为 `startup.bat`；Linux 或 macOS 的 Native 采集器安装包执行 `./startup.sh`，Windows 的 Native 采集器安装包执行 `bin\\startup.bat` 4. 开始探索 HertzBeat Collector - 浏览器访问 [http://ip:1157/](http://ip:1157/) 即可开始探索使用，默认账户密码 admin/hertzbeat。 + 浏览器访问主 HertzBeat 服务 [http://manager-host:1157/](http://manager-host:1157/) 的概览页面，即可确认新采集器已注册。 + +:::important Native 采集器限制说明 +Native 采集器适合不依赖外部 JVM classpath 扩展的监控类型。 + +关于包选择、安装包命名和平台相关限制，详见 [Native 采集器指南](native-collector)。 + +基于 `ext-lib` 的 JDBC 驱动加载能力是 JVM 采集器的能力。Native 采集器当前不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 + +因此，凡是依赖外置 JDBC 驱动的监控类型，请使用 JVM 采集器，不要使用 Native 采集器。当前至少包括： + +- MySQL，需要 `mysql-connector-j` +- OceanBase，同样依赖 MySQL JDBC 驱动 +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` + +建议部署方式： + +- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型优先使用 Native 采集器 +- 需要 `ext-lib` 扩展驱动时使用 JVM 采集器 +::: **HAVE FUN** @@ -120,15 +150,15 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 1. 启动失败，需您提前准备JAVA运行环境 安装JAVA运行环境-可参考[官方网站](https://www.oracle.com/java/technologies/downloads/) - 要求：JAVA21环境 + 要求：JAVA25环境 下载JAVA安装包: [镜像站](https://mirrors.huaweicloud.com/openjdk/) 安装后命令行检查是否成功安装 ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md index 9723ac7357a..0031ecebd45 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md @@ -13,7 +13,7 @@ description: Apache HertzBeat 监控系统快速安装指南 - Docker、安装 ## 安装方式 -HertzBeat 提供四种安装选项： +HertzBeat 提供多种安装选项： 1. **Docker**（推荐）- 最快设置，生产就绪 2. **二进制包** - 传统部署，手动配置 @@ -58,12 +58,13 @@ HertzBeat 提供四种安装选项： #### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包`hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml`(可选) 3. 部署启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群(可选) - - 下载您系统环境对应采集器安装包`hertzbeat-collector-xx.tar.gz`到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) + - 如果你不需要 MySQL、OceanBase、Oracle、DB2 这类依赖 `ext-lib` JDBC 驱动的监控，优先选择 Native 采集器安装包，通常启动更快、内存更省。详见 [Native 采集器指南](native-collector)。 + - 下载您系统环境对应采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或匹配目标平台的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主HertzBeat服务的对外IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml @@ -78,15 +79,17 @@ HertzBeat 提供四种安装选项： manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` - - 浏览器访问主HertzBeat服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 + - Native 采集器的代价是安装包按平台区分、不支持运行时 `ext-lib` JDBC 加载，也不适合依赖 JVM 风格运行时 classpath 扩展的场景。详见 [Native 采集器指南](native-collector)。 + - 如果需要通过 `ext-lib` 加载 MySQL、OceanBase、Oracle、DB2 等外置 JDBC 驱动，请使用 JVM 采集器安装包 + - JVM 采集器安装包使用 `$ ./bin/startup.sh` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 + - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](package-deploy) #### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程`hertzbeat-startup`和前端工程`web-app` -2. 后端：需要`maven3+`, `java21`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 +2. 后端：需要`maven3+`, `java25`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 3. 前端：需要`nodejs npm angular-cli`环境，待本地后端启动后，在`web-app`目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` @@ -112,7 +115,7 @@ HertzBeat 提供四种安装选项： - 2 CPU 核心 - 4GB RAM - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ **支持系统：** Linux、macOS、Windows（通过 Docker 或 WSL） diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index ae69b63e0a9..020f5376b5f 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: 虚拟线程 description: 说明 HertzBeat 虚拟线程执行器的默认值、回滚开关和调优方式。 --- -HertzBeat 基于 JDK 21 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 +HertzBeat 基于 JDK 25 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 ## 1. 到哪里配置 diff --git a/home/sidebars.json b/home/sidebars.json index bda68ea497e..c40c129b177 100755 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -15,6 +15,7 @@ "start/docker-deploy", "start/docker-compose-deploy", "start/package-deploy", + "start/native-collector", { "type": "link", "label": "Install via Helm", diff --git a/home/src/components/StructuredData.js b/home/src/components/StructuredData.js index 97472f05e6a..cbeeb493187 100644 --- a/home/src/components/StructuredData.js +++ b/home/src/components/StructuredData.js @@ -55,7 +55,7 @@ export default function StructuredData() { "Cloud-edge collaboration for isolated networks", "Status page builder for service communication" ], - "softwareRequirements": "Docker 20.10+ or Java 21+", + "softwareRequirements": "Docker 20.10+ or Java 25+", "memoryRequirements": "4GB minimum, 8GB recommended", "processorRequirements": "2 CPU cores minimum", "storageRequirements": "10GB minimum" diff --git a/home/src/pages/faq.js b/home/src/pages/faq.js index cc6d74af086..cc45b8c4d58 100644 --- a/home/src/pages/faq.js +++ b/home/src/pages/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "What are HertzBeat's system requirements?", - answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 21+. Supported on Linux, macOS, Windows." + answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 25+. Supported on Linux, macOS, Windows." }, { question: "How do I upgrade HertzBeat?", diff --git a/home/src/pages/zh-cn/faq.js b/home/src/pages/zh-cn/faq.js index ae16a05b411..f422546dfca 100644 --- a/home/src/pages/zh-cn/faq.js +++ b/home/src/pages/zh-cn/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "HertzBeat 的系统要求是什么？", - answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 21+。支持系统：Linux、macOS、Windows。" + answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 25+。支持系统：Linux、macOS、Windows。" }, { question: "如何升级 HertzBeat？", diff --git a/home/static/llms-zh.txt b/home/static/llms-zh.txt index 0ae302d4691..8971f7a70f7 100644 --- a/home/static/llms-zh.txt +++ b/home/static/llms-zh.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU 核心 最低 - 4GB RAM 最低（推荐 8GB） - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ - 平台：Linux、macOS、Windows ## 支持的监控类型 diff --git a/home/static/llms.txt b/home/static/llms.txt index 18ab6d7bbf9..65ef12af761 100644 --- a/home/static/llms.txt +++ b/home/static/llms.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU cores minimum - 4GB RAM minimum (8GB recommended) - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ - Platforms: Linux, macOS, Windows ## Supported Monitoring Types diff --git a/pom.xml b/pom.xml index fdbdcd0960a..341a12c9066 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 1.8.0 - 21 + 25 ${java.version} ${java.version} 3.2.0 @@ -610,6 +610,10 @@ org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} + + + full + org.apache.maven.plugins diff --git a/script/assembly/collector/assembly-native.xml b/script/assembly/collector/assembly-native.xml new file mode 100644 index 00000000000..c341eaffd05 --- /dev/null +++ b/script/assembly/collector/assembly-native.xml @@ -0,0 +1,82 @@ + + + + ${native.package.id} + true + ${native.package.baseDirectory} + + ${native.package.format} + + + + + ${native.launcher.dir} + true + bin + 0755 + + + + src/main/resources + + application.yml + logback-spring.xml + banner.txt + + true + ${file.separator}config + + + + ../../ + ${file.separator} + + README.md + + + + ../../material/licenses/collector + ${file.separator} + + LICENSE + NOTICE + + + + ../../material/licenses/collector + licenses + + LICENSE-* + + + + + + + ${native.binary.source} + ${file.separator} + ${native.executable.packageName} + 0755 + + + diff --git a/script/assembly/collector/assembly.xml b/script/assembly/collector/assembly.xml index f4af4618f28..d02ee97e0cc 100644 --- a/script/assembly/collector/assembly.xml +++ b/script/assembly/collector/assembly.xml @@ -58,6 +58,9 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> banner.txt META-INF/** + + META-INF/spring.factories + true ${file.separator}config @@ -68,7 +71,7 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> target / - *.jar + ${project.build.finalName}.jar diff --git a/script/assembly/collector/bin-native-win/restart.bat b/script/assembly/collector/bin-native-win/restart.bat new file mode 100644 index 00000000000..50521a8d85b --- /dev/null +++ b/script/assembly/collector/bin-native-win/restart.bat @@ -0,0 +1,25 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +echo Restarting Apache HertzBeat ${project.artifactId} ... +call "%~dp0shutdown.bat" +timeout /t 2 /nobreak >nul +call "%~dp0startup.bat" +exit /b %ERRORLEVEL% diff --git a/script/assembly/collector/bin-native-win/shutdown.bat b/script/assembly/collector/bin-native-win/shutdown.bat new file mode 100644 index 00000000000..eb1dc08e316 --- /dev/null +++ b/script/assembly/collector/bin-native-win/shutdown.bat @@ -0,0 +1,49 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set SERVER_PORT=1159 + +set PID= +if exist "%PID_FILE%" ( + set /p PID=<"%PID_FILE%" +) + +if not defined PID ( + for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$conn = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1; if ($conn) { $conn.OwningProcess }"') do set PID=%%i +) + +if not defined PID ( + echo Apache HertzBeat %SERVER_NAME% is already stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %PID% -ErrorAction SilentlyContinue) { Stop-Process -Id %PID% -Force; exit 0 } exit 1" +del /q "%PID_FILE%" >nul 2>&1 + +echo Shutdown Apache HertzBeat %SERVER_NAME% Success! +exit /b 0 diff --git a/script/assembly/collector/bin-native-win/startup.bat b/script/assembly/collector/bin-native-win/startup.bat new file mode 100644 index 00000000000..33bfec4f48c --- /dev/null +++ b/script/assembly/collector/bin-native-win/startup.bat @@ -0,0 +1,102 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} +set BINARY_NAME=${native.executable.packageName} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set CONF_DIR=%DEPLOY_DIR%\config +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set APP_PATH=%DEPLOY_DIR%\%BINARY_NAME% +set SERVER_PORT=1159 +set STDOUT_LOG=%LOGS_DIR%\startup.out.log +set STDERR_LOG=%LOGS_DIR%\startup.err.log + +if "%1"=="status" goto status + +if not exist "%APP_PATH%" ( + echo ERROR: native executable not found: %APP_PATH% + exit /b 1 +) + +if not exist "%LOGS_DIR%" ( + mkdir "%LOGS_DIR%" +) + +if exist "%PID_FILE%" ( + set /p RUNNING_PID=<"%PID_FILE%" + powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" + if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% already started! + echo PID: %RUNNING_PID% + exit /b 1 + ) + del /q "%PID_FILE%" >nul 2>&1 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$portInUse = Get-NetTCPConnection -State Listen -LocalPort %SERVER_PORT% -ErrorAction SilentlyContinue; if ($portInUse) { exit 0 } exit 1" +if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% port %SERVER_PORT% is already used! + exit /b 1 +) + +echo You can review logs at hertzbeat\logs +echo Starting the HertzBeat %SERVER_NAME% ... + +for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$p = Start-Process -FilePath ''%APP_PATH%'' -ArgumentList ''--spring.config.location=%CONF_DIR%\'' -RedirectStandardOutput ''%STDOUT_LOG%'' -RedirectStandardError ''%STDERR_LOG%'' -PassThru; $p.Id"') do set APP_PID=%%i + +if not defined APP_PID ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + exit /b 1 +) + +>"%PID_FILE%" echo %APP_PID% + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$deadline = (Get-Date).AddSeconds(30); do { Start-Sleep -Seconds 1; if (-not (Get-Process -Id %APP_PID% -ErrorAction SilentlyContinue)) { exit 1 } $listening = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Where-Object { $_.OwningProcess -eq %APP_PID% }; if ($listening) { exit 0 } } while ((Get-Date) -lt $deadline); exit 0" +if errorlevel 1 ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + del /q "%PID_FILE%" >nul 2>&1 + exit /b 1 +) + +echo Service Start Success! +echo Service PID: %APP_PID% +exit /b 0 + +:status +if not exist "%PID_FILE%" ( + echo The HertzBeat %SERVER_NAME% is stopped + exit /b 0 +) + +set /p RUNNING_PID=<"%PID_FILE%" +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" +if errorlevel 1 ( + echo The HertzBeat %SERVER_NAME% is stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +echo The HertzBeat %SERVER_NAME% is running...! +echo PID: %RUNNING_PID% +exit /b 0 diff --git a/script/assembly/collector/bin-native/restart.sh b/script/assembly/collector/bin-native/restart.sh new file mode 100644 index 00000000000..af640ba89f1 --- /dev/null +++ b/script/assembly/collector/bin-native/restart.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +startTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Now!\033[0m" +./shutdown.sh +echo +sleep 2 +echo +./startup.sh +endTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Success！Spend $((endTime - startTime)) seconds \033[0m" diff --git a/script/assembly/collector/bin-native/shutdown.sh b/script/assembly/collector/bin-native/shutdown.sh new file mode 100644 index 00000000000..1c0478edbac --- /dev/null +++ b/script/assembly/collector/bin-native/shutdown.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +PID="$(find_running_pid)" +if [ -z "$PID" ]; then + echo "Apache HertzBeat ${SERVER_NAME} is already stopped" + rm -f "$PID_FILE" + exit 0 +fi + +kill "$PID" +for _ in $(seq 1 30); do + if ! kill -0 "$PID" 2>/dev/null; then + rm -f "$PID_FILE" + echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" + exit 0 + fi + sleep 1 +done + +kill -9 "$PID" 2>/dev/null +rm -f "$PID_FILE" +echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" diff --git a/script/assembly/collector/bin-native/startup.sh b/script/assembly/collector/bin-native/startup.sh new file mode 100644 index 00000000000..bd57563fad8 --- /dev/null +++ b/script/assembly/collector/bin-native/startup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +BIN_DIR="$(pwd)" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" +SERVER_PORT=1159 + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +RUNNING_PID="$(find_running_pid)" +if [ "$1" = "status" ]; then + if [ -n "$RUNNING_PID" ]; then + echo "The HertzBeat $SERVER_NAME is running...!" + echo "PID: $RUNNING_PID" + else + echo "The HertzBeat $SERVER_NAME is stopped" + fi + exit 0 +fi + +if [ ! -x "$APP_PATH" ]; then + echo "ERROR: native executable not found: $APP_PATH" + exit 1 +fi + +if [ -n "$RUNNING_PID" ]; then + echo "ERROR: The HertzBeat $SERVER_NAME already started!" + echo "PID: $RUNNING_PID" + exit 1 +fi + +mkdir -p "$LOGS_DIR" + +if command -v lsof >/dev/null 2>&1; then + SERVER_PORT_COUNT="$(lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | wc -l)" + if [ "$SERVER_PORT_COUNT" -gt 0 ]; then + echo "ERROR: The HertzBeat $SERVER_NAME port $SERVER_PORT is already used!" + exit 1 + fi +fi + +echo "You can review logs at hertzbeat/logs" +echo "Starting the HertzBeat $SERVER_NAME ..." +nohup "$APP_PATH" --spring.config.location="$CONF_DIR/" >"$LOGS_DIR/startup.log" 2>&1 & +APP_PID=$! +echo "$APP_PID" >"$PID_FILE" + +COUNT=0 +while [ $COUNT -lt 30 ]; do + sleep 1 + if ! kill -0 "$APP_PID" 2>/dev/null; then + echo "ERROR: Service start failed, check $LOGS_DIR/startup.log" + rm -f "$PID_FILE" + exit 1 + fi + if command -v lsof >/dev/null 2>&1 && lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | grep -q "$APP_PID"; then + break + fi + COUNT=$((COUNT + 1)) +done + +echo "Service Start Success!" +echo "Service PID: $APP_PID" diff --git a/script/assembly/collector/bin/startup.sh b/script/assembly/collector/bin/startup.sh index 1d2db5f9c27..dbdab8c581b 100644 --- a/script/assembly/collector/bin/startup.sh +++ b/script/assembly/collector/bin/startup.sh @@ -103,7 +103,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/assembly/server/bin/startup.sh b/script/assembly/server/bin/startup.sh index ebf25d7ae30..45bff0dc90f 100644 --- a/script/assembly/server/bin/startup.sh +++ b/script/assembly/server/bin/startup.sh @@ -107,7 +107,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/ci/github-actions/setup-deps/action.yml b/script/ci/github-actions/setup-deps/action.yml index 0e1698fe720..3ad4560a589 100644 --- a/script/ci/github-actions/setup-deps/action.yml +++ b/script/ci/github-actions/setup-deps/action.yml @@ -21,11 +21,11 @@ description: Install host system dependencies (with mvnd) runs: using: composite steps: - - name: Set up JDK 21 + - name: Set up JDK 25 uses: actions/setup-java@v4 with: distribution: "zulu" - java-version: 21 + java-version: 25 - name: Install mvnd shell: bash @@ -40,4 +40,4 @@ runs: - name: Verify mvnd installation shell: bash - run: mvnd --version \ No newline at end of file + run: mvnd --version diff --git a/script/docker/collector/Dockerfile b/script/docker/collector/Dockerfile index a6f89d8c89d..5a33eb5b392 100644 --- a/script/docker/collector/Dockerfile +++ b/script/docker/collector/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" @@ -26,7 +26,7 @@ RUN sed -i 's#http://#https://#g' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && apt-get install -y openssh-server RUN mkdir /var/run/sshd -ADD apache-hertzbeat-collector-*-bin.tar.gz /opt/ +ADD apache-hertzbeat-collector-${VERSION}-bin.tar.gz /opt/ ENV JAVA_OPTS "" ENV TZ=Asia/Shanghai diff --git a/script/docker/collector/build.sh b/script/docker/collector/build.sh index 278df32c783..03f93312b0d 100644 --- a/script/docker/collector/build.sh +++ b/script/docker/collector/build.sh @@ -21,13 +21,20 @@ cd `dirname $0` CURRENT_DIR=`pwd` # cd dist dir cd ../../../dist -# auto detect hertzbeat version -VERSION=`ls apache-hertzbeat-collector-*-bin.tar.gz| awk -F"-" '{print $4}'` +# auto detect the JVM collector package version +PACKAGE_FILE=$(find . -maxdepth 1 -name 'apache-hertzbeat-collector-*-bin.tar.gz' \ + ! -name 'apache-hertzbeat-collector-native-*-bin.tar.gz' | head -n 1) +VERSION=$(basename "$PACKAGE_FILE" | sed -e 's/^apache-hertzbeat-collector-//' -e 's/-bin\.tar\.gz$//') # use the version param if [ -n "$1" ]; then VERSION="$1"; fi +if [ -z "$VERSION" ]; then + echo "Can not find the JVM collector package under dist/. Build apache-hertzbeat-collector-{version}-bin.tar.gz first." + exit 1 +fi + # docker compile context CONTEXT_DIR=`pwd` diff --git a/script/docker/server/Dockerfile b/script/docker/server/Dockerfile index 773c21c0d15..6427e6300b9 100644 --- a/script/docker/server/Dockerfile +++ b/script/docker/server/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" diff --git a/script/ext-lib/README b/script/ext-lib/README index 5898fde6b91..0afb3b3bbec 100644 --- a/script/ext-lib/README +++ b/script/ext-lib/README @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder for JVM-based server / collector packages, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +jcc-11.5.9.0.jar +Note: + +- `ext-lib` is loaded by the JVM server package and the JVM collector package. +- The native collector package does not support loading external JDBC driver jars from `ext-lib` at runtime. +- If you need MySQL, OceanBase, Oracle, or DB2 monitoring with external JDBC drivers, use the JVM collector package. From 3809599a8c98093d0e6ac7d2e81af615559bc66b Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 23:29:57 +0800 Subject: [PATCH 11/12] build: fix native release packaging workflow --- .github/workflows/collector-native-build.yml | 22 +++++-------------- .../hertzbeat-collector-collector/pom.xml | 9 ++++++++ home/docs/community/how-to-release.md | 6 +++-- home/docs/start/native-collector.md | 5 +++++ .../current/community/how-to-release.md | 6 +++-- .../current/start/native-collector.md | 5 +++++ 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/collector-native-build.yml b/.github/workflows/collector-native-build.yml index 97a14b3e8fd..d44877dba8f 100644 --- a/.github/workflows/collector-native-build.yml +++ b/.github/workflows/collector-native-build.yml @@ -15,37 +15,26 @@ # specific language governing permissions and limitations # under the License. -name: Collector Native CI +name: Collector Native Release + +run-name: Native collector release build (${{ github.ref_name }}) on: workflow_dispatch: - push: - branches: [ action* ] - paths: - - '.github/workflows/collector-native-build.yml' - - 'pom.xml' - - 'hertzbeat-collector/**' - - 'script/assembly/collector/**' - pull_request: - branches: [ master, dev ] - paths: - - '.github/workflows/collector-native-build.yml' - - 'pom.xml' - - 'hertzbeat-collector/**' - - 'script/assembly/collector/**' jobs: build-native-collector: name: Native collector (${{ matrix.platform }}) permissions: contents: read + timeout-minutes: 120 runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - platform: linux-amd64 - runner: ubuntu-latest + runner: ubuntu-24.04 archive_ext: tar.gz - platform: linux-arm64 runner: ubuntu-24.04-arm @@ -97,3 +86,4 @@ jobs: with: name: apache-hertzbeat-collector-native-${{ matrix.platform }} path: ${{ steps.package.outputs.archive }} + retention-days: 14 diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index d40bdf60abb..e4f32ac721f 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -412,6 +412,15 @@ org.graalvm.buildtools native-maven-plugin 0.11.4 + + + build-native-image + package + + compile-no-fork + + + ${native.image.name} diff --git a/home/docs/community/how-to-release.md b/home/docs/community/how-to-release.md index c92feedccb6..9c92b940958 100644 --- a/home/docs/community/how-to-release.md +++ b/home/docs/community/how-to-release.md @@ -211,13 +211,15 @@ mvn clean install mvn clean package -Pcluster ``` -> Build the native collector binaries with GraalVM JDK 25 and `native-image` on matching GitHub-hosted runners or equivalent local hosts +> Build the native collector binary for the current host with GraalVM JDK 25 and `native-image` ```shell mvn clean package -pl hertzbeat-collector-collector -am -Pnative ``` -> The repository workflow `.github/workflows/collector-native-build.yml` can build and upload all supported native collector packages on GitHub-hosted runners. Release signing and final publishing remain a release manager responsibility. +> The repository workflow `.github/workflows/collector-native-build.yml` is a release helper, not a regular PR or push CI workflow. +> +> It is intentionally manual-only because multi-platform native builds are relatively slow and consume scarce Linux ARM, macOS, and Windows runners. During release preparation, open the Actions page, select `Collector Native Release`, run it from the release branch or tag, and then download the uploaded artifacts for signing and publishing. The release package are here: diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md index 9f16f217b15..e1affbbc040 100644 --- a/home/docs/start/native-collector.md +++ b/home/docs/start/native-collector.md @@ -72,4 +72,9 @@ That means: - Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring. - Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. +## How are the official multi-platform packages built? + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` builds a native collector package for the current host only. +- The official Linux, macOS, and Windows native release packages are produced by manually running the `Collector Native Release` GitHub Actions workflow during release preparation, not on every push or pull request. + For package deployment steps, refer to [Install HertzBeat via Package](package-deploy). diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md index e5a8594d134..7415ca15f9e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md @@ -211,13 +211,15 @@ mvn clean install mvn clean package -Pcluster ``` -> 在匹配的平台 GitHub Hosted Runner 或对应本地主机上，使用带 `native-image` 的 GraalVM JDK 25 构建 Native 采集器安装包 +> 使用带 `native-image` 的 GraalVM JDK 25，为当前宿主机构建 Native 采集器安装包 ```shell mvn clean package -pl hertzbeat-collector-collector -am -Pnative ``` -> 仓库中的 `.github/workflows/collector-native-build.yml` 可以在 GitHub Hosted Runner 上构建并上传全部受支持平台的 Native 采集器安装包。正式发布时的签名与最终上传仍由 release manager 负责。 +> 仓库中的 `.github/workflows/collector-native-build.yml` 是 release 辅助工作流，不参与日常 PR 或 push 的常规 CI。 +> +> 之所以只保留手动触发，是因为跨平台 Native 构建耗时更长，也会占用相对稀缺的 Linux ARM、macOS 和 Windows Runner。准备发版时，请在 GitHub Actions 页面选择 `Collector Native Release`，基于 release 分支或 tag 手动触发，然后下载上传的产物用于签名和发布。 生成的二进制包在: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md index 1d26af32a47..6ef2037be31 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -72,4 +72,9 @@ Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/appli - 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包。 - 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 +## 官方多平台安装包是怎么构建的？ + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` 只会为当前宿主机构建一个 Native 采集器安装包。 +- 官方发布使用的 Linux、macOS、Windows Native 安装包，会在发布准备阶段手动触发 `Collector Native Release` GitHub Actions 工作流来生成，而不是在每次 push 或 pull request 时自动构建。 + 具体安装步骤可参考 [通过安装包安装 HertzBeat](package-deploy)。 From a5b406f20ab8a48d5c621f4731b0a361299d5355 Mon Sep 17 00:00:00 2001 From: Logic Date: Thu, 12 Mar 2026 23:41:37 +0800 Subject: [PATCH 12/12] chore: add license header for spring factories --- .../src/main/resources/META-INF/spring.factories | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories index 16c40f8c6f3..731da64e615 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories @@ -1,2 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Intentionally left blank. # Native collector defaults are applied from Collector.main.