diff --git a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java index 7495cf83eb9..41080c79851 100644 --- a/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java +++ b/hertzbeat-ai/src/main/java/org/apache/hertzbeat/ai/tools/impl/MonitorToolsImpl.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.ai.config.McpContextHolder; import org.apache.hertzbeat.manager.pojo.dto.MonitorDto; +import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo; import org.apache.hertzbeat.manager.service.MonitorService; import org.apache.hertzbeat.manager.service.AppService; import org.apache.hertzbeat.ai.utils.UtilityClass; @@ -32,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.apache.hertzbeat.common.entity.manager.Monitor; import org.apache.hertzbeat.common.entity.manager.Param; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import java.util.ArrayList; import java.util.List; @@ -280,7 +280,9 @@ public String addMonitor( // Validate that all required parameters for this monitor type are provided try { - MonitorDto monitorDto = MonitorDto.builder().monitor(monitor).params(paramList).build(); + MonitorDto monitorDto = new MonitorDto(); + monitorDto.setMonitor(monitor); + monitorDto.setParams(paramList); monitorService.validate(monitorDto, false); } catch (IllegalArgumentException argumentException) { if (argumentException.getMessage().contains("required")) { @@ -456,7 +458,7 @@ public String getMonitorParams( } // Get parameter definitions from app service - List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); + List paramDefines = appService.getAppParamDefines(app.toLowerCase().trim()); if (paramDefines == null || paramDefines.isEmpty()) { return String.format("No parameter definitions found for monitor type '%s'. " @@ -468,7 +470,7 @@ public String getMonitorParams( response.append(String.format("Parameter Definitions for Monitor Type '%s' (Total: %d):\n\n", app, paramDefines.size())); - for (ParamDefine paramDefine : paramDefines) { + for (ParamDefineInfo paramDefine : paramDefines) { response.append("• Field: ").append(paramDefine.getField()).append("\n"); // Add display name if available diff --git a/hertzbeat-alerter/pom.xml b/hertzbeat-alerter/pom.xml index 7af87836e9d..c3a3890d2c1 100644 --- a/hertzbeat-alerter/pom.xml +++ b/hertzbeat-alerter/pom.xml @@ -40,6 +40,11 @@ hertzbeat-common-core provided + + org.apache.hertzbeat + hertzbeat-common-spring + provided + org.apache.hertzbeat diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java index 23426ddcf06..b8abee9edf7 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/AlerterWorkerPool.java @@ -18,12 +18,20 @@ package org.apache.hertzbeat.alert; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -31,16 +39,25 @@ */ @Component @Slf4j -public class AlerterWorkerPool { +public class AlerterWorkerPool implements DisposableBean { private ThreadPoolExecutor workerExecutor; - private ThreadPoolExecutor notifyExecutor; - private ThreadPoolExecutor logWorkerExecutor; + private ManagedExecutor notifyExecutor; + private ManagedExecutor logWorkerExecutor; + private Map notifyChannelPermits; + private int notifyMaxConcurrentPerChannel; public AlerterWorkerPool() { + this(VirtualThreadProperties.defaults()); + } + + @Autowired + public AlerterWorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; initWorkExecutor(); - initNotifyExecutor(); - initLogWorkerExecutor(); + initNotifyExecutor(properties); + initLogWorkerExecutor(properties); } private void initWorkExecutor() { @@ -61,16 +78,32 @@ private void initWorkExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initNotifyExecutor() { + private void initNotifyExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter notifyExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.enabled()) { + VirtualThreadProperties.AlerterProperties alerterProperties = properties.alerter(); + VirtualThreadProperties.PoolProperties notifyProperties = alerterProperties.notifyPool(); + notifyMaxConcurrentPerChannel = Math.max(1, alerterProperties.notifyMaxConcurrentPerChannel()); + notifyChannelPermits = new ConcurrentHashMap<>(8); + notifyExecutor = ManagedExecutors.newVirtualExecutor("notify-worker", "notify-worker-", + notifyProperties.mode(), notifyProperties.maxConcurrentJobs(), handler); + return; + } + notifyMaxConcurrentPerChannel = 0; + notifyChannelPermits = null; + notifyExecutor = ManagedExecutors.wrap("notify-worker", createLegacyNotifyExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyNotifyExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter notifyExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("notify-worker-%d") .build(); - notifyExecutor = new ThreadPoolExecutor(6, + return new ThreadPoolExecutor(6, 6, 10, TimeUnit.SECONDS, @@ -79,16 +112,27 @@ private void initNotifyExecutor() { new ThreadPoolExecutor.AbortPolicy()); } - private void initLogWorkerExecutor() { + private void initLogWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Alerter logWorkerExecutor has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties logWorkerProperties = properties.alerter().logWorker(); + logWorkerExecutor = ManagedExecutors.newQueuedVirtualExecutor("alerter-log-worker", "log-worker-", + logWorkerProperties.maxConcurrentJobs(), logWorkerProperties.queueCapacity(), handler); + return; + } + logWorkerExecutor = ManagedExecutors.wrap("alerter-log-worker", createLegacyLogWorkerExecutor(handler)); + } + + private ThreadPoolExecutor createLegacyLogWorkerExecutor(Thread.UncaughtExceptionHandler handler) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("Alerter logWorkerExecutor has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("log-worker-%d") .build(); - logWorkerExecutor = new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, + return new ThreadPoolExecutor(10, 10, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1000), threadFactory, new ThreadPoolExecutor.AbortPolicy()); @@ -113,6 +157,41 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { notifyExecutor.execute(runnable); } + /** + * Executes the given runnable task using the notify executor with per-channel concurrency control. + * + * @param channelType notification channel type + * @param runnable the task to be executed + * @throws RejectedExecutionException if the task cannot be accepted for execution + */ + public void executeNotify(byte channelType, Runnable runnable) throws RejectedExecutionException { + if (notifyChannelPermits == null) { + notifyExecutor.execute(runnable); + return; + } + Semaphore semaphore = notifyChannelPermits.computeIfAbsent(channelType, + key -> new Semaphore(notifyMaxConcurrentPerChannel)); + if (!semaphore.tryAcquire()) { + throw new RejectedExecutionException( + "notify-worker rejected task because channel concurrency limit was reached for type " + channelType); + } + boolean submitted = false; + try { + notifyExecutor.execute(() -> { + try { + runnable.run(); + } finally { + semaphore.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + semaphore.release(); + } + } + } + /** * Executes the given runnable task using the logWorkerExecutor. * @@ -122,4 +201,11 @@ public void executeNotify(Runnable runnable) throws RejectedExecutionException { public void executeLogJob(Runnable runnable) throws RejectedExecutionException { logWorkerExecutor.execute(runnable); } + + @Override + public void destroy() { + workerExecutor.shutdownNow(); + notifyExecutor.close(); + logWorkerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java index 978fb3d234a..49e9004e77d 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleScheduler.java @@ -25,12 +25,19 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; @@ -40,18 +47,33 @@ */ @Slf4j @Component -public class PeriodicAlertRuleScheduler implements CommandLineRunner { +public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean { private final MetricsPeriodicAlertCalculator metricsCalculator; private final LogPeriodicAlertCalculator logCalculator; private final AlertDefineDao alertDefineDao; private final ScheduledExecutorService scheduledExecutor; - private final Map> scheduledFutures; + private final ExecutorService periodicExecutor; + private final Semaphore periodicPermits; + private final boolean virtualThreadsEnabled; + private final Map scheduledTasks; public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, LogPeriodicAlertCalculator logCalculator, AlertDefineDao alertDefineDao) { + this(metricsCalculator, logCalculator, alertDefineDao, VirtualThreadProperties.defaults()); + } + + @Autowired + public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator, + LogPeriodicAlertCalculator logCalculator, + AlertDefineDao alertDefineDao, + VirtualThreadProperties virtualThreadProperties) { this.metricsCalculator = metricsCalculator; this.logCalculator = logCalculator; this.alertDefineDao = alertDefineDao; + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("Scheduled periodic alert threshold has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled periodic alert threshold has uncaughtException."); @@ -61,17 +83,27 @@ public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculat .setNameFormat("periodic-alert-threshold-worker-%d") .build(); this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory); - this.scheduledFutures = new ConcurrentHashMap<>(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.virtualThreadsEnabled = properties.enabled(); + int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs()); + this.periodicExecutor = virtualThreadsEnabled + ? Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("periodic-alert-task-", 0) + .uncaughtExceptionHandler(handler) + .factory()) + : null; + this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null; + this.scheduledTasks = new ConcurrentHashMap<>(); } public void cancelSchedule(Long ruleId) { if (ruleId == null) { return; } - ScheduledFuture future = scheduledFutures.get(ruleId); - if (future != null) { - future.cancel(true); - scheduledFutures.remove(ruleId); + ScheduledTaskState state = scheduledTasks.remove(ruleId); + if (state != null) { + state.cancel(); } } @@ -83,14 +115,12 @@ public void updateSchedule(AlertDefine rule) { cancelSchedule(rule.getId()); if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) || rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate(() -> { - if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { - metricsCalculator.calculate(rule); - } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { - logCalculator.calculate(rule); - } - }, 0, rule.getPeriod(), java.util.concurrent.TimeUnit.SECONDS); - scheduledFutures.put(rule.getId(), future); + ScheduledTaskState state = new ScheduledTaskState(rule); + ScheduledFuture future = scheduledExecutor.scheduleAtFixedRate( + virtualThreadsEnabled ? state::trigger : () -> executeRule(rule), + 0, rule.getPeriod(), TimeUnit.SECONDS); + state.setScheduledFuture(future); + scheduledTasks.put(rule.getId(), state); } } @@ -106,4 +136,107 @@ public void run(String... args) throws Exception { updateSchedule(rule); } } + + @Override + public void destroy() { + scheduledTasks.values().forEach(ScheduledTaskState::cancel); + scheduledTasks.clear(); + scheduledExecutor.shutdownNow(); + if (periodicExecutor != null) { + periodicExecutor.shutdownNow(); + } + } + + private void executeRule(AlertDefine rule) { + if (rule.getType().equals(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC)) { + metricsCalculator.calculate(rule); + } else if (rule.getType().equals(LOG_ALERT_THRESHOLD_TYPE_PERIODIC)) { + logCalculator.calculate(rule); + } + } + + private final class ScheduledTaskState { + + private final AlertDefine rule; + private ScheduledFuture scheduledFuture; + private Future runningFuture; + private boolean running; + private boolean pending; + private boolean cancelled; + + private ScheduledTaskState(AlertDefine rule) { + this.rule = rule; + } + + private synchronized void setScheduledFuture(ScheduledFuture scheduledFuture) { + this.scheduledFuture = scheduledFuture; + } + + private synchronized void trigger() { + if (cancelled) { + return; + } + if (running) { + pending = true; + return; + } + running = true; + submitLocked(); + } + + private synchronized void cancel() { + cancelled = true; + pending = false; + ScheduledFuture periodicFuture = scheduledFuture; + Future currentFuture = runningFuture; + if (periodicFuture != null) { + periodicFuture.cancel(true); + } + if (currentFuture != null) { + currentFuture.cancel(true); + } + } + + private void submitLocked() { + try { + runningFuture = periodicExecutor.submit(() -> { + boolean permitAcquired = false; + try { + periodicPermits.acquire(); + permitAcquired = true; + if (!Thread.currentThread().isInterrupted()) { + executeRule(rule); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e); + } finally { + if (permitAcquired) { + periodicPermits.release(); + } + onComplete(); + } + }); + } catch (RuntimeException e) { + running = false; + throw e; + } + } + + private synchronized void onComplete() { + runningFuture = null; + if (cancelled) { + running = false; + pending = false; + return; + } + if (!pending) { + running = false; + return; + } + pending = false; + submitLocked(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java index 1e245631d4f..e85b145355f 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluator.java @@ -20,6 +20,11 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.apache.hertzbeat.alert.util.AlertTemplateUtil; @@ -32,10 +37,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * Alarm Evaluator - Final alarm logic trigger @@ -47,36 +48,48 @@ */ @Component @Slf4j -public class AlarmEvaluator { +public class AlarmEvaluator implements DisposableBean { private static final String WINDOW_START_TIME = "window_start_time"; private static final String WINDOW_END_TIME = "window_end_time"; private static final String MATCHING_LOGS_COUNT = "matching_logs_count"; private final AlarmCommonReduce alarmCommonReduce; - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce) { + this(alarmCommonReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmEvaluator(AlarmCommonReduce alarmCommonReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmCommonReduce = alarmCommonReduce; - initAlarmEvaluator(); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initAlarmEvaluator(properties); } - public void initAlarmEvaluator() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + public ManagedExecutor initAlarmEvaluator(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().windowEvaluator(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-window-evaluator", "alerter-window-evaluator-", + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-window-evaluator", new java.util.concurrent.ThreadPoolExecutor(2, 10, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } public void sendAndProcessWindowData(WindowAggregator.WindowData windowData) { @@ -314,4 +327,9 @@ private void addLogEntryToMap(LogEntry logEntry, Map context) { } } } -} \ No newline at end of file + + @Override + public void destroy() { + workerExecutor.close(); + } +} diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java index 6d9207216b7..b294e817cd8 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatch.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterWorkerPool; import org.apache.hertzbeat.alert.config.AlertSseManager; @@ -121,14 +122,27 @@ public void dispatchAlarm(GroupAlert groupAlert) { } private void sendNotify(GroupAlert alert) { - matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> workerPool.executeNotify(() -> rule.getReceiverId() - .forEach(receiverId -> { - try { - sendNoticeMsg(getOneReceiverById(receiverId), - getOneTemplateById(rule.getTemplateId()), alert); - } catch (AlertNoticeException e) { - log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); - } - })))); + matchNoticeRulesByAlert(alert).ifPresent(noticeRules -> noticeRules.forEach(rule -> { + NoticeTemplate noticeTemplate = getOneTemplateById(rule.getTemplateId()); + rule.getReceiverId().forEach(receiverId -> { + NoticeReceiver receiver = getOneReceiverById(receiverId); + if (receiver == null || receiver.getType() == null) { + log.warn("DispatchTask skip invalid receiver, receiverId: {}, alertId: {}", receiverId, alert.getId()); + return; + } + try { + workerPool.executeNotify(receiver.getType(), () -> { + try { + sendNoticeMsg(receiver, noticeTemplate, alert); + } catch (AlertNoticeException e) { + log.warn("DispatchTask sendNoticeMsg error, message: {}", e.getMessage()); + } + }); + } catch (RejectedExecutionException e) { + log.warn("DispatchTask rejected notify task, receiverId: {}, type: {}, message: {}", + receiverId, receiver.getType(), e.getMessage()); + } + }); + })); } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java index 0ef336921ac..75f18f9ee37 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduce.java @@ -18,16 +18,16 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.util.List; import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; /** @@ -35,33 +35,45 @@ */ @Service @Slf4j -public class AlarmCommonReduce { +public class AlarmCommonReduce implements DisposableBean { private final AlarmGroupReduce alarmGroupReduce; - - private ThreadPoolExecutor workerExecutor; + + private final ManagedExecutor workerExecutor; public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) { - initWorkExecutor(); + this(alarmGroupReduce, VirtualThreadProperties.defaults()); + } + + @Autowired + public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, VirtualThreadProperties virtualThreadProperties) { this.alarmGroupReduce = alarmGroupReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = initWorkExecutor(properties); } - private void initWorkExecutor() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("alerter-reduce-worker has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("alerter-reduce-worker-%d") - .build(); - workerExecutor = new ThreadPoolExecutor(2, + private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("alerter-reduce-worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + if (properties.enabled()) { + VirtualThreadProperties.QueueProperties queueProperties = properties.alerter().reduce(); + return ManagedExecutors.newQueuedVirtualExecutor("alerter-reduce-worker", "alerter-reduce-worker-", + queueProperties.maxConcurrentJobs(), queueProperties.queueCapacity(), handler); + } + return ManagedExecutors.wrap("alerter-reduce-worker", new java.util.concurrent.ThreadPoolExecutor(2, 2, 10, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - threadFactory, - new ThreadPoolExecutor.AbortPolicy()); + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(), + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) + .setDaemon(true) + .setNameFormat("alerter-reduce-worker-%d") + .build(), + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy())); } @@ -103,10 +115,6 @@ Runnable reduceAlarmTask(SingleAlert alert) { * Fingerprint is based on labels excluding timestamp related fields */ private String generateAlertFingerprint(Map labels) { - // Remove timestamp related fields - labels.remove("timestamp"); - labels.remove("start_at"); - labels.remove("active_at"); return labels.entrySet().stream() .filter(e -> !"timestamp".equals(e.getKey()) && !"starts_at".equals(e.getKey()) && !"actives_at".equals(e.getKey()) @@ -116,4 +124,9 @@ private String generateAlertFingerprint(Map labels) { .map(e -> e.getKey() + ":" + e.getValue()) .collect(Collectors.joining(",")); } + + @Override + public void destroy() { + workerExecutor.close(); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 94c6cb873e2..597ef9a58a6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -26,17 +26,22 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -45,7 +50,7 @@ */ @Component @Slf4j -public class AlarmGroupReduce { +public class AlarmGroupReduce implements DisposableBean { /** * Default initial group wait time 30s @@ -88,16 +93,60 @@ public class AlarmGroupReduce { */ private final Map groupCacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService workerExecutor; + + private final ScheduledDispatchTask checkTask; + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) { + this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties) { + this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true); + } + + AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties virtualThreadProperties, boolean autoStart) { this.alarmInhibitReduce = alarmInhibitReduce; this.groupDefines = new ConcurrentHashMap<>(8); this.groupCacheMap = new ConcurrentHashMap<>(8); + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.scheduledExecutor = createScheduler(); + this.workerExecutor = createVirtualExecutor(properties); + this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups); List groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue(); refreshGroupDefines(groupConverges); - startCheckAndSendGroups(); + if (autoStart) { + startCheckAndSendGroups(); + } } private void startCheckAndSendGroups() { + scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCheckAndSendGroups() { + checkTask.dispatch(); + } + + void beforeCheckAndSendGroupsRun() { + } + + @Override + public void destroy() { + scheduledExecutor.shutdownNow(); + if (workerExecutor != null) { + workerExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Check alarm groups calculate has uncaughtException."); @@ -106,21 +155,36 @@ private void startCheckAndSendGroups() { .setDaemon(true) .setNameFormat("alarm-group-calculate-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - long now = System.currentTimeMillis(); - groupCacheMap.forEach((groupKey, cache) -> { - if (shouldSendGroup(cache, now)) { - sendGroupAlert(cache); - cache.setLastSendTime(now); - cache.getAlertFingerprints().clear(); - } - }); - } catch (Exception e) { - log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); - } - }, 10000, CHECK_INTERVAL, java.util.concurrent.TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createVirtualExecutor(VirtualThreadProperties properties) { + if (!properties.enabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("alarm-group-calculate-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Check alarm groups calculate worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCheckAndSendGroups() { + beforeCheckAndSendGroupsRun(); + try { + long now = System.currentTimeMillis(); + groupCacheMap.forEach((groupKey, cache) -> { + if (shouldSendGroup(cache, now)) { + sendGroupAlert(cache); + cache.setLastSendTime(now); + cache.getAlertFingerprints().clear(); + } + }); + } catch (Exception e) { + log.error("Check alarm groups calculate has exception.: {}", e.getMessage(), e); + } } /** @@ -339,4 +403,63 @@ private static class GroupAlertCache { private long lastSendTime; private long lastRepeatTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java index 08cdf7b3b78..4ebfdd22bf6 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduce.java @@ -18,25 +18,28 @@ package org.apache.hertzbeat.alert.reduce; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.Collections; -import java.util.stream.Collectors; - import lombok.Data; import lombok.AllArgsConstructor; @@ -46,7 +49,7 @@ */ @Component @Slf4j -public class AlarmInhibitReduce { +public class AlarmInhibitReduce implements DisposableBean { /** * Interval for checking and cleaning up expired source alerts @@ -73,22 +76,69 @@ public class AlarmInhibitReduce { /** * Default TTL for source alerts (4 hours) */ - private static long SOURCE_ALERT_TTL = 4 * 60 * 60 * 1000L; + private final long sourceAlertTtl; + + private final ScheduledExecutorService cleanupScheduler; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao , AlerterProperties alerterProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true); + } + + @Autowired + public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) { + this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true); + } + + AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao, + AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties, + boolean autoStart) { this.alarmSilenceReduce = alarmSilenceReduce; + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) { - SOURCE_ALERT_TTL = alerterProperties.getInhibit().getTtl(); + this.sourceAlertTtl = alerterProperties.getInhibit().getTtl(); + } else { + this.sourceAlertTtl = 4 * 60 * 60 * 1000L; } inhibitRules = new ConcurrentHashMap<>(8); sourceAlertCache = new ConcurrentHashMap<>(8); + this.cleanupScheduler = createCleanupScheduler(); + this.cleanupExecutor = createCleanupExecutor(properties); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache); List inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue(); refreshInhibitRules(inhibits); - startScheduledCleanupCache(); + if (autoStart) { + startScheduledCleanupCache(); + } } private void startScheduledCleanupCache() { + cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } + + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanupCacheRun() { + } + + @Override + public void destroy() { + cleanupScheduler.shutdownNow(); + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private ScheduledExecutorService createCleanupScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setUncaughtExceptionHandler((thread, throwable) -> { log.error("Scheduled clean up inhibit cache has uncaughtException."); @@ -97,17 +147,30 @@ private void startScheduledCleanupCache() { .setDaemon(true) .setNameFormat("inhibit-clean-up-%d") .build(); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); - // Scheduled cleanup of all expired source alerts - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - sourceAlertCache.values().forEach(this::cleanupExpiredEntries); - // Remove empty rule caches - sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); - } catch (Exception e) { - log.error("Error during scheduled cleanup", e); - } - }, CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + private ExecutorService createCleanupExecutor(VirtualThreadProperties properties) { + if (!properties.enabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("inhibit-clean-up-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("Scheduled clean up inhibit cache worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void runCleanupCache() { + beforeCleanupCacheRun(); + try { + sourceAlertCache.values().forEach(this::cleanupExpiredEntries); + sourceAlertCache.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + } catch (Exception e) { + log.error("Error during scheduled cleanup", e); + } } /** @@ -264,7 +327,7 @@ private void cacheSourceAlert(SingleAlert alert, AlertInhibit rule) { SourceAlertEntry entry = new SourceAlertEntry( alert, System.currentTimeMillis(), - System.currentTimeMillis() + SOURCE_ALERT_TTL + System.currentTimeMillis() + sourceAlertTtl ); ruleCache.put(alert.getFingerprint(), entry); cleanupExpiredEntries(ruleCache); @@ -315,4 +378,63 @@ private static class SourceAlertEntry { private final long createTime; private final long expiryTime; } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + if (executor != null) { + executor.execute(this::runOnce); + } else { + runOnce(); + } + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java index 444b85d79dc..ba83880959d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/AlerterWorkerPoolTest.java @@ -18,9 +18,18 @@ package org.apache.hertzbeat.alert; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.BeforeEach; +import org.apache.hertzbeat.common.concurrent.AdmissionMode; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -28,60 +37,178 @@ */ class AlerterWorkerPoolTest { - private static final int NUMBER_OF_THREADS = 10; + private static final int NUMBER_OF_TASKS = 10; + private AlerterWorkerPool pool; - private AtomicInteger counter; - private CountDownLatch latch; - @BeforeEach - void setUp() { - pool = new AlerterWorkerPool(); - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); + @AfterEach + void tearDown() { + if (pool != null) { + pool.destroy(); + } } @Test void executeJob() throws InterruptedException { - for (int i = 0; i < NUMBER_OF_THREADS; i++) { + pool = new AlerterWorkerPool(); + AtomicInteger counter = new AtomicInteger(); + CountDownLatch latch = new CountDownLatch(NUMBER_OF_TASKS); + + for (int i = 0; i < NUMBER_OF_TASKS; i++) { pool.executeJob(() -> { counter.incrementAndGet(); latch.countDown(); }); } - latch.await(); - assertEquals(NUMBER_OF_THREADS, counter.get()); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals(NUMBER_OF_TASKS, counter.get()); } @Test - void executeNotify() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeNotify(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); + void executeNotifyRunsOnVirtualThread() throws Exception { + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeNotify((byte) 1, () -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeNotifyRejectsWhenGlobalConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 8), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); + + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 2, () -> { + })); + } finally { + release.countDown(); } - latch.await(); + } + + @Test + void executeNotifyRejectsWhenChannelLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 8), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 1), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + pool = new AlerterWorkerPool(properties); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + pool.executeNotify((byte) 1, () -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertEquals(NUMBER_OF_THREADS, counter.get()); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeNotify((byte) 1, () -> { + })); + } finally { + release.countDown(); + } } @Test void executeLogJob() throws InterruptedException { - counter = new AtomicInteger(); - latch = new CountDownLatch(NUMBER_OF_THREADS); - - for (int i = 0; i < NUMBER_OF_THREADS; i++) { - pool.executeLogJob(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); - } - latch.await(); + pool = new AlerterWorkerPool(); + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + pool.executeLogJob(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } - assertEquals(NUMBER_OF_THREADS, counter.get()); + @Test + void executeLogJobRejectsWhenQueueCapacityReached() throws InterruptedException { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + new VirtualThreadProperties.QueueProperties(1, 1), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + pool = new AlerterWorkerPool(properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + pool.executeLogJob(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + pool.executeLogJob(secondStarted::countDown); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + try { + assertThrows(RejectedExecutionException.class, () -> pool.executeLogJob(() -> { + })); + } finally { + releaseFirst.countDown(); + } + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java new file mode 100644 index 00000000000..a18b8113e7d --- /dev/null +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/periodic/PeriodicAlertRuleSchedulerTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.alert.calculate.periodic; + +import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hertzbeat.alert.dao.AlertDefineDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.apache.hertzbeat.common.entity.alerter.AlertDefine; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link PeriodicAlertRuleScheduler}. + */ +@ExtendWith(MockitoExtension.class) +class PeriodicAlertRuleSchedulerTest { + + @Mock + private MetricsPeriodicAlertCalculator metricsCalculator; + + @Mock + private LogPeriodicAlertCalculator logCalculator; + + @Mock + private AlertDefineDao alertDefineDao; + + private PeriodicAlertRuleScheduler scheduler; + + @BeforeEach + void setUp() { + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + VirtualThreadProperties.defaults()); + } + + @AfterEach + void tearDown() { + if (scheduler != null) { + scheduler.destroy(); + } + } + + @Test + void updateScheduleRunsPeriodicCalculationOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(1L); + scheduler.updateSchedule(rule); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void updateScheduleKeepsSingleInFlightExecutionPerRule() throws InterruptedException { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + int count = invocations.incrementAndGet(); + try { + if (count == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (count == 2) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(2L)); + + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + Thread.sleep(1200L); + assertEquals(1, maxConcurrent.get()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void cancelScheduleInterruptsRunningVirtualTask() throws InterruptedException { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + doAnswer(invocation -> { + started.countDown(); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + AlertDefine rule = metricRule(3L); + scheduler.updateSchedule(rule); + + assertTrue(started.await(5, TimeUnit.SECONDS)); + scheduler.cancelSchedule(rule.getId()); + assertTrue(interrupted.await(5, TimeUnit.SECONDS)); + } + + @Test + void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException { + scheduler.destroy(); + scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, alertDefineDao, + periodicProperties(1)); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + doAnswer(invocation -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.updateAndGet(current -> Math.max(current, active)); + AlertDefine rule = invocation.getArgument(0); + try { + if (rule.getId().equals(4L)) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (rule.getId().equals(5L)) { + secondStarted.countDown(); + } + } finally { + concurrent.decrementAndGet(); + } + return null; + }).when(metricsCalculator).calculate(any(AlertDefine.class)); + + scheduler.updateSchedule(metricRule(4L)); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + scheduler.updateSchedule(metricRule(5L)); + Thread.sleep(200L); + assertEquals(1, maxConcurrent.get()); + assertEquals(1L, secondStarted.getCount()); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private AlertDefine metricRule(Long id) { + return AlertDefine.builder() + .id(id) + .name("periodic-rule-" + id) + .type(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC) + .period(1) + .enable(true) + .build(); + } + + private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) { + return new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + maxConcurrentJobs, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + } +} diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java index e27cb833f32..d520821bec7 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/calculate/realtime/window/AlarmEvaluatorTest.java @@ -19,9 +19,11 @@ import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertDefine; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.apache.hertzbeat.common.entity.log.LogEntry; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -32,14 +34,20 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -98,6 +106,13 @@ void setUp() { windowData.addMatchingLog(matchingEvent); } + @AfterEach + void tearDown() { + if (alarmEvaluator != null) { + alarmEvaluator.destroy(); + } + } + @Test void testProcessWindowDataWithIndividualMode() throws InterruptedException { // Given - alert define with individual mode @@ -343,4 +358,104 @@ void testMultipleMatchingLogsInWindow() throws InterruptedException { assertEquals(2, alerts.size()); assertEquals(2, alerts.get(0).getTriggerTimes()); // Each alert should have trigger times = total count } + + @Test + void testSendAndProcessWindowDataRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + alertDefine.setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + alarmEvaluator.sendAndProcessWindowData(windowData); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testSendAndProcessWindowDataQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + VirtualThreadProperties.QueueProperties.reduceDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + alarmEvaluator.destroy(); + alarmEvaluator = new AlarmEvaluator(alarmCommonReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class)); + + WindowAggregator.WindowData firstWindow = cloneWindowDataWithBody("first"); + WindowAggregator.WindowData secondWindow = cloneWindowDataWithBody("second"); + firstWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + secondWindow.getAlertDefine().setLabels(Map.of(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL)); + + alarmEvaluator.sendAndProcessWindowData(firstWindow); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmEvaluator.sendAndProcessWindowData(secondWindow); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + + private WindowAggregator.WindowData cloneWindowDataWithBody(String body) { + LogEntry logEntry = LogEntry.builder() + .timeUnixNano(System.currentTimeMillis() * 1_000_000L) + .severityText("ERROR") + .body(body) + .build(); + AlertDefine define = AlertDefine.builder() + .id(alertDefine.getId()) + .name(alertDefine.getName()) + .type(alertDefine.getType()) + .expr(alertDefine.getExpr()) + .times(alertDefine.getTimes()) + .template(alertDefine.getTemplate()) + .labels(alertDefine.getLabels()) + .annotations(alertDefine.getAnnotations()) + .enable(alertDefine.isEnable()) + .build(); + MatchingLogEvent event = MatchingLogEvent.builder() + .logEntry(logEntry) + .alertDefine(define) + .eventTimestamp(System.currentTimeMillis()) + .workerTimestamp(System.currentTimeMillis()) + .build(); + WindowAggregator.WindowData clonedWindowData = new WindowAggregator.WindowData( + new WindowAggregator.WindowKey(define.getId(), System.currentTimeMillis() - 60000, System.currentTimeMillis()), + define); + clonedWindowData.addMatchingLog(event); + return clonedWindowData; + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java index 8102afcdc0b..ff4c92ef4cb 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/AlertNoticeDispatchTest.java @@ -19,7 +19,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyByte; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -149,4 +152,32 @@ void testSendNoticeMsgNoHandler() { assertFalse(alertNoticeDispatch.sendNoticeMsg(receiver, null, alert)); } + + @Test + void testDispatchAlarmUsesTypedNotifyExecution() { + NoticeTemplate template = new NoticeTemplate(); + template.setId(1L); + template.setName("default-template"); + + when(alertStoreHandler.store(alert)).thenReturn(alert); + when(noticeConfigService.getReceiverFilterRule(alert)).thenReturn(Collections.singletonList( + org.apache.hertzbeat.common.entity.alerter.NoticeRule.builder() + .receiverId(Collections.singletonList(1L)) + .templateId(1L) + .build())); + when(noticeConfigService.getReceiverById(1L)).thenReturn(receiver); + when(noticeConfigService.getOneTemplateById(1L)).thenReturn(template); + doNothing().when(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + doAnswer(invocation -> { + Runnable task = invocation.getArgument(1); + task.run(); + return null; + }).when(workerPool).executeNotify(anyByte(), any(Runnable.class)); + + alertNoticeDispatch.dispatchAlarm(alert); + + verify(workerPool).executeNotify(eq((byte) 1), any(Runnable.class)); + verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert)); + verify(emitterManager).broadcast(any(String.class)); + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java index a98f36dfff8..02081885293 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmCommonReduceTest.java @@ -17,7 +17,20 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -40,14 +53,86 @@ class AlarmCommonReduceTest { @BeforeEach void setUp() { - - testAlert = SingleAlert.builder().build(); + testAlert = SingleAlert.builder().labels(new HashMap<>(Map.of("alertname", "test"))).build(); alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce); } + @AfterEach + void tearDown() { + if (alarmCommonReduce != null) { + alarmCommonReduce.destroy(); + } + } + @Test void testReduceAndSendAlarm() { alarmCommonReduce.reduceAndSendAlarm(testAlert); } + @Test + void testReduceAndSendAlarmRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(testAlert); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testReduceAndSendAlarmQueuesWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + VirtualThreadProperties.PoolProperties.collectorDefaults(), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + new VirtualThreadProperties.AlerterProperties( + VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(), + 10, + VirtualThreadProperties.QueueProperties.logWorkerDefaults(), + new VirtualThreadProperties.QueueProperties(1, 0), + VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(), + 4), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + alarmCommonReduce.destroy(); + alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, properties); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger invocationOrder = new AtomicInteger(); + doAnswer(invocation -> { + int order = invocationOrder.incrementAndGet(); + if (order == 1) { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (order == 2) { + secondStarted.countDown(); + } + return null; + }).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "first"))).build()); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmCommonReduce.reduceAndSendAlarm(SingleAlert.builder() + .labels(new HashMap<>(Map.of("name", "second"))).build()); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + } + } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index a8e8dc05338..a042032cab4 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -38,6 +38,9 @@ package org.apache.hertzbeat.alert.reduce; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; @@ -47,9 +50,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -73,7 +82,15 @@ void setUp() { MockitoAnnotations.openMocks(this); when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); - alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao); + alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmGroupReduce != null) { + alarmGroupReduce.destroy(); + } } @Test @@ -112,6 +129,42 @@ void whenMatchingGroupRule_shouldGroup() { verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait } + @Test + void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmGroupReduce.destroy(); + alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmGroupReduce.dispatchCheckAndSendGroups(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private Map createLabels(String... keyValues) { Map labels = new HashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { @@ -119,4 +172,67 @@ private Map createLabels(String... keyValues) { } return labels; } + + private static final class TestAlarmGroupReduce extends AlarmGroupReduce { + + private final CountDownLatch virtualThreadLatch; + + private final AtomicBoolean virtualThread; + + private final CountDownLatch firstStarted; + + private final CountDownLatch releaseFirst; + + private final CountDownLatch secondStarted; + + private final AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent; + + private final AtomicInteger invocations; + + private TestAlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao, + VirtualThreadProperties properties, CountDownLatch virtualThreadLatch, + AtomicBoolean virtualThread, CountDownLatch firstStarted, + CountDownLatch releaseFirst, CountDownLatch secondStarted, + AtomicInteger maxConcurrent, AtomicInteger invocations) { + super(alarmInhibitReduce, alertGroupConvergeDao, properties, false); + this.virtualThreadLatch = virtualThreadLatch; + this.virtualThread = virtualThread; + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + this.invocations = invocations; + this.concurrent = maxConcurrent == null ? null : new AtomicInteger(); + } + + @Override + void beforeCheckAndSendGroupsRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null || invocations == null) { + return; + } + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1 && firstStarted != null && releaseFirst != null) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2 && secondStarted != null) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java index 18bc17eb578..7b264dffb7d 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmInhibitReduceTest.java @@ -39,6 +39,7 @@ package org.apache.hertzbeat.alert.reduce; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -49,14 +50,19 @@ import java.util.Map; import java.util.ArrayList; import java.util.List; - +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.dao.AlertInhibitDao; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertInhibit; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -89,7 +95,15 @@ void setUp() { inhibitProperties.setTtl(60000); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); + } + + @AfterEach + void tearDown() { + if (alarmInhibitReduce != null) { + alarmInhibitReduce.destroy(); + } } @Test @@ -290,7 +304,9 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { AlerterProperties.InhibitProperties inhibitProperties = new AlerterProperties.InhibitProperties(); inhibitProperties.setTtl(100); when(alerterProperties.getInhibit()).thenReturn(inhibitProperties); - alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), false); AlertInhibit rule = AlertInhibit.builder() .id(1L) @@ -324,6 +340,42 @@ void whenSourceAlertExpires_shouldNotInhibit() throws InterruptedException { verify(alarmSilenceReduce).silenceAlarm(targetGroupAlert); } + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null); + + alarmInhibitReduce.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + alarmInhibitReduce.destroy(); + alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties, + new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted, + maxConcurrent, new AtomicInteger()); + + alarmInhibitReduce.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + alarmInhibitReduce.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + private GroupAlert createGroupAlert(String status, Map labels, List alerts) { return GroupAlert.builder() .status(status) @@ -347,4 +399,68 @@ private SingleAlert createSingleAlert(String status, String fingerprint, Map { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); - assertDoesNotThrow(() -> workerPool.executeJob(mockTask)); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); } @Test - void testExecuteJobThrowsException() { + void testExecuteJobRejectsWhenConcurrencyLimitReached() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties( + true, + new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1), + VirtualThreadProperties.PoolProperties.commonDefaults(), + VirtualThreadProperties.PoolProperties.managerDefaults(), + VirtualThreadProperties.AlerterProperties.defaults(), + VirtualThreadProperties.PoolProperties.warehouseDefaults(), + VirtualThreadProperties.AsyncProperties.defaults()); + workerPool = new WorkerPool(properties); - workerPool = mock(WorkerPool.class); - doThrow(new RejectedExecutionException()).when(workerPool).executeJob(mockTask); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + workerPool.executeJob(() -> { + started.countDown(); + try { + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(5, TimeUnit.SECONDS)); - assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(mockTask)); + try { + assertThrows(RejectedExecutionException.class, () -> workerPool.executeJob(() -> { + })); + } finally { + release.countDown(); + } } - - @Test - void testDestroy() { - assertDoesNotThrow(() -> workerPool.destroy()); - } - } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java index 3112e1e5baf..03a6977da63 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServerTest.java @@ -18,19 +18,27 @@ package org.apache.hertzbeat.collector.dispatch.entrance; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.netty.channel.Channel; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.collector.dispatch.CollectorInfoProperties; import org.apache.hertzbeat.collector.dispatch.DispatchProperties; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.remoting.RemotingClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -62,7 +70,7 @@ class CollectServerTest { private DispatchProperties.EntranceProperties.NettyProperties nettyProperties; @Mock - private CommonThreadPool threadPool; + private BackgroundTaskExecutor threadPool; @Mock private CollectorInfoProperties infoProperties; @@ -142,4 +150,69 @@ void testOnChannelActive() { assertNotNull(scheduledExecutor); } + @Test + void testDispatchHeartbeatRunsOnVirtualThread() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + org.mockito.Mockito.doAnswer(invocation -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void testDispatchHeartbeatDoesNotRunConcurrently() throws Exception { + VirtualThreadProperties properties = new VirtualThreadProperties(); + collectServer = new CollectServer(collectJobService, timerDispatch, properties(), threadPool, infoProperties, properties); + + RemotingClient remotingClient = mock(RemotingClient.class); + ReflectionTestUtils.setField(collectServer, "remotingClient", remotingClient); + + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + org.mockito.Mockito.doAnswer(invocation -> { + int running = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(running, Math::max); + int currentInvocation = invocations.incrementAndGet(); + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + concurrent.decrementAndGet(); + return null; + }).when(remotingClient).sendMsg(any(ClusterMsg.Message.class)); + + collectServer.dispatchHeartbeat("collector1"); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + collectServer.dispatchHeartbeat("collector1"); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + private DispatchProperties properties() { + return properties; + } + } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java index c03709bad7f..f915f2ee4eb 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/dispatch/CommonDispatcher.java @@ -114,7 +114,7 @@ public CommonDispatcher(MetricsCollectorQueue jobRequestQueue, public void start() { try { // Pull the collection task from the task queue and put it into the thread pool for execution - workerPool.executeJob(() -> { + workerPool.executeLongRunning(() -> { Thread.currentThread().setName("metrics-task-dispatcher"); while (!Thread.currentThread().isInterrupted()) { MetricsCollect metricsCollect = null; @@ -379,4 +379,4 @@ protected static class MetricsTime { private Metrics metrics; private Timeout timeout; } -} \ No newline at end of file +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index eb1b92e0de8..9615e547b94 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -75,3 +75,12 @@ push: common: queue: type: netty + +hertzbeat: + # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. + vthreads: + enabled: true + common: + mode: UNBOUNDED_VT + collector: + mode: LIMIT_AND_REJECT diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java index ed936372334..9b6699622b2 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCache.java @@ -20,14 +20,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; -import lombok.extern.slf4j.Slf4j; - import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; /** * Singleton LRU global resource cache for client-server connections @@ -56,10 +57,20 @@ public class GlobalConnectionCache { */ private final ConcurrentLinkedHashMap> cacheMap; + private final ScheduledExecutorService scheduledExecutor; + + private final ExecutorService cleanupExecutor; + + private final ScheduledDispatchTask cleanupTask; + /** * Private constructor to prevent instantiation */ private GlobalConnectionCache() { + this(true); + } + + GlobalConnectionCache(boolean autoStart) { cacheMap = new ConcurrentLinkedHashMap.Builder>() .maximumWeightedCapacity(Integer.MAX_VALUE) .listener((key, value) -> { @@ -72,7 +83,13 @@ private GlobalConnectionCache() { log.info("GlobalConnectionCache discarded key: {}, value: {}.", key, value); }) .build(); - initCacheMonitor(); + this.scheduledExecutor = createCacheMonitorScheduler(); + this.cleanupExecutor = createCleanupExecutor(); + this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanTimeoutOrUnHealthyCache); + if (autoStart) { + initCacheMonitor(); + Runtime.getRuntime().addShutdownHook(new Thread(this::destroy)); + } } /** @@ -95,18 +112,42 @@ public static GlobalConnectionCache getInstance() { * Initialize the cache monitor for cleaning up expired connections */ private void initCacheMonitor() { + scheduledExecutor.scheduleWithFixedDelay(this::dispatchCleanupCache, 2, 100, TimeUnit.SECONDS); + } + + /** + * Clean and remove timeout or unhealthy cache entries + */ + void dispatchCleanupCache() { + cleanupTask.dispatch(); + } + + void beforeCleanTimeoutOrUnHealthyCacheRun() { + } + + void destroy() { + scheduledExecutor.shutdownNow(); + cleanupExecutor.shutdownNow(); + } + + private ScheduledExecutorService createCacheMonitorScheduler() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("connection-cache-timeout-detector-%d") .setDaemon(true) .build(); - ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(this::cleanTimeoutOrUnHealthyCache, 2, 100, TimeUnit.SECONDS); + return Executors.newSingleThreadScheduledExecutor(threadFactory); } - /** - * Clean and remove timeout or unhealthy cache entries - */ - private void cleanTimeoutOrUnHealthyCache() { + private ExecutorService createCleanupExecutor() { + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("connection-cache-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("Connection cache cleanup has uncaughtException.", throwable)) + .factory()); + } + + private void runCleanTimeoutOrUnHealthyCache() { + beforeCleanTimeoutOrUnHealthyCacheRun(); try { cacheMap.forEach((key, value) -> { Long[] cacheTime = timeoutMap.get(key); @@ -204,4 +245,59 @@ public void removeCache(Object key) { log.error("Connection close error for key {}: {}", key, e.getMessage(), e); } } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java index 22cb0e8a1bd..54001cdae0d 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClient.java @@ -22,6 +22,7 @@ import java.security.cert.CertificateExpiredException; import java.security.cert.X509Certificate; import java.util.Date; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -52,6 +53,14 @@ public class CommonHttpClient { private static PoolingHttpClientConnectionManager connectionManager; + private static ScheduledExecutorService scheduledExecutor; + + private static ExecutorService cleanupExecutor; + + private static ScheduledDispatchTask cleanupTask; + + private static volatile Runnable beforeCleanupHook; + /** * all max total connection */ @@ -137,15 +146,9 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr // clean up available but idle connections .evictIdleConnections(100, TimeUnit.SECONDS) .build(); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("http-connection-pool-cleaner-%d") - .setDaemon(true) - .build(); - ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, threadFactory); - scheduledExecutor.scheduleWithFixedDelay(() -> { - connectionManager.closeExpiredConnections(); - connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); - }, 40L, 40L, TimeUnit.SECONDS); + initializeCleanupExecutors(); + scheduledExecutor.scheduleWithFixedDelay(CommonHttpClient::dispatchConnectionPoolCleanup, + 40L, 40L, TimeUnit.SECONDS); // shutdown hook Runtime.getRuntime().addShutdownHook(new Thread(CommonHttpClient::close)); @@ -156,13 +159,119 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr public static CloseableHttpClient getHttpClient() { return httpClient; } + + public static PoolingHttpClientConnectionManager getConnectionManager() { + return connectionManager; + } + + static void dispatchConnectionPoolCleanup() { + if (cleanupTask != null) { + cleanupTask.dispatch(); + } + } + + static void setConnectionManagerForTest(PoolingHttpClientConnectionManager manager) { + connectionManager = manager; + } + + static void setBeforeCleanupHookForTest(Runnable hook) { + beforeCleanupHook = hook; + } public static void close() { try { - httpClient.close(); + if (httpClient != null) { + httpClient.close(); + } } catch (Exception e) { log.error("close http client error", e); } + if (scheduledExecutor != null) { + scheduledExecutor.shutdownNow(); + } + if (cleanupExecutor != null) { + cleanupExecutor.shutdownNow(); + } + } + + private static void initializeCleanupExecutors() { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("http-connection-pool-cleaner-%d") + .setDaemon(true) + .build(); + scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); + cleanupExecutor = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("http-connection-pool-cleaner-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> + log.error("HTTP connection pool cleanup has uncaughtException.", throwable)) + .factory()); + cleanupTask = new ScheduledDispatchTask(cleanupExecutor, CommonHttpClient::runConnectionPoolCleanup); + } + + private static void runConnectionPoolCleanup() { + Runnable hook = beforeCleanupHook; + if (hook != null) { + hook.run(); + } + if (connectionManager == null) { + return; + } + connectionManager.closeExpiredConnections(); + connectionManager.closeIdleConnections(40, TimeUnit.SECONDS); + } + + private static final class ScheduledDispatchTask { + + private final ExecutorService executor; + + private final Runnable task; + + private boolean running; + + private int pendingRuns; + + private ScheduledDispatchTask(ExecutorService executor, Runnable task) { + this.executor = executor; + this.task = task; + } + + private void dispatch() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns++; + shouldSchedule = !running; + if (shouldSchedule) { + running = true; + } + } + if (shouldSchedule) { + scheduleRun(); + } + } + + private void scheduleRun() { + executor.execute(this::runOnce); + } + + private void runOnce() { + try { + task.run(); + } finally { + scheduleNextIfNeeded(); + } + } + + private void scheduleNextIfNeeded() { + boolean shouldSchedule; + synchronized (this) { + pendingRuns = Math.max(0, pendingRuns - 1); + shouldSchedule = pendingRuns > 0; + if (!shouldSchedule) { + running = false; + return; + } + } + scheduleRun(); + } } - } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java index a11ff4cfe44..a42cc9a4283 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/WorkerPool.java @@ -18,15 +18,19 @@ package org.apache.hertzbeat.collector.dispatch; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.stereotype.Component; - +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * Collection task worker thread pool @@ -35,25 +39,55 @@ @Slf4j public class WorkerPool implements DisposableBean { - private ThreadPoolExecutor workerExecutor; + private final ManagedExecutor workerExecutor; + + private final ManagedExecutor longRunningExecutor; public WorkerPool() { - initWorkExecutor(); + this(VirtualThreadProperties.defaults()); } - private void initWorkExecutor() { - // thread factory - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + @Autowired + public WorkerPool(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + this.workerExecutor = createWorkerExecutor(properties); + this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor); + } + + private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) { + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("[Important] WorkerPool workerExecutor has uncaughtException.", throwable); + log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); + }; + if (properties.enabled()) { + VirtualThreadProperties.PoolProperties poolProperties = properties.collector(); + return ManagedExecutors.newVirtualExecutor("collector-worker", "collect-worker-", + poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler); + } + return ManagedExecutors.wrap("collector-worker", createLegacyExecutor(handler)); + } + + private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) { + if (!properties.enabled()) { + return fallback; + } + return ManagedExecutors.newPlatformExecutor("collector-long-running", "collect-long-running-", + (thread, throwable) -> { + log.error("[Important] WorkerPool longRunningExecutor has uncaughtException.", throwable); log.error("Thread Name {} : {}", thread.getName(), throwable.getMessage(), throwable); - }) + }); + } + + private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(handler) .setDaemon(true) .setNameFormat("collect-worker-%d") .build(); int coreSize = Math.max(2, Runtime.getRuntime().availableProcessors()); int maxSize = Runtime.getRuntime().availableProcessors() * 16; - workerExecutor = new ThreadPoolExecutor(coreSize, + return new ThreadPoolExecutor(coreSize, maxSize, 10, TimeUnit.SECONDS, @@ -72,10 +106,20 @@ public void executeJob(Runnable runnable) throws RejectedExecutionException { workerExecutor.execute(runnable); } + /** + * Run the long-lived dispatcher job outside of the collector admission limit. + * + * @param runnable dispatcher task + */ + public void executeLongRunning(Runnable runnable) { + longRunningExecutor.execute(runnable); + } + @Override public void destroy() throws Exception { - if (workerExecutor != null) { - workerExecutor.shutdownNow(); + workerExecutor.close(); + if (longRunningExecutor != workerExecutor) { + longRunningExecutor.close(); } } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java index 9b34a11d4bd..a5bf494750f 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/CollectServer.java @@ -32,19 +32,24 @@ import org.apache.hertzbeat.collector.dispatch.entrance.processor.GoOnlineProcessor; import org.apache.hertzbeat.collector.dispatch.entrance.processor.HeartbeatProcessor; import org.apache.hertzbeat.collector.timer.TimerDispatch; +import org.apache.hertzbeat.common.concurrent.BackgroundTaskExecutor; +import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.dto.CollectorInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.CommonThreadPool; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.RemotingClient; import org.apache.hertzbeat.remoting.event.NettyEventListener; import org.apache.hertzbeat.remoting.netty.NettyClientConfig; import org.apache.hertzbeat.remoting.netty.NettyRemotingClient; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -70,11 +75,42 @@ public class CollectServer implements CommandLineRunner { private ScheduledExecutorService scheduledExecutor; + private final ExecutorService heartbeatExecutor; + + private final Object heartbeatLock = new Object(); + + private boolean heartbeatRunning; + + private boolean heartbeatPending; + + private final Runnable closeApplicationAction; + public CollectServer(final CollectJobService collectJobService, final TimerDispatch timerDispatch, final DispatchProperties properties, - final CommonThreadPool threadPool, + final BackgroundTaskExecutor threadPool, final CollectorInfoProperties infoProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, + VirtualThreadProperties.defaults()); + } + + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final BackgroundTaskExecutor threadPool, + final CollectorInfoProperties infoProperties, + final VirtualThreadProperties virtualThreadProperties) { + this(collectJobService, timerDispatch, properties, threadPool, infoProperties, null, virtualThreadProperties); + } + + @Autowired + public CollectServer(final CollectJobService collectJobService, + final TimerDispatch timerDispatch, + final DispatchProperties properties, + final BackgroundTaskExecutor threadPool, + final CollectorInfoProperties infoProperties, + final ConfigurableApplicationContext applicationContext, + final VirtualThreadProperties virtualThreadProperties) { if (properties == null || properties.getEntrance() == null || properties.getEntrance().getNetty() == null) { log.error("init error, please config dispatch entrance netty props in application.yml"); throw new IllegalArgumentException("please config dispatch entrance netty props"); @@ -87,10 +123,12 @@ public CollectServer(final CollectJobService collectJobService, this.timerDispatch = timerDispatch; this.collectJobService.setCollectServer(this); this.infoProperties = infoProperties; + this.heartbeatExecutor = createHeartbeatExecutor(virtualThreadProperties); + this.closeApplicationAction = createCloseApplicationAction(applicationContext); this.init(properties, threadPool); } - private void init(final DispatchProperties properties, final CommonThreadPool threadPool) { + private void init(final DispatchProperties properties, final BackgroundTaskExecutor threadPool) { NettyClientConfig nettyClientConfig = new NettyClientConfig(); DispatchProperties.EntranceProperties.NettyProperties nettyProperties = properties.getEntrance().getNetty(); nettyClientConfig.setServerHost(nettyProperties.getManagerHost()); @@ -101,13 +139,19 @@ private void init(final DispatchProperties properties, final CommonThreadPool th this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK, new CollectCyclicDataProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.DELETE_CYCLIC_TASK, new DeleteCyclicTaskProcessor(this)); this.remotingClient.registerProcessor(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK, new CollectOneTimeDataProcessor(this)); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor()); - this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, new GoCloseProcessor(this)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_OFFLINE, new GoOfflineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_ONLINE, new GoOnlineProcessor(timerDispatch)); + this.remotingClient.registerProcessor(ClusterMsg.MessageType.GO_CLOSE, + new GoCloseProcessor(this, timerDispatch, closeApplicationAction)); } public void shutdown() { - this.scheduledExecutor.shutdownNow(); + if (this.scheduledExecutor != null) { + this.scheduledExecutor.shutdownNow(); + } + if (this.heartbeatExecutor != null) { + this.heartbeatExecutor.shutdownNow(); + } this.remotingClient.shutdown(); } @@ -120,6 +164,21 @@ public void sendMsg(final ClusterMsg.Message message) { this.remotingClient.sendMsg(message); } + void dispatchHeartbeat(String identity) { + if (heartbeatExecutor == null) { + sendHeartbeat(identity); + return; + } + synchronized (heartbeatLock) { + if (heartbeatRunning) { + heartbeatPending = true; + return; + } + heartbeatRunning = true; + } + submitHeartbeat(identity); + } + @Override public void run(String... args) throws Exception { this.remotingClient.start(); @@ -161,19 +220,8 @@ public void onChannelActive(Channel channel) { .build(); scheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); // schedule send heartbeat message - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() - .setIdentity(identity) - .setDirection(ClusterMsg.Direction.REQUEST) - .setType(ClusterMsg.MessageType.HEARTBEAT) - .build(); - CollectServer.this.sendMsg(heartbeat); - log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); - } catch (Exception e) { - log.error("schedule send heartbeat to server error.{}", e.getMessage()); - } - }, 5, 5, TimeUnit.SECONDS); + scheduledExecutor.scheduleAtFixedRate(() -> CollectServer.this.dispatchHeartbeat(identity), + 5, 5, TimeUnit.SECONDS); } } @@ -182,4 +230,82 @@ public void onChannelIdle(Channel channel) { log.info("handle idle event triggered. collector is going offline."); } } + + private ExecutorService createHeartbeatExecutor(VirtualThreadProperties virtualThreadProperties) { + VirtualThreadProperties properties = + virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties; + if (!properties.enabled()) { + return null; + } + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("heartbeat-worker-vt-", 0) + .uncaughtExceptionHandler((thread, throwable) -> { + log.error("HeartBeat worker has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }) + .factory()); + } + + private void submitHeartbeat(String identity) { + boolean submitted = false; + try { + heartbeatExecutor.execute(() -> { + try { + sendHeartbeat(identity); + } finally { + onHeartbeatComplete(identity); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (heartbeatLock) { + heartbeatRunning = false; + heartbeatPending = false; + } + } + } + } + + private void onHeartbeatComplete(String identity) { + boolean shouldRunAgain; + synchronized (heartbeatLock) { + if (heartbeatPending) { + heartbeatPending = false; + shouldRunAgain = true; + } else { + heartbeatRunning = false; + shouldRunAgain = false; + } + } + if (shouldRunAgain) { + submitHeartbeat(identity); + } + } + + private void sendHeartbeat(String identity) { + try { + ClusterMsg.Message heartbeat = ClusterMsg.Message.newBuilder() + .setIdentity(identity) + .setDirection(ClusterMsg.Direction.REQUEST) + .setType(ClusterMsg.MessageType.HEARTBEAT) + .build(); + CollectServer.this.sendMsg(heartbeat); + log.info("collector send cluster server heartbeat, time: {}.", System.currentTimeMillis()); + } catch (Exception e) { + log.error("schedule send heartbeat to server error.{}", e.getMessage()); + } + } + + private Runnable createCloseApplicationAction(ConfigurableApplicationContext applicationContext) { + if (applicationContext == null) { + return () -> {}; + } + return () -> { + SpringApplication.exit(applicationContext, () -> 0); + if (applicationContext.isActive()) { + applicationContext.close(); + } + }; + } } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java index 4b068f1e45f..b259af80313 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoCloseProcessor.java @@ -23,9 +23,7 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; -import org.springframework.boot.SpringApplication; /** * handle collector close message @@ -34,24 +32,25 @@ @Slf4j public class GoCloseProcessor implements NettyRemotingProcessor { private final CollectServer collectServer; - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + private final Runnable closeApplicationAction; - public GoCloseProcessor(final CollectServer collectServer) { + public GoCloseProcessor(final CollectServer collectServer, + final TimerDispatch timerDispatch, + final Runnable closeApplicationAction) { this.collectServer = collectServer; + this.timerDispatch = timerDispatch; + this.closeApplicationAction = closeApplicationAction; } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { log.error("[Auth Failed]receive client auth failed message and go close. {}", message.getMsg()); } this.timerDispatch.goOffline(); this.collectServer.shutdown(); - SpringApplication.exit(SpringContextHolder.getApplicationContext(), () -> 0); - SpringContextHolder.shutdown(); + closeApplicationAction.run(); log.info("receive offline message and close success"); return null; } diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java index aad85194ee4..ccacf943e53 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOfflineProcessor.java @@ -23,7 +23,6 @@ import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; /** @@ -32,14 +31,15 @@ */ @Slf4j public class GoOfflineProcessor implements NettyRemotingProcessor { - - private TimerDispatch timerDispatch; + + private final TimerDispatch timerDispatch; + + public GoOfflineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } timerDispatch.goOffline(); log.info("receive offline message and handle success"); if (message.getMsg().toStringUtf8().contains(CommonConstants.COLLECTOR_AUTH_FAILED)) { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java index acebb42e3ad..e35c66a151b 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessor.java @@ -24,7 +24,6 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.dto.ServerInfo; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.AesUtil; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.remoting.netty.NettyRemotingProcessor; @@ -36,13 +35,14 @@ @Slf4j public class GoOnlineProcessor implements NettyRemotingProcessor { - private TimerDispatch timerDispatch; + private final TimerDispatch timerDispatch; + + public GoOnlineProcessor(TimerDispatch timerDispatch) { + this.timerDispatch = timerDispatch; + } @Override public ClusterMsg.Message handle(ChannelHandlerContext ctx, ClusterMsg.Message message) { - if (this.timerDispatch == null) { - this.timerDispatch = SpringContextHolder.getBean(TimerDispatch.class); - } if (message.getMsg().isEmpty()) { log.warn("The message that server response to collector is empty, please upgrade server"); } else { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java index 4aa6e4c0250..a8d6a600021 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/TimerDispatcher.java @@ -25,9 +25,11 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.constants.ScheduleTypeEnum; +import org.apache.hertzbeat.collector.dispatch.MetricsTaskDispatch; import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectResponseEventListener; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; @@ -35,6 +37,8 @@ import org.apache.hertzbeat.common.timer.HashedWheelTimer; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.Timer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.DisposableBean; import org.springframework.scheduling.support.CronExpression; import org.springframework.stereotype.Component; @@ -69,7 +73,19 @@ public class TimerDispatcher implements TimerDispatch, DisposableBean { */ private final AtomicBoolean started; + private final Supplier metricsTaskDispatchSupplier; + public TimerDispatcher() { + this(() -> timeout -> { + }); + } + + @Autowired + public TimerDispatcher(ObjectProvider metricsTaskDispatchProvider) { + this(resolveMetricsTaskDispatchSupplier(metricsTaskDispatchProvider)); + } + + private TimerDispatcher(Supplier metricsTaskDispatchSupplier) { this.wheelTimer = new HashedWheelTimer(r -> { Thread ret = new Thread(r, "wheelTimer"); ret.setDaemon(true); @@ -79,6 +95,16 @@ public TimerDispatcher() { this.currentTempTaskMap = new ConcurrentHashMap<>(8); this.eventListeners = new ConcurrentHashMap<>(8); this.started = new AtomicBoolean(true); + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; + } + + private static Supplier resolveMetricsTaskDispatchSupplier( + ObjectProvider metricsTaskDispatchProvider) { + if (metricsTaskDispatchProvider == null) { + return () -> timeout -> { + }; + } + return metricsTaskDispatchProvider::getObject; } @Override @@ -87,7 +113,8 @@ public void addJob(Job addJob, CollectResponseEventListener eventListener) { log.warn("Collector is offline, can not dispatch collect jobs."); return; } - WheelTimerTask timerJob = new WheelTimerTask(addJob); + // Delay dispatcher lookup to avoid a startup cycle with CommonDispatcher. + WheelTimerTask timerJob = new WheelTimerTask(addJob, metricsTaskDispatchSupplier); if (addJob.isCyclic()) { Long nextExecutionTime = getNextExecutionInterval(addJob); Timeout timeout = wheelTimer.newTimeout(timerJob, nextExecutionTime, TimeUnit.SECONDS); diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java index 5dfe1922a57..30d97ba2e11 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/timer/WheelTimerTask.java @@ -27,7 +27,6 @@ import org.apache.hertzbeat.common.entity.job.Configmap; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.timer.Timeout; import org.apache.hertzbeat.common.timer.TimerTask; import org.apache.hertzbeat.common.util.AesUtil; @@ -35,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -44,11 +44,15 @@ public class WheelTimerTask implements TimerTask { private final Job job; - private final MetricsTaskDispatch metricsTaskDispatch; + private final Supplier metricsTaskDispatchSupplier; private static final Gson GSON = new Gson(); - public WheelTimerTask(Job job) { - this.metricsTaskDispatch = SpringContextHolder.getBean(MetricsTaskDispatch.class); + public WheelTimerTask(Job job, MetricsTaskDispatch metricsTaskDispatch) { + this(job, () -> metricsTaskDispatch); + } + + public WheelTimerTask(Job job, Supplier metricsTaskDispatchSupplier) { + this.metricsTaskDispatchSupplier = metricsTaskDispatchSupplier; this.job = job; // The initialization job will monitor the actual parameter value and replace the collection field initJobMetrics(job); @@ -93,7 +97,7 @@ private void initJobMetrics(Job job) { @Override public void run(Timeout timeout) throws Exception { job.setDispatchTime(System.currentTimeMillis()); - metricsTaskDispatch.dispatchMetricsTask(timeout); + metricsTaskDispatchSupplier.get().dispatchMetricsTask(timeout); } public Job getJob() { diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java new file mode 100644 index 00000000000..d066434d410 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/cache/GlobalConnectionCacheTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.cache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link GlobalConnectionCache}. + */ +class GlobalConnectionCacheTest { + + private TestGlobalConnectionCache globalConnectionCache; + + @BeforeEach + void setUp() { + globalConnectionCache = new TestGlobalConnectionCache(); + } + + @AfterEach + void tearDown() { + if (globalConnectionCache != null) { + globalConnectionCache.destroy(); + } + } + + @Test + void dispatchCleanupCacheRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + globalConnectionCache.setVirtualThreadHook(latch, virtualThread); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger maxConcurrent = new AtomicInteger(); + globalConnectionCache.setConcurrencyHook(firstStarted, releaseFirst, secondStarted, maxConcurrent); + + globalConnectionCache.dispatchCleanupCache(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + globalConnectionCache.dispatchCleanupCache(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchCleanupCacheClosesExpiredConnections() throws Exception { + TestConnection connection = new TestConnection(); + globalConnectionCache.addCache("expired", connection, -1L); + + globalConnectionCache.dispatchCleanupCache(); + + assertTrue(connection.closed.await(5, TimeUnit.SECONDS)); + assertEquals(1, connection.closeCount.get()); + assertTrue(globalConnectionCache.getCache("expired", false).isEmpty()); + } + + private static final class TestGlobalConnectionCache extends GlobalConnectionCache { + + private CountDownLatch virtualThreadLatch; + + private AtomicBoolean virtualThread; + + private CountDownLatch firstStarted; + + private CountDownLatch releaseFirst; + + private CountDownLatch secondStarted; + + private AtomicInteger maxConcurrent; + + private final AtomicInteger concurrent = new AtomicInteger(); + + private final AtomicInteger invocations = new AtomicInteger(); + + private TestGlobalConnectionCache() { + super(false); + } + + private void setVirtualThreadHook(CountDownLatch latch, AtomicBoolean flag) { + this.virtualThreadLatch = latch; + this.virtualThread = flag; + } + + private void setConcurrencyHook(CountDownLatch firstStarted, CountDownLatch releaseFirst, + CountDownLatch secondStarted, AtomicInteger maxConcurrent) { + this.firstStarted = firstStarted; + this.releaseFirst = releaseFirst; + this.secondStarted = secondStarted; + this.maxConcurrent = maxConcurrent; + } + + @Override + void beforeCleanTimeoutOrUnHealthyCacheRun() { + if (virtualThread != null) { + virtualThread.set(Thread.currentThread().isVirtual()); + } + if (virtualThreadLatch != null) { + virtualThreadLatch.countDown(); + } + if (maxConcurrent == null) { + return; + } + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + } + } + + private static final class TestConnection extends AbstractConnection { + + private final CountDownLatch closed = new CountDownLatch(1); + + private final AtomicInteger closeCount = new AtomicInteger(); + + @Override + public Object getConnection() { + return new Object(); + } + + @Override + public void closeConnection() { + closeCount.incrementAndGet(); + closed.countDown(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java new file mode 100644 index 00000000000..2ec28df4528 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/collect/common/http/CommonHttpClientVirtualThreadTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.common.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for CommonHttpClient cleanup dispatch. + */ +class CommonHttpClientVirtualThreadTest { + + private final PoolingHttpClientConnectionManager originalConnectionManager = CommonHttpClient.getConnectionManager(); + + @AfterEach + void tearDown() { + CommonHttpClient.setBeforeCleanupHookForTest(null); + CommonHttpClient.setConnectionManagerForTest(originalConnectionManager); + } + + @Test + void dispatchConnectionPoolCleanupRunsOnVirtualThread() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void dispatchConnectionPoolCleanupDoesNotRunConcurrently() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicInteger concurrent = new AtomicInteger(); + AtomicInteger maxConcurrent = new AtomicInteger(); + AtomicInteger invocations = new AtomicInteger(); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + CommonHttpClient.setConnectionManagerForTest(manager); + CommonHttpClient.setBeforeCleanupHookForTest(() -> { + int active = concurrent.incrementAndGet(); + maxConcurrent.accumulateAndGet(active, Math::max); + int currentInvocation = invocations.incrementAndGet(); + try { + if (currentInvocation == 1) { + firstStarted.countDown(); + releaseFirst.await(5, TimeUnit.SECONDS); + } else if (currentInvocation == 2) { + secondStarted.countDown(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + concurrent.decrementAndGet(); + } + }); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS)); + + releaseFirst.countDown(); + assertTrue(secondStarted.await(5, TimeUnit.SECONDS)); + assertEquals(1, maxConcurrent.get()); + } + + @Test + void dispatchConnectionPoolCleanupClosesExpiredAndIdleConnections() throws Exception { + CountDownLatch latch = new CountDownLatch(1); + PoolingHttpClientConnectionManager manager = mock(PoolingHttpClientConnectionManager.class); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(manager).closeExpiredConnections(); + CommonHttpClient.setConnectionManagerForTest(manager); + + CommonHttpClient.dispatchConnectionPoolCleanup(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(manager, times(1)).closeExpiredConnections(); + verify(manager, times(1)).closeIdleConnections(40, TimeUnit.SECONDS); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java index d43de3e97d2..b544c8c39bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/test/java/org/apache/hertzbeat/collector/dispatch/entrance/processor/GoOnlineProcessorTest.java @@ -20,19 +20,15 @@ import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import io.netty.channel.ChannelHandlerContext; -import org.apache.hertzbeat.collector.timer.TimerDispatch; import org.apache.hertzbeat.collector.timer.TimerDispatcher; import org.apache.hertzbeat.common.entity.job.Job; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.message.ClusterMsg; -import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.common.util.JsonUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.lang.reflect.Field; @@ -51,32 +47,27 @@ class GoOnlineProcessorTest { @Mock private ChannelHandlerContext channelHandlerContext; - private MockedStatic springContextHolderMockedStatic; - @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - goOnlineProcessor = new GoOnlineProcessor(); timerDispatcher = new TimerDispatcher(); - springContextHolderMockedStatic = Mockito.mockStatic(SpringContextHolder.class); - springContextHolderMockedStatic.when(() -> SpringContextHolder.getBean(TimerDispatch.class)).thenReturn(timerDispatcher); + goOnlineProcessor = new GoOnlineProcessor(timerDispatcher); } @AfterEach void tearDown() throws Exception { - springContextHolderMockedStatic.close(); timerDispatcher.destroy(); } @Test void verifyTaskMapPreservation() throws Exception { Job job = Job.builder() - .app("test") - .id(12345L) - .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) - .configmap(Lists.newArrayList()) - .isCyclic(true) - .build(); + .app("test") + .id(12345L) + .metrics(Lists.newArrayList(Metrics.builder().interval(100L).build())) + .configmap(Lists.newArrayList()) + .isCyclic(true) + .build(); timerDispatcher.addJob(job, null); Field cyclicTaskMapField = TimerDispatcher.class.getDeclaredField("currentCyclicTaskMap"); @@ -85,20 +76,20 @@ void verifyTaskMapPreservation() throws Exception { assertEquals(1, currentCyclicTaskMap.size(), "Task map should have 1 job initially"); ClusterMsg.Message responseMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.RESPONSE) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.RESPONSE) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, responseMsg); assertEquals(1, currentCyclicTaskMap.size(), "Task map should still have 1 job after receiving RESPONSE"); ClusterMsg.Message requestMsg = ClusterMsg.Message.newBuilder() - .setType(ClusterMsg.MessageType.GO_ONLINE) - .setDirection(ClusterMsg.Direction.REQUEST) - .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) - .setIdentity("test-identity") - .build(); + .setType(ClusterMsg.MessageType.GO_ONLINE) + .setDirection(ClusterMsg.Direction.REQUEST) + .setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job))) + .setIdentity("test-identity") + .build(); goOnlineProcessor.handle(channelHandlerContext, requestMsg); assertEquals(0, currentCyclicTaskMap.size(), "Task map should be empty after receiving REQUEST"); } diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java index fabdf973628..b4f91304f16 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-kafka/src/main/java/org/apache/hertzbeat/collector/collect/kafka/KafkaCollectImpl.java @@ -46,7 +46,6 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; -import org.springframework.util.Assert; import java.util.Collection; import java.util.Collections; @@ -223,14 +222,14 @@ private static boolean filterInternalTopics(String topic, Boolean monitorInterna @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null, "Metrics cannot be null"); + require(metrics != null, "Metrics cannot be null"); KafkaProtocol kafkaProtocol = metrics.getKclient(); // Ensure that metrics and kafkaProtocol are not null - Assert.isTrue(metrics != null && kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); + require(kafkaProtocol != null, "Kafka collect must have kafkaProtocol params"); // Ensure that host and port are not empty - Assert.hasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); - Assert.hasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); + requireHasText(kafkaProtocol.getHost(), "Kafka Protocol host is required."); + requireHasText(kafkaProtocol.getPort(), "Kafka Protocol port is required."); } @Override @@ -384,4 +383,14 @@ private long getLatestOffset(AdminClient adminClient, TopicPartition topicPartit public String supportProtocol() { return DispatchConstants.PROTOCOL_KAFKA; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java index b99304e60e4..0cf3107c580 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/src/main/java/org/apache/hertzbeat/collector/collect/mongodb/MongodbSingleCollectImpl.java @@ -43,7 +43,6 @@ import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.CommonUtil; import org.bson.Document; -import org.springframework.util.Assert; /** * Mongodb single collect @@ -87,11 +86,11 @@ public class MongodbSingleCollectImpl extends AbstractCollect { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException{ - Assert.isTrue(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); + require(metrics != null && metrics.getMongodb() != null, "Mongodb collect must has mongodb params"); MongodbProtocol mongodbProtocol = metrics.getMongodb(); - Assert.hasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); - Assert.hasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); - Assert.hasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); + requireHasText(mongodbProtocol.getCommand(), "Mongodb Protocol command is required."); + requireHasText(mongodbProtocol.getHost(), "Mongodb Protocol host is required."); + requireHasText(mongodbProtocol.getPort(), "Mongodb Protocol port is required."); } @Override @@ -226,4 +225,14 @@ private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) { connectionCommonCache.addCache(identifier, mongodbConnect, 3600 * 1000L); return mongoClient; } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java index 33a361aa0bf..4034c1e15bd 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/src/main/java/org/apache/hertzbeat/collector/collect/nebulagraph/NgqlCollectImpl.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.hertzbeat.collector.collect.AbstractCollect; @@ -33,8 +34,6 @@ import org.apache.hertzbeat.common.entity.job.protocol.NgqlProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.entity.message.CollectRep.MetricsData.Builder; -import org.springframework.util.Assert; -import org.springframework.util.StopWatch; /** * connect nebulaGraph and collect metrics use NGQL @@ -53,18 +52,17 @@ public class NgqlCollectImpl extends AbstractCollect { @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { NgqlProtocol ngql = metrics.getNgql(); - Assert.hasText(ngql.getHost(), "NGQL protocol host is required"); - Assert.hasText(ngql.getPort(), "Port protocol host is required"); - Assert.hasText(ngql.getParseType(), "NGQL protocol parseType is required"); - Assert.hasText(ngql.getUsername(), "NGQL protocol username is required"); - Assert.hasText(ngql.getPassword(), "NGQL protocol password is required"); + requireHasText(ngql.getHost(), "NGQL protocol host is required"); + requireHasText(ngql.getPort(), "Port protocol host is required"); + requireHasText(ngql.getParseType(), "NGQL protocol parseType is required"); + requireHasText(ngql.getUsername(), "NGQL protocol username is required"); + requireHasText(ngql.getPassword(), "NGQL protocol password is required"); } @Override public void collect(Builder builder, Metrics metrics) { NgqlProtocol ngql = metrics.getNgql(); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); + long startTimeNanos = System.nanoTime(); NebulaTemplate nebulaTemplate = new NebulaTemplate(); try { boolean initSuccess = nebulaTemplate.initSession(ngql); @@ -79,8 +77,7 @@ public void collect(Builder builder, Metrics metrics) { return; } - stopWatch.stop(); - long responseTime = stopWatch.getTotalTimeMillis(); + long responseTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); try { switch (ngql.getParseType()) { case PARSE_TYPE_FILTER_COUNT -> filterCount(nebulaTemplate, ngql, metrics.getAliasFields(), builder, responseTime); @@ -247,4 +244,10 @@ private Map showJobs(NebulaTemplate template, String protocolSpa result.put("running_jobs", String.valueOf(jobs.stream().filter(job -> Objects.equals(job.get("Status"), STATUS_RUNNING)).count())); return result; } + + private static void requireHasText(String value, String message) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException(message); + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml index ef3d9261cf3..92d05010fe1 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/pom.xml @@ -28,12 +28,6 @@ hertzbeat-collector-rocketmq ${project.artifactId} - - 17 - 17 - UTF-8 - - org.apache.hertzbeat @@ -47,4 +41,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java index c8c3889dd85..a98ff1ada44 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/main/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectImpl.java @@ -19,7 +19,6 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -28,10 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -40,6 +35,8 @@ import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; import org.apache.hertzbeat.collector.util.JsonPathParser; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; @@ -55,23 +52,21 @@ import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.util.Assert; /** * rocketmq collect */ @Slf4j -public class RocketmqSingleCollectImpl extends AbstractCollect implements DisposableBean { +public class RocketmqSingleCollectImpl extends AbstractCollect { private static final int WAIT_TIMEOUT = 10; + static final int QUEUE_CAPACITY = 5000; private static final Set SYSTEM_GROUP_SET = new HashSet<>(); - private final ExecutorService executorService; + private final ManagedExecutor executorService; static { // system consumer group @@ -86,24 +81,27 @@ public class RocketmqSingleCollectImpl extends AbstractCollect implements Dispos } public RocketmqSingleCollectImpl() { + this(createExecutor()); + } + + RocketmqSingleCollectImpl(ManagedExecutor executorService) { + this.executorService = executorService; + } + + private static ManagedExecutor createExecutor() { Runtime runtime = Runtime.getRuntime(); int corePoolSize = Math.max(8, runtime.availableProcessors()); int maximumPoolSize = Math.max(16, runtime.availableProcessors()); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setUncaughtExceptionHandler((thread, throwable) -> { - log.error("RocketMQCollectGroup has uncaughtException."); - log.error(throwable.getMessage(), throwable); - }) - .setDaemon(true) - .setNameFormat("rocketMQ-collector-%d") - .build(); - this.executorService = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(5000), threadFactory, new ThreadPoolExecutor.DiscardOldestPolicy()); + Thread.UncaughtExceptionHandler handler = (thread, throwable) -> { + log.error("RocketMQCollectGroup has uncaughtException."); + log.error(throwable.getMessage(), throwable); + }; + return ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-collector", "rocketmq-collector-", + corePoolSize, maximumPoolSize, QUEUE_CAPACITY, handler); } - @Override public void destroy() { - ThreadUtils.shutdownGracefully(this.executorService, 10L, TimeUnit.SECONDS); + this.executorService.close(); } /** @@ -112,10 +110,10 @@ public void destroy() { */ @Override public void preCheck(Metrics metrics) throws IllegalArgumentException { - Assert.isTrue(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); + require(metrics != null && metrics.getRocketmq() != null, "Rocketmq collect must has rocketmq params"); RocketmqProtocol rocketmq = metrics.getRocketmq(); - Assert.hasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); - Assert.hasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); + requireHasText(rocketmq.getNamesrvHost(), "Rocketmq Protocol namesrvHost is required."); + requireHasText(rocketmq.getNamesrvPort(), "Rocketmq Protocol namesrvPort is required."); } @Override @@ -270,7 +268,7 @@ private void collectConsumerData(DefaultMQAdminExt mqAdminExt, RocketmqCollectDa if (SYSTEM_GROUP_SET.contains(consumerGroup)) { continue; } - executorService.submit(() -> { + executeConsumerTask(() -> { RocketmqCollectData.ConsumerInfo consumerInfo = new RocketmqCollectData.ConsumerInfo(); consumerInfoList.add(consumerInfo); consumerInfo.setConsumerGroup(consumerGroup); @@ -369,4 +367,18 @@ private void fillBuilder(RocketmqCollectData rocketmqCollectData, CollectRep.Met builder.addValueRow(valueRowBuilder.build()); } } + + void executeConsumerTask(Runnable runnable) { + executorService.execute(runnable); + } + + private static void require(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + private static void requireHasText(String value, String message) { + require(value != null && !value.trim().isEmpty(), message); + } } diff --git a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java index 9996e727276..b40ed2599d7 100644 --- a/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java +++ b/hertzbeat-collector/hertzbeat-collector-rocketmq/src/test/java/org/apache/hertzbeat/collector/collect/rocketmq/RocketmqSingleCollectTest.java @@ -19,12 +19,20 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.apache.hertzbeat.common.concurrent.ManagedExecutor; +import org.apache.hertzbeat.common.concurrent.ManagedExecutors; import org.apache.hertzbeat.common.entity.job.Metrics; import org.apache.hertzbeat.common.entity.job.protocol.RocketmqProtocol; import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,6 +47,13 @@ public void setUp() throws Exception { collect = new RocketmqSingleCollectImpl(); } + @AfterEach + void tearDown() { + if (collect != null) { + collect.destroy(); + } + } + @Test void preCheck() { // metrics is null @@ -97,4 +112,51 @@ void collect() { void supportProtocol() { assertEquals(DispatchConstants.PROTOCOL_ROCKETMQ, collect.supportProtocol()); } + + @Test + void executeConsumerTaskRunsOnVirtualThread() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean virtualThread = new AtomicBoolean(false); + + collect.executeConsumerTask(() -> { + virtualThread.set(Thread.currentThread().isVirtual()); + latch.countDown(); + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(virtualThread.get()); + } + + @Test + void executeConsumerTaskDiscardsOldestWhenQueueIsFull() throws InterruptedException { + ManagedExecutor executor = ManagedExecutors.newDiscardOldestVirtualExecutor("rocketmq-test", + "rocketmq-test-", 1, 1, 1, (thread, throwable) -> { + }); + RocketmqSingleCollectImpl testCollect = new RocketmqSingleCollectImpl(executor); + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch thirdStarted = new CountDownLatch(1); + AtomicBoolean secondExecuted = new AtomicBoolean(false); + try { + testCollect.executeConsumerTask(() -> { + firstStarted.countDown(); + try { + releaseFirst.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertTrue(firstStarted.await(5, TimeUnit.SECONDS)); + + testCollect.executeConsumerTask(() -> secondExecuted.set(true)); + testCollect.executeConsumerTask(thirdStarted::countDown); + + releaseFirst.countDown(); + assertTrue(thirdStarted.await(5, TimeUnit.SECONDS)); + assertFalse(secondExecuted.get()); + } finally { + releaseFirst.countDown(); + testCollect.destroy(); + } + } } diff --git a/hertzbeat-common-core/pom.xml b/hertzbeat-common-core/pom.xml index 8cccffe102a..949cf19ed53 100644 --- a/hertzbeat-common-core/pom.xml +++ b/hertzbeat-common-core/pom.xml @@ -28,6 +28,7 @@ hertzbeat-common-core ${project.artifactId} + Framework-agnostic shared runtime models, utilities, and protocol support. diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java new file mode 100644 index 00000000000..5f1a0e4fec2 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/AdmissionMode.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +/** + * Task admission mode for managed executors. + */ +public enum AdmissionMode { + + /** + * Start each task on its own virtual thread without an executor-level concurrency cap. + */ + UNBOUNDED_VT, + + /** + * Reject immediately when the configured concurrency cap has been reached. + */ + LIMIT_AND_REJECT, + + /** + * Block the submitter until a permit is available. + */ + LIMIT_AND_BLOCK +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java new file mode 100644 index 00000000000..030800faff5 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/BackgroundTaskExecutor.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.RejectedExecutionException; + +/** + * Generic background task executor abstraction for runtime components. + */ +public interface BackgroundTaskExecutor { + + /** + * Execute a short-lived task. + * + * @param runnable task + * @throws RejectedExecutionException when execution is rejected + */ + void execute(Runnable runnable) throws RejectedExecutionException; + + /** + * Execute a long-lived background task. + * + * @param runnable task + */ + void executeLongRunning(Runnable runnable); + + /** + * Release executor resources. + * + * @throws Exception close exception + */ + void destroy() throws Exception; +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java new file mode 100644 index 00000000000..c86d7a97d60 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.concurrent.Executor; + +/** + * Executor with a stable logical name and close semantics. + */ +public interface ManagedExecutor extends Executor, AutoCloseable { + + /** + * Logical executor name for logging and metrics tags. + * + * @return executor name + */ + String name(); + + @Override + void close(); +} diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java new file mode 100644 index 00000000000..24226deadae --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/concurrent/ManagedExecutors.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.concurrent; + +import java.util.ArrayDeque; +import java.util.Objects; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Factory methods for managed executors. + */ +public final class ManagedExecutors { + + private ManagedExecutors() { + } + + /** + * Wrap an existing executor service. + * + * @param name executor name + * @param executorService delegate executor service + * @return managed executor wrapper + */ + public static ManagedExecutor wrap(String name, ExecutorService executorService) { + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a per-task virtual-thread executor with optional admission control. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param mode admission mode + * @param maxConcurrentTasks max concurrent tasks for limited modes + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newVirtualExecutor(String name, String threadNamePrefix, AdmissionMode mode, + int maxConcurrentTasks, Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, semaphore(mode, maxConcurrentTasks), mode); + } + + /** + * Create a platform-thread-per-task executor for a small number of long-running tasks. + * + * @param name executor name + * @param threadNamePrefix thread name prefix + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newPlatformExecutor(String name, String threadNamePrefix, + Thread.UncaughtExceptionHandler handler) { + ThreadFactory threadFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + ExecutorService executorService = Executors.newThreadPerTaskExecutor(threadFactory); + return new DefaultManagedExecutor(name, executorService, null, AdmissionMode.UNBOUNDED_VT); + } + + /** + * Create a queued executor that preserves queue semantics while executing tasks on virtual threads. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity, {@code <= 0} means unbounded + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newQueuedVirtualExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for queued executors"); + } + return new QueuedVirtualManagedExecutor(name, threadNamePrefix, maxConcurrentTasks, queueCapacity, handler); + } + + /** + * Create a virtual-thread executor that preserves {@link java.util.concurrent.ThreadPoolExecutor} + * core/max/queue semantics with discard-oldest overflow handling. + * + * @param name executor name + * @param threadNamePrefix virtual-thread name prefix + * @param coreConcurrentTasks core concurrent tasks + * @param maxConcurrentTasks max concurrent tasks + * @param queueCapacity queue capacity + * @param handler uncaught exception handler + * @return managed executor + */ + public static ManagedExecutor newDiscardOldestVirtualExecutor(String name, String threadNamePrefix, + int coreConcurrentTasks, int maxConcurrentTasks, + int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + if (coreConcurrentTasks <= 0) { + throw new IllegalArgumentException("coreConcurrentTasks must be greater than zero"); + } + if (maxConcurrentTasks < coreConcurrentTasks) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than or equal to coreConcurrentTasks"); + } + if (queueCapacity <= 0) { + throw new IllegalArgumentException("queueCapacity must be greater than zero"); + } + return new DiscardOldestVirtualManagedExecutor(name, threadNamePrefix, coreConcurrentTasks, + maxConcurrentTasks, queueCapacity, handler); + } + + private static Semaphore semaphore(AdmissionMode mode, int maxConcurrentTasks) { + if (mode == AdmissionMode.UNBOUNDED_VT) { + return null; + } + if (maxConcurrentTasks <= 0) { + throw new IllegalArgumentException("maxConcurrentTasks must be greater than zero for limited executors"); + } + return new Semaphore(maxConcurrentTasks); + } + + private static final class DefaultManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final Semaphore permits; + private final AdmissionMode admissionMode; + + private DefaultManagedExecutor(String name, ExecutorService delegate, Semaphore permits, + AdmissionMode admissionMode) { + this.name = Objects.requireNonNull(name, "name"); + this.delegate = Objects.requireNonNull(delegate, "delegate"); + this.permits = permits; + this.admissionMode = Objects.requireNonNull(admissionMode, "admissionMode"); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + acquirePermit(); + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + releasePermit(); + } + }); + submitted = true; + } finally { + if (!submitted) { + releasePermit(); + } + } + } + + @Override + public void close() { + delegate.shutdownNow(); + } + + private void acquirePermit() { + if (permits == null) { + return; + } + switch (admissionMode) { + case LIMIT_AND_REJECT: + if (!permits.tryAcquire()) { + throw new RejectedExecutionException(name + " rejected task because concurrency limit was reached"); + } + break; + case LIMIT_AND_BLOCK: + try { + permits.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException(name + " interrupted while waiting for an execution permit", e); + } + break; + case UNBOUNDED_VT: + break; + default: + throw new IllegalStateException("Unsupported admission mode: " + admissionMode); + } + } + + private void releasePermit() { + if (permits != null) { + permits.release(); + } + } + } + + private static final class QueuedVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final ExecutorService dispatcher; + private final BlockingDeque queue; + private final Semaphore permits; + private final Semaphore permitSignals; + private final AtomicBoolean closed; + + private QueuedVirtualManagedExecutor(String name, String threadNamePrefix, int maxConcurrentTasks, + int queueCapacity, Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.queue = queueCapacity > 0 ? new LinkedBlockingDeque<>(queueCapacity) : new LinkedBlockingDeque<>(); + this.permits = new Semaphore(maxConcurrentTasks); + this.permitSignals = new Semaphore(0); + this.closed = new AtomicBoolean(false); + ThreadFactory dispatcherFactory = Thread.ofPlatform() + .daemon(true) + .name(threadNamePrefix + "dispatcher-", 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.dispatcher = Executors.newSingleThreadExecutor(dispatcherFactory); + this.dispatcher.execute(this::dispatchLoop); + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + if (closed.get()) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (!queue.offerLast(command)) { + throw new RejectedExecutionException(name + " rejected task because queue capacity was reached"); + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + dispatcher.shutdownNow(); + delegate.shutdownNow(); + queue.clear(); + } + + private void dispatchLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + Runnable command = queue.takeFirst(); + if (!permits.tryAcquire()) { + queue.putFirst(command); + permitSignals.acquire(); + continue; + } + submit(command); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + permits.release(); + permitSignals.release(); + } + }); + submitted = true; + } finally { + if (!submitted) { + permits.release(); + permitSignals.release(); + } + } + } + } + + private static final class DiscardOldestVirtualManagedExecutor implements ManagedExecutor { + + private final String name; + private final ExecutorService delegate; + private final int coreConcurrentTasks; + private final int maxConcurrentTasks; + private final ArrayDeque queue; + private final int queueCapacity; + private final Object lock; + private boolean closed; + private int runningTasks; + + private DiscardOldestVirtualManagedExecutor(String name, String threadNamePrefix, int coreConcurrentTasks, + int maxConcurrentTasks, int queueCapacity, + Thread.UncaughtExceptionHandler handler) { + this.name = Objects.requireNonNull(name, "name"); + ThreadFactory virtualFactory = Thread.ofVirtual() + .name(threadNamePrefix, 0) + .uncaughtExceptionHandler(handler) + .factory(); + this.delegate = Executors.newThreadPerTaskExecutor(virtualFactory); + this.coreConcurrentTasks = coreConcurrentTasks; + this.maxConcurrentTasks = maxConcurrentTasks; + this.queueCapacity = queueCapacity; + this.queue = new ArrayDeque<>(queueCapacity); + this.lock = new Object(); + this.closed = false; + this.runningTasks = 0; + } + + @Override + public String name() { + return name; + } + + @Override + public void execute(Runnable command) { + Objects.requireNonNull(command, "command"); + Runnable taskToStart = null; + synchronized (lock) { + if (closed) { + throw new RejectedExecutionException(name + " rejected task because executor is closed"); + } + if (runningTasks < coreConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else if (queue.size() < queueCapacity) { + queue.offerLast(command); + return; + } else if (runningTasks < maxConcurrentTasks) { + runningTasks++; + taskToStart = command; + } else { + queue.pollFirst(); + queue.offerLast(command); + return; + } + } + submit(taskToStart); + } + + @Override + public void close() { + synchronized (lock) { + if (closed) { + return; + } + closed = true; + queue.clear(); + } + delegate.shutdownNow(); + } + + private void submit(Runnable command) { + boolean submitted = false; + try { + delegate.execute(() -> { + try { + command.run(); + } finally { + onTaskComplete(); + } + }); + submitted = true; + } finally { + if (!submitted) { + synchronized (lock) { + runningTasks--; + } + throw new RejectedExecutionException(name + " rejected task because delegate submission failed"); + } + } + } + + private void onTaskComplete() { + Runnable nextTask = null; + synchronized (lock) { + if (closed) { + runningTasks--; + return; + } + nextTask = queue.pollFirst(); + if (nextTask == null) { + runningTasks--; + return; + } + } + submit(nextTask); + } + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/BaseKafkaProperties.java diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java new file mode 100644 index 00000000000..956b8a54140 --- /dev/null +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/config/VirtualThreadProperties.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.common.config; + +import org.apache.hertzbeat.common.concurrent.AdmissionMode; + +/** + * Framework-agnostic virtual-thread runtime configuration. + */ +public record VirtualThreadProperties( + boolean enabled, + PoolProperties collector, + PoolProperties common, + PoolProperties manager, + AlerterProperties alerter, + PoolProperties warehouse, + AsyncProperties async) { + + private static final int DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS = 512; + private static final int DEFAULT_MANAGER_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS = 64; + private static final int DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS = 10; + private static final int DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL = 4; + + public VirtualThreadProperties { + collector = normalizePool(collector, PoolProperties.collectorDefaults()); + common = common == null ? PoolProperties.commonDefaults() : common; + manager = normalizePool(manager, PoolProperties.managerDefaults()); + alerter = alerter == null ? AlerterProperties.defaults() : alerter; + warehouse = warehouse == null ? PoolProperties.warehouseDefaults() : warehouse; + async = async == null ? AsyncProperties.defaults() : async; + } + + public VirtualThreadProperties() { + this(true, PoolProperties.collectorDefaults(), PoolProperties.commonDefaults(), + PoolProperties.managerDefaults(), AlerterProperties.defaults(), + PoolProperties.warehouseDefaults(), AsyncProperties.defaults()); + } + + /** + * Create a detached properties instance with runtime defaults. + * + * @return defaults instance + */ + public static VirtualThreadProperties defaults() { + return new VirtualThreadProperties(); + } + + /** + * Pool-level configuration. + */ + public record PoolProperties( + AdmissionMode mode, + int maxConcurrentJobs) { + + public PoolProperties { + mode = mode == null ? AdmissionMode.UNBOUNDED_VT : mode; + } + + public PoolProperties() { + this(AdmissionMode.UNBOUNDED_VT, 0); + } + + public static PoolProperties collectorDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, defaultCollectorConcurrency()); + } + + public static PoolProperties warehouseDefaults() { + return new PoolProperties(); + } + + public static PoolProperties commonDefaults() { + return new PoolProperties(); + } + + public static PoolProperties managerDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_MANAGER_MAX_CONCURRENT_JOBS); + } + + public static PoolProperties alerterNotifyDefaults() { + return new PoolProperties(AdmissionMode.LIMIT_AND_REJECT, DEFAULT_NOTIFY_MAX_CONCURRENT_JOBS); + } + + private static int defaultCollectorConcurrency() { + return DEFAULT_COLLECTOR_MAX_CONCURRENT_JOBS; + } + } + + /** + * Alerter-specific executor configuration. + */ + public record AlerterProperties( + PoolProperties notifyPool, + int periodicMaxConcurrentJobs, + QueueProperties logWorker, + QueueProperties reduce, + QueueProperties windowEvaluator, + int notifyMaxConcurrentPerChannel) { + + public AlerterProperties { + notifyPool = normalizePool(notifyPool, PoolProperties.alerterNotifyDefaults()); + periodicMaxConcurrentJobs = periodicMaxConcurrentJobs <= 0 + ? DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS : periodicMaxConcurrentJobs; + logWorker = normalizeQueue(logWorker, QueueProperties.logWorkerDefaults()); + reduce = normalizeQueue(reduce, QueueProperties.reduceDefaults()); + windowEvaluator = normalizeQueue(windowEvaluator, QueueProperties.windowEvaluatorDefaults()); + notifyMaxConcurrentPerChannel = notifyMaxConcurrentPerChannel <= 0 + ? DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL : notifyMaxConcurrentPerChannel; + } + + public AlerterProperties() { + this(PoolProperties.alerterNotifyDefaults(), DEFAULT_PERIODIC_MAX_CONCURRENT_JOBS, + QueueProperties.logWorkerDefaults(), QueueProperties.reduceDefaults(), + QueueProperties.windowEvaluatorDefaults(), DEFAULT_NOTIFY_MAX_CONCURRENT_PER_CHANNEL); + } + + public static AlerterProperties defaults() { + return new AlerterProperties(); + } + } + + /** + * Queue-preserving executor configuration. + */ + public record QueueProperties( + int maxConcurrentJobs, + int queueCapacity) { + + public QueueProperties() { + this(0, 0); + } + + public static QueueProperties reduceDefaults() { + return new QueueProperties(2, 0); + } + + public static QueueProperties logWorkerDefaults() { + return new QueueProperties(10, 1000); + } + + public static QueueProperties windowEvaluatorDefaults() { + return new QueueProperties(2, 0); + } + } + + /** + * Async executor configuration. + */ + public record AsyncProperties( + boolean enabled, + int concurrencyLimit, + boolean rejectWhenLimitReached, + long taskTerminationTimeout) { + + public AsyncProperties() { + this(true, 256, true, 5000L); + } + + public static AsyncProperties defaults() { + return new AsyncProperties(); + } + } + + private static PoolProperties normalizePool(PoolProperties configured, PoolProperties defaults) { + if (configured == null) { + return defaults; + } + if (configured.mode() != AdmissionMode.UNBOUNDED_VT && configured.maxConcurrentJobs() <= 0) { + return new PoolProperties(configured.mode(), defaults.maxConcurrentJobs()); + } + return configured; + } + + private static QueueProperties normalizeQueue(QueueProperties configured, QueueProperties defaults) { + if (configured == null) { + return defaults; + } + int maxConcurrentJobs = configured.maxConcurrentJobs() <= 0 + ? defaults.maxConcurrentJobs() : configured.maxConcurrentJobs(); + int queueCapacity = configured.queueCapacity(); + if (queueCapacity <= 0 && defaults.queueCapacity() > 0) { + queueCapacity = defaults.queueCapacity(); + } + return new QueueProperties(maxConcurrentJobs, queueCapacity); + } +} diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java similarity index 89% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java index 2b2c3a5f6ae..c689b913949 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/dto/sms/SmsConfig.java @@ -21,17 +21,13 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; /** - * SMS configuration + * Framework-agnostic SMS runtime configuration. */ @Data @AllArgsConstructor @NoArgsConstructor -@Component -@ConfigurationProperties(prefix = "alerter.sms") public class SmsConfig { /** diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java similarity index 93% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java index 4b4e1d53265..e7fb6e52c1f 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Job.java @@ -17,7 +17,6 @@ package org.apache.hertzbeat.common.entity.job; - import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Collections; import java.util.Comparator; @@ -35,13 +34,11 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.common.entity.manager.ParamDefine; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; -import org.springframework.util.CollectionUtils; /** - * Collect task details + * Collect task details. */ @Data @AllArgsConstructor @@ -51,15 +48,15 @@ public class Job { /** - * Task Job id + * Task Job id. */ private long id; /** - * Tenant id + * Tenant id. */ private long tenantId = 0; /** - * Monitoring Task ID + * Monitoring Task ID. */ private long monitorId; /** @@ -68,11 +65,11 @@ public class Job { */ private Map metadata; /** - * bind labels + * bind labels. */ private Map labels; /** - * bind annotations + * bind annotations. */ private Map annotations; /** @@ -86,12 +83,12 @@ public class Job { */ private String category; /** - * Type of monitoring eg: linux | mysql | jvm + * Type of monitoring eg: linux | mysql | jvm. */ private String app; /** * The internationalized name of the monitoring type - * PING CONNECT + * PING CONNECT. */ private Map name; /** @@ -101,55 +98,54 @@ public class Job { */ private Map help; /** - * The monitor help link + * The monitor help link. */ private Map helpLink; /** - * Task dispatch start timestamp + * Task dispatch start timestamp. */ private long timestamp; /** - * Default task collection time interval (unit: second) eg: 30,60,600 + * Default task collection time interval (unit: second) eg: 30,60,600. */ private long defaultInterval = 600L; /** - * Refresh time list for one cycle of the job + * Refresh time list for one cycle of the job. */ private ConcurrentLinkedDeque intervals; /** - * Whether it is a recurring periodic task true is yes, false is no + * Whether it is a recurring periodic task true is yes, false is no. */ private boolean isCyclic = false; /** - * monitor input need params + * monitor input need params. */ - private List params; + private List params; /** - * Metrics configuration eg: cpu memory - * eg: cpu memory + * Metrics configuration eg: cpu memory. */ private List metrics; /** - * Monitoring configuration parameter properties and values eg: username password timeout host + * Monitoring configuration parameter properties and values eg: username password timeout host. */ private List configmap; /** - * Whether it is a service discovery job, true is yes, false is no + * Whether it is a service discovery job, true is yes, false is no. */ private boolean isSd = false; /** - * Whether to use the Prometheus proxy + * Whether to use the Prometheus proxy. */ private boolean prometheusProxyMode = false; /** - * Scheduling type: interval or cron + * Scheduling type: interval or cron. */ private String scheduleType = "interval"; /** - * Cron expression for scheduling, used when scheduleType is "cron" + * Cron expression for scheduling, used when scheduleType is "cron". */ private String cronExpression = null; @@ -160,7 +156,7 @@ public class Job { private Map envConfigmaps; /** - * collector use - timestamp when the task was scheduled by the time wheel + * collector use - timestamp when the task was scheduled by the time wheel. */ @JsonIgnore private transient long dispatchTime; @@ -179,13 +175,13 @@ public class Job { private transient LinkedList> priorMetrics; /** - * collector use - Temporarily store one-time task metrics response data + * collector use - Temporarily store one-time task metrics response data. */ @JsonIgnore private transient List responseDataTemp; /** - * collector use - construct to initialize metrics execution view + * collector use - construct to initialize metrics execution view. */ public synchronized void constructPriorMetrics() { long now = System.currentTimeMillis(); @@ -232,7 +228,7 @@ public synchronized void constructPriorMetrics() { } /** - * collector use - to get the next set of priority metric group tasks + * collector use - to get the next set of priority metric group tasks. * * @param metrics Current Metrics * @param first Is it the first time to get @@ -314,7 +310,7 @@ public void initIntervals() { } /** - * The greatest common divisor + * The greatest common divisor. */ public static long gcd(long a, long b) { while (b != 0) { @@ -326,7 +322,7 @@ public static long gcd(long a, long b) { } /** - * The least common multiple + * The least common multiple. */ public static long lcm(List array) { if (array != null && !array.isEmpty()) { @@ -340,9 +336,8 @@ public static long lcm(List array) { } /** - * * @param metricsIntervals A unique list composed of intervals for all metrics - * Generate a list of refresh intervals for metric collection + * Generate a list of refresh intervals for metric collection. */ public synchronized void generateMetricsIntervals(List metricsIntervals) { // 1. To find the least common multiple (LCM) of all metric refresh intervals @@ -368,7 +363,7 @@ public synchronized void generateMetricsIntervals(List metricsIntervals) { } public synchronized long getInterval() { - if (!CollectionUtils.isEmpty(this.intervals)) { + if (this.intervals != null && !this.intervals.isEmpty()) { Long interval = this.intervals.removeFirst(); if (interval != null) { this.intervals.addLast(interval); diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java similarity index 100% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/Metrics.java diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java similarity index 56% rename from hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java rename to hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java index f93093157d2..eae3e24e286 100644 --- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/job/Configmap.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/entity/job/RuntimeParamDefine.java @@ -17,38 +17,60 @@ package org.apache.hertzbeat.common.entity.job; -import java.io.Serializable; +import java.util.List; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; /** - * Monitoring configuration parameter properties and values - * During the process, you need to replace the content with the identifier ^_^key^_^ - * in the protocol configuration parameter with the real value in the configuration parameter + * Framework-agnostic parameter definition used by runtime templates and jobs. */ @Data +@Builder @AllArgsConstructor @NoArgsConstructor -@Builder -public class Configmap implements Serializable { +public class RuntimeParamDefine { - /** - * Parameter key, replace the content with the identifier ^^_key_^^ in the protocol - * configuration parameter with the real value in the configuration parameter - */ - private String key; + private String app; - /** - * parameter value - */ - private Object value; + private Map name; + + private String field; + + private String type; + + private boolean required = false; + + private String defaultValue; + + private String placeholder; + + private String range; + + private Short limit; + + private List