From 9d17f2b5c6d85ad72d4812f580e84a4e938d0cf7 Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Sat, 24 Jan 2026 18:57:12 +0530 Subject: [PATCH 01/24] Fixed persister error queue and batch processing along with adding additional batch configuration --- .../PersisterBatchConsumerConfig.java | 36 +++++++++-- .../consumer/PersisterBatchListner.java | 62 +++++++++++++++---- .../consumer/PersisterConsumerConfig.java | 37 ++++++++--- .../consumer/PersisterMessageListener.java | 15 ++--- .../src/main/resources/application.properties | 4 ++ 5 files changed, 123 insertions(+), 31 deletions(-) diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java index 32f6bc90585..e2bca1e79fd 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java @@ -26,9 +26,13 @@ import org.springframework.kafka.support.serializer.JsonDeserializer; import jakarta.annotation.PostConstruct; +import org.springframework.util.StringUtils; + +import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Configuration @@ -60,14 +64,37 @@ public class PersisterBatchConsumerConfig { @Value("${persister.batch.size}") private Integer batchSize; + @Value("${persister.batch.topics:}") + private String batchTopicsConfig; + + private Set configuredBatchTopics = new HashSet<>(); + @PostConstruct public void setTopics() { + // Parse configured batch topics from property + if (StringUtils.hasText(batchTopicsConfig)) { + configuredBatchTopics = Arrays.stream(batchTopicsConfig.split(",")) + .map(String::trim) + .filter(StringUtils::hasText) + .collect(Collectors.toSet()); + log.info("Configured batch topics from property: {}", configuredBatchTopics); + } + + // Add topics that either contain "-batch" OR are in the configured list topicMap.getTopicMap().keySet().forEach(topic -> { - if(topic.contains("-batch")){ + if (topic.contains("-batch") || configuredBatchTopics.contains(topic)) { topics.add(topic); } }); - log.info("Topics subscribed for batch listner: "+topics.toString()); + log.info("Topics subscribed for batch listener: {}", topics); + } + + public Set getConfiguredBatchTopics() { + return configuredBatchTopics; + } + + public Set getBatchTopics() { + return topics; } @Bean("consumerFactoryBatch") @@ -112,12 +139,13 @@ public KafkaMessageListenerContainer container() throws Exceptio // set more properties // properties.setPauseEnabled(true); // properties.setPauseAfter(0); - // properties.setGenericErrorHandler(kafkaConsumerErrorHandler); properties.setMessageListener(indexerMessageListener); log.info("Custom KafkaListenerContainer built..."); - return new KafkaMessageListenerContainer<>(consumerFactory(), properties); + KafkaMessageListenerContainer container = new KafkaMessageListenerContainer<>(consumerFactory(), properties); + container.setCommonErrorHandler(kafkaConsumerErrorHandler); + return container; } @Bean("startBatchContainer") diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java index 806693d943d..9daf2422baa 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java @@ -7,16 +7,16 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; +import org.egov.tracer.kafka.ErrorQueueProducer; +import org.egov.tracer.model.ErrorQueueContract; +import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.listener.BatchMessageListener; import org.springframework.stereotype.Service; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; + @Service @Slf4j @@ -31,15 +31,24 @@ public class PersisterBatchListner implements BatchMessageListener> dataList) { + long startTime = System.currentTimeMillis(); + int totalRecordsSuccess = 0; + int totalRecordsFailed = 0; + Map> topicTorcvDataList = new HashMap<>(); dataList.forEach(data -> { @@ -55,21 +64,50 @@ public void onMessage(List> dataList) { } catch (JsonProcessingException e) { log.error("Failed to serialize incoming message", e); + pushToErrorQueue(data.topic(), data.value(), e); } }); for(Map.Entry> entry : topicTorcvDataList.entrySet()){ - persistService.persist(entry.getKey(),entry.getValue()); - if(!entry.getKey().equalsIgnoreCase(persistAuditKafkaTopic)){ - Map producerRecord = new HashMap<>(); - producerRecord.put("topic", entry.getKey()); - producerRecord.put("value", entry.getValue()); - kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + try { + persistService.persist(entry.getKey(),entry.getValue()); + if(!entry.getKey().equalsIgnoreCase(persistAuditKafkaTopic)){ + Map producerRecord = new HashMap<>(); + producerRecord.put("topic", entry.getKey()); + producerRecord.put("value", entry.getValue()); + kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + } + totalRecordsSuccess += entry.getValue().size(); + } catch (Exception e) { + log.error("Error while persisting message from topic: {}", entry.getKey(), e); + entry.getValue().forEach(data -> { + pushToErrorQueue(entry.getKey(), data, e); + }); + totalRecordsFailed += entry.getValue().size(); } } - } + long timeTaken = System.currentTimeMillis() - startTime; + log.info("Total messages persisted successfully: {}, failed: {}, time taken: {} ms", totalRecordsSuccess, totalRecordsFailed, timeTaken); + } + private void pushToErrorQueue(String topic, Object body, Exception e) { + try { + ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() + .id(UUID.randomUUID().toString()) + .source(topic) + .body(body) + .ts(System.currentTimeMillis()) + .message(e.getMessage()) + .exception(Arrays.asList(e.getStackTrace())) + .correlationId(MDC.get(CORRELATION_ID_MDC)) + .build(); + errorQueueProducer.sendMessage(errorQueueContract); + log.info("Message pushed to error queue for topic: {}", topic); + } catch (Exception ex) { + log.error("Failed to push message to error queue for topic: {}", topic, ex); + } + } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java index 23bd8ff550e..e89642383d4 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java @@ -23,9 +23,14 @@ import org.springframework.kafka.support.serializer.JsonDeserializer; import jakarta.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.util.StringUtils; + +import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Configuration @@ -51,14 +56,29 @@ public class PersisterConsumerConfig { private Set topics = new HashSet<>(); + @Value("${persister.batch.topics:}") + private String batchTopicsConfig; + + private Set configuredBatchTopics = new HashSet<>(); + @PostConstruct - public void setTopics(){ + public void setTopics() { + // Parse configured batch topics from property + if (StringUtils.hasText(batchTopicsConfig)) { + configuredBatchTopics = Arrays.stream(batchTopicsConfig.split(",")) + .map(String::trim) + .filter(StringUtils::hasText) + .collect(Collectors.toSet()); + log.info("Configured batch topics from property: {}", configuredBatchTopics); + } + + // Add topics that do NOT contain "-batch" AND are NOT in the configured batch list topicMap.getTopicMap().keySet().forEach(topic -> { - if(!topic.contains("-batch")){ - topics.add(topic); - } - }); - log.info("Topics subscribed for single listner: "+topics.toString()); + if (!topic.contains("-batch") && !configuredBatchTopics.contains(topic)) { + topics.add(topic); + } + }); + log.info("Topics subscribed for single listener: {}", topics); } @Bean @@ -96,12 +116,13 @@ public KafkaMessageListenerContainer container() throws Exceptio // set more properties // properties.setPauseEnabled(true); // properties.setPauseAfter(0); - // properties.setGenericErrorHandler(kafkaConsumerErrorHandler); properties.setMessageListener(indexerMessageListener); log.info("Custom KafkaListenerContainer built..."); - return new KafkaMessageListenerContainer<>(consumerFactory(), properties); + KafkaMessageListenerContainer container = new KafkaMessageListenerContainer<>(consumerFactory(), properties); + container.setCommonErrorHandler(kafkaConsumerErrorHandler); + return container; } @Bean diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index 632189368fd..a9c24035ec7 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -1,5 +1,6 @@ package org.egov.infra.persist.consumer; +import lombok.SneakyThrows; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; @@ -36,15 +37,13 @@ public class PersisterMessageListener implements MessageListener @Value("${audit.generate.kafka.topic}") private String auditGenerateKafkaTopic; - @Override + @SneakyThrows + @Override public void onMessage(ConsumerRecord data) { String rcvData = null; - - try { - rcvData = objectMapper.writeValueAsString(data.value()); - } catch (JsonProcessingException e) { - log.error("Failed to serialize incoming message", e); - } + long startTime = System.currentTimeMillis(); + + rcvData = objectMapper.writeValueAsString(data.value()); persistService.persist(data.topic(),rcvData); if(!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)){ @@ -53,6 +52,8 @@ public void onMessage(ConsumerRecord data) { producerRecord.put("value", data.value()); kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); } + + log.info("Message processed successfully in {} ms.", System.currentTimeMillis() - startTime); } } diff --git a/core-services/egov-persister/src/main/resources/application.properties b/core-services/egov-persister/src/main/resources/application.properties index 8a657e5239c..348e21e7d4d 100644 --- a/core-services/egov-persister/src/main/resources/application.properties +++ b/core-services/egov-persister/src/main/resources/application.properties @@ -27,6 +27,7 @@ spring.kafka.consumer.value-deserializer=org.egov.tracer.kafka.deserializer.Hash spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer spring.kafka.consumer.group-id=egov-infra-persist spring.kafka.consumer.auto_commit=true +spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.auto_commit_interval=100 spring.kafka.consumer.session_timeout_ms_config=15000 spring.kafka.consumer.auto_offset_reset=earliest @@ -50,6 +51,9 @@ tracer.errorsPublish=true persister.bulk.enabled=false persister.batch.size=100 +# Comma-separated list of topics to be processed in batch mode +# Topics containing "-batch" in name are automatically included +persister.batch.topics= default.version=1.0.0 From 2840986221b2187d90d71dae1d428433685db59c Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Sat, 24 Jan 2026 20:20:18 +0530 Subject: [PATCH 02/24] Fixed issue in transaction error throw --- .../java/org/egov/EgovPersistApplication.java | 2 + .../aspectj/TransactionInterceptorAspect.java | 26 ++++++++++ .../consumer/PersisterMessageListener.java | 51 +++++++++++++++---- .../infra/persist/service/PersistService.java | 17 +++---- 4 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java diff --git a/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java b/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java index 7bf25a4a617..b9a5ba85e98 100644 --- a/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java +++ b/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java @@ -15,6 +15,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.*; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; @@ -24,6 +25,7 @@ import java.io.InputStream; import java.util.*; +@EnableAspectJAutoProxy @SpringBootApplication @Slf4j public class EgovPersistApplication { diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java new file mode 100644 index 00000000000..559f0e90609 --- /dev/null +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java @@ -0,0 +1,26 @@ +package org.egov.infra.persist.aspectj; + +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.kafka.listener.ListenerExecutionFailedException; +import org.springframework.stereotype.Component; + +@Slf4j +@Aspect +@Component +public class TransactionInterceptorAspect { + + @Around("@annotation(org.springframework.transaction.annotation.Transactional)") + public Object aroundTransactional(ProceedingJoinPoint joinPoint) throws Throwable { + try { + Object result = joinPoint.proceed(); + log.info("Transactional method succeeded: {}", joinPoint.getSignature()); + return result; + } catch (Exception e) { + log.error("Transactional method failed: {}", joinPoint.getSignature()); + throw new ListenerExecutionFailedException(e.getMessage(), e); + } + } +} diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index a9c24035ec7..3d23ff38a2e 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -4,6 +4,9 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; +import org.egov.tracer.kafka.ErrorQueueProducer; +import org.egov.tracer.model.ErrorQueueContract; +import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.core.KafkaTemplate; @@ -14,9 +17,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.springframework.transaction.annotation.Transactional; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.UUID; + +import static org.egov.tracer.constants.TracerConstants.CORRELATION_ID_MDC; @Service @Slf4j @@ -37,23 +45,46 @@ public class PersisterMessageListener implements MessageListener @Value("${audit.generate.kafka.topic}") private String auditGenerateKafkaTopic; - @SneakyThrows + @Autowired + private ErrorQueueProducer errorQueueProducer; + @Override public void onMessage(ConsumerRecord data) { String rcvData = null; long startTime = System.currentTimeMillis(); - rcvData = objectMapper.writeValueAsString(data.value()); - persistService.persist(data.topic(),rcvData); - - if(!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)){ - Map producerRecord = new HashMap<>(); - producerRecord.put("topic", data.topic()); - producerRecord.put("value", data.value()); - kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + try { + rcvData = objectMapper.writeValueAsString(data.value()); + persistService.persist(data.topic(),rcvData); + if(!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)){ + Map producerRecord = new HashMap<>(); + producerRecord.put("topic", data.topic()); + producerRecord.put("value", data.value()); + kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + } + log.info("Message processed successfully in {} ms.", System.currentTimeMillis() - startTime); + } catch (Exception e) { + log.error("Error while persisting message from topic: {}", data.topic(), e); + pushToErrorQueue(data.topic(), data.value(), e); } + } - log.info("Message processed successfully in {} ms.", System.currentTimeMillis() - startTime); + private void pushToErrorQueue(String topic, Object body, Exception e) { + try { + ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() + .id(UUID.randomUUID().toString()) + .source(topic) + .body(body) + .ts(System.currentTimeMillis()) + .message(e.getMessage()) + .exception(Arrays.asList(e.getStackTrace())) + .correlationId(MDC.get(CORRELATION_ID_MDC)) + .build(); + errorQueueProducer.sendMessage(errorQueueContract); + log.info("Message pushed to error queue for topic: {}", topic); + } catch (Exception ex) { + log.error("Failed to push message to error queue for topic: {}", topic, ex); + } } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java index 464dd102757..b2e140901f5 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java @@ -63,21 +63,20 @@ public void persist(String topic, List jsons) { applicableMappings.put(document, filterMappings(map.get(topic), document)); } - applicableMappings.forEach((jsonObj, mappings) -> { + for (Map.Entry> entry : applicableMappings.entrySet()) { + Object jsonObj = entry.getKey(); + List mappings = entry.getValue(); + for (Mapping mapping : mappings) { - List queryMaps = mapping.getQueryMaps(); - for (QueryMap queryMap : queryMaps) { + for (QueryMap queryMap : mapping.getQueryMaps()) { + String query = queryMap.getQuery(); - List jsonMaps = queryMap.getJsonMaps(); String basePath = queryMap.getBasePath(); - - List rows = new LinkedList<>(persistRepository.getRows(jsonMaps, jsonObj, basePath)); - + List rows = persistRepository.getRows(queryMap.getJsonMaps(), jsonObj, basePath); persistRepository.persist(query, rows); } - } - }); + } } private List filterMappings(List mappings, Object json){ From ca8ae3fd35ce97b72055fd0ee63f4d14d38ae313 Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Sat, 24 Jan 2026 20:51:38 +0530 Subject: [PATCH 03/24] Allowing to filter out batch topics only when batch persister is enabled --- .../egov/infra/persist/consumer/PersisterConsumerConfig.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java index e89642383d4..113f9d95b83 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java @@ -59,12 +59,15 @@ public class PersisterConsumerConfig { @Value("${persister.batch.topics:}") private String batchTopicsConfig; + @Value("${persister.bulk.enabled:false}") + private Boolean batchPersisterEnabled; + private Set configuredBatchTopics = new HashSet<>(); @PostConstruct public void setTopics() { // Parse configured batch topics from property - if (StringUtils.hasText(batchTopicsConfig)) { + if (batchPersisterEnabled && StringUtils.hasText(batchTopicsConfig)) { configuredBatchTopics = Arrays.stream(batchTopicsConfig.split(",")) .map(String::trim) .filter(StringUtils::hasText) From 61d6b31f8e6c1a2433f908de87ab927e64d1fe7b Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Sun, 25 Jan 2026 16:51:54 +0530 Subject: [PATCH 04/24] Added task executor for controlled execution of transations and dead letter queue re-processor --- .../PersisterBatchConsumerConfig.java | 16 +++++ .../consumer/PersisterConsumerConfig.java | 30 ++++++-- .../consumer/PersisterMessageListener.java | 71 +++++++++++++++---- .../src/main/resources/application.properties | 8 +++ 4 files changed, 108 insertions(+), 17 deletions(-) diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java index e2bca1e79fd..74782bf089a 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java @@ -15,6 +15,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; @@ -26,12 +27,16 @@ import org.springframework.kafka.support.serializer.JsonDeserializer; import jakarta.annotation.PostConstruct; +import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.util.StringUtils; import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -61,6 +66,12 @@ public class PersisterBatchConsumerConfig { private Set topics = new HashSet<>(); + @Value("${persister.custom.executor.batchMaxPoolSize}") + private Integer maxPoolSize; + + @Value("${persister.custom.executor.enabled}") + private Boolean customExecutorEnabled; + @Value("${persister.batch.size}") private Integer batchSize; @@ -140,6 +151,11 @@ public KafkaMessageListenerContainer container() throws Exceptio // properties.setPauseEnabled(true); // properties.setPauseAfter(0); properties.setMessageListener(indexerMessageListener); + if (customExecutorEnabled) { + ExecutorService executorService = Executors.newFixedThreadPool(maxPoolSize); + AsyncTaskExecutor taskExecutor = new ConcurrentTaskExecutor(executorService); + properties.setListenerTaskExecutor(taskExecutor); + } log.info("Custom KafkaListenerContainer built..."); diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java index 113f9d95b83..2d00d4fd48e 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java @@ -11,6 +11,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; @@ -24,12 +25,13 @@ import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.util.StringUtils; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -62,6 +64,18 @@ public class PersisterConsumerConfig { @Value("${persister.bulk.enabled:false}") private Boolean batchPersisterEnabled; + @Value("${persister.custom.executor.maxPoolSize}") + private Integer maxPoolSize; + + @Value("${persister.custom.executor.enabled}") + private Boolean customExecutorEnabled; + + @Value("${persister.deadLetter.reprocess.enabled}") + private Boolean deadLetterReprocessEnabled; + + @Value("${tracer.errorsTopic}") + private String deadLetterErrorTopic; + private Set configuredBatchTopics = new HashSet<>(); @PostConstruct @@ -81,6 +95,9 @@ public void setTopics() { topics.add(topic); } }); + if (deadLetterReprocessEnabled && StringUtils.hasText(deadLetterErrorTopic)) { + topics.add(deadLetterErrorTopic); + } log.info("Topics subscribed for single listener: {}", topics); } @@ -120,6 +137,11 @@ public KafkaMessageListenerContainer container() throws Exceptio // properties.setPauseEnabled(true); // properties.setPauseAfter(0); properties.setMessageListener(indexerMessageListener); + if (customExecutorEnabled) { + ExecutorService executorService = Executors.newFixedThreadPool(maxPoolSize); + AsyncTaskExecutor taskExecutor = new ConcurrentTaskExecutor(executorService); + properties.setListenerTaskExecutor(taskExecutor); + } log.info("Custom KafkaListenerContainer built..."); diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index 3d23ff38a2e..887ee5c1ce1 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -1,7 +1,7 @@ package org.egov.infra.persist.consumer; -import lombok.SneakyThrows; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.errors.SerializationException; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; import org.egov.tracer.kafka.ErrorQueueProducer; @@ -17,12 +17,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import org.springframework.transaction.annotation.Transactional; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import static org.egov.tracer.constants.TracerConstants.CORRELATION_ID_MDC; @@ -45,6 +41,12 @@ public class PersisterMessageListener implements MessageListener @Value("${audit.generate.kafka.topic}") private String auditGenerateKafkaTopic; + @Value("${persister.deadLetter.reprocess.error.topic}") + private String deadLetterReprocessErrorTopic; + + @Value("${tracer.errorsTopic}") + private String tracerErrorsTopic; + @Autowired private ErrorQueueProducer errorQueueProducer; @@ -53,19 +55,37 @@ public void onMessage(ConsumerRecord data) { String rcvData = null; long startTime = System.currentTimeMillis(); + String topic = data.topic(); + String deadLetterTopic = null; + Object body = null; + + + try { - rcvData = objectMapper.writeValueAsString(data.value()); - persistService.persist(data.topic(),rcvData); + if (Objects.equals(topic, tracerErrorsTopic)) { + LinkedHashMap message = (LinkedHashMap) data.value(); + topic = message.get("source").toString(); + body = message.get("body"); + deadLetterTopic = data.topic(); + } else { + body = data.value(); + } + rcvData = objectMapper.writeValueAsString(body); + persistService.persist(topic, rcvData); if(!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)){ Map producerRecord = new HashMap<>(); - producerRecord.put("topic", data.topic()); - producerRecord.put("value", data.value()); + producerRecord.put("topic", topic); + producerRecord.put("value", body); kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); } - log.info("Message processed successfully in {} ms.", System.currentTimeMillis() - startTime); + log.info("Message from topic: {} processed successfully in {} ms.", topic, System.currentTimeMillis() - startTime); } catch (Exception e) { - log.error("Error while persisting message from topic: {}", data.topic(), e); - pushToErrorQueue(data.topic(), data.value(), e); + log.error("Error while persisting message from topic: {}", topic, e); + if(deadLetterTopic == null) { + pushToErrorQueue(topic, body, e); + } else { + sendErrorMessage(deadLetterReprocessErrorTopic, deadLetterTopic, body, e); + } } } @@ -87,4 +107,29 @@ private void pushToErrorQueue(String topic, Object body, Exception e) { } } + public void sendErrorMessage(String errorTopic, String topic, Object body, Exception ex) { + ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() + .id(UUID.randomUUID().toString()) + .source(topic) + .body(body) + .ts(System.currentTimeMillis()) + .message(ex.getMessage()) + .exception(Arrays.asList(ex.getStackTrace())) + .correlationId(MDC.get(CORRELATION_ID_MDC)) + .build(); + try { + log.info("Sending message to topic - " + errorTopic); + kafkaTemplate.send(errorTopic, errorQueueContract); + } catch (SerializationException serializationException) { + log.info("SerializationException exception occurred while sending exception to error queue"); + try { + kafkaTemplate.send(errorTopic, objectMapper.writeValueAsString(errorQueueContract)); + } catch (JsonProcessingException e) { + log.error("exception occurred while converting ErrorQueueContract to json string", e); + } + } catch (Exception e) { + log.error("exception occurred while sending exception to error queue", e); + } + } + } diff --git a/core-services/egov-persister/src/main/resources/application.properties b/core-services/egov-persister/src/main/resources/application.properties index 348e21e7d4d..e5a83fe6c56 100644 --- a/core-services/egov-persister/src/main/resources/application.properties +++ b/core-services/egov-persister/src/main/resources/application.properties @@ -60,6 +60,14 @@ default.version=1.0.0 # Audit integration configs audit.persist.kafka.topic=audit-create audit.generate.kafka.topic=process-audit-records + +persister.custom.executor.enabled=true +persister.custom.executor.maxPoolSize=10 +persister.custom.executor.batchMaxPoolSize=1 + +persister.deadLetter.reprocess.enabled=true +persister.deadLetter.reprocess.error.topic=egov-persister-deadletter-processed + otel.traces.exporter=otlp otel.service.name=egov-persister otel.logs.exporter=none From f28e4fc3c47431aeee1253b93c4b4b0d4ee84a91 Mon Sep 17 00:00:00 2001 From: Holash Chand Date: Sun, 25 Jan 2026 19:14:10 +0530 Subject: [PATCH 05/24] Fixed batch processing logic --- .../aspectj/TransactionInterceptorAspect.java | 5 +- .../PersisterBatchConsumerConfig.java | 192 ++++++------------ .../consumer/PersisterBatchListner.java | 161 +++++++++++---- .../consumer/PersisterConsumerConfig.java | 14 +- .../consumer/PersisterMessageListener.java | 5 +- .../persist/repository/PersistRepository.java | 21 +- .../infra/persist/service/PersistService.java | 82 +++++++- .../src/main/resources/application.properties | 10 +- .../PersisterBatchConsumerConfigTest.java | 29 --- 9 files changed, 287 insertions(+), 232 deletions(-) delete mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfigTest.java diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java index 559f0e90609..7d62a02c0d5 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java @@ -15,11 +15,8 @@ public class TransactionInterceptorAspect { @Around("@annotation(org.springframework.transaction.annotation.Transactional)") public Object aroundTransactional(ProceedingJoinPoint joinPoint) throws Throwable { try { - Object result = joinPoint.proceed(); - log.info("Transactional method succeeded: {}", joinPoint.getSignature()); - return result; + return joinPoint.proceed(); } catch (Exception e) { - log.error("Transactional method failed: {}", joinPoint.getSignature()); throw new ListenerExecutionFailedException(e.getMessage(), e); } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java index 74782bf089a..24794892e57 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java @@ -1,8 +1,8 @@ - - package org.egov.infra.persist.consumer; - +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; @@ -12,31 +12,19 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.listener.*; +import org.springframework.kafka.listener.BatchMessageListener; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.KafkaMessageListenerContainer; import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer; import org.springframework.kafka.support.serializer.JsonDeserializer; - -import jakarta.annotation.PostConstruct; -import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.util.StringUtils; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.*; import java.util.stream.Collectors; @@ -53,7 +41,7 @@ public class PersisterBatchConsumerConfig { private StoppingErrorHandler stoppingErrorHandler;*/ @Autowired - private BatchMessageListener indexerMessageListener; + private BatchMessageListener batchMessageListener; @Autowired private TopicMap topicMap; @@ -64,148 +52,92 @@ public class PersisterBatchConsumerConfig { @Autowired private KafkaConsumerErrorHandler kafkaConsumerErrorHandler; - private Set topics = new HashSet<>(); - - @Value("${persister.custom.executor.batchMaxPoolSize}") - private Integer maxPoolSize; - - @Value("${persister.custom.executor.enabled}") - private Boolean customExecutorEnabled; - @Value("${persister.batch.size}") private Integer batchSize; @Value("${persister.batch.topics:}") private String batchTopicsConfig; - private Set configuredBatchTopics = new HashSet<>(); + @Getter + private Set batchTopics = new HashSet<>(); + + private KafkaMessageListenerContainer batchContainer; @PostConstruct - public void setTopics() { + public void init() { + parseTopicConfigurations(); + createBatchContainer(); + } + + private void parseTopicConfigurations() { // Parse configured batch topics from property + Set configuredBatchTopics = new HashSet<>(); if (StringUtils.hasText(batchTopicsConfig)) { configuredBatchTopics = Arrays.stream(batchTopicsConfig.split(",")) .map(String::trim) .filter(StringUtils::hasText) .collect(Collectors.toSet()); - log.info("Configured batch topics from property: {}", configuredBatchTopics); } - // Add topics that either contain "-batch" OR are in the configured list + // Batch topics = topics containing "-batch" OR explicitly configured + Set finalConfiguredBatchTopics = configuredBatchTopics; topicMap.getTopicMap().keySet().forEach(topic -> { - if (topic.contains("-batch") || configuredBatchTopics.contains(topic)) { - topics.add(topic); + if (topic.contains("-batch") || finalConfiguredBatchTopics.contains(topic)) { + batchTopics.add(topic); } }); - log.info("Topics subscribed for batch listener: {}", topics); - } - - public Set getConfiguredBatchTopics() { - return configuredBatchTopics; - } - - public Set getBatchTopics() { - return topics; + log.info("Batch topics: {}", batchTopics); } - @Bean("consumerFactoryBatch") - public ConsumerFactory consumerFactory() { - Map props = kafkaProperties.buildConsumerProperties(); - - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); - props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000"); - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, batchSize); - - - JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class,false); - - ErrorHandlingDeserializer errorHandlingDeserializer - = new ErrorHandlingDeserializer<>(jsonDeserializer); - - return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), errorHandlingDeserializer); - - } - - @Bean("kafkaListenerContainerFactoryBatch") - public KafkaListenerContainerFactory> kafkaListenerContainerFactory() { - ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory()); - factory.setConcurrency(3); - factory.getContainerProperties().setPollTimeout(30000); - factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.BATCH); - factory.setCommonErrorHandler(kafkaConsumerErrorHandler); + /** + * Creates ONE container for all batch topics. + */ + private void createBatchContainer() { + if (batchTopics.isEmpty()) { + log.info("No batch topics configured, skipping batch container"); + return; + } + try { + ContainerProperties properties = new ContainerProperties(batchTopics.toArray(new String[0])); + properties.setMessageListener(batchMessageListener); + properties.setAckMode(ContainerProperties.AckMode.BATCH); - // BATCH PROPERTY - factory.setBatchListener(true); + batchContainer = new KafkaMessageListenerContainer<>(createConsumerFactory(), properties); + batchContainer.setCommonErrorHandler(kafkaConsumerErrorHandler); + batchContainer.setBeanName("batchContainer"); + batchContainer.start(); - log.info("Custom KafkaListenerContainerFactory built..."); - return factory; + log.info("Started batch container for {} topics: {}", batchTopics.size(), batchTopics); - } - - @Bean("batchContainer") - public KafkaMessageListenerContainer container() throws Exception { - ContainerProperties properties = new ContainerProperties(this.topics.toArray(new String[topics.size()])); - // set more properties - // properties.setPauseEnabled(true); - // properties.setPauseAfter(0); - properties.setMessageListener(indexerMessageListener); - if (customExecutorEnabled) { - ExecutorService executorService = Executors.newFixedThreadPool(maxPoolSize); - AsyncTaskExecutor taskExecutor = new ConcurrentTaskExecutor(executorService); - properties.setListenerTaskExecutor(taskExecutor); + } catch (Exception e) { + log.error("Failed to create batch container", e); } - - log.info("Custom KafkaListenerContainer built..."); - - KafkaMessageListenerContainer container = new KafkaMessageListenerContainer<>(consumerFactory(), properties); - container.setCommonErrorHandler(kafkaConsumerErrorHandler); - return container; } - @Bean("startBatchContainer") - public boolean startContainer() { - KafkaMessageListenerContainer container = null; - try { - container = container(); - } catch (Exception e) { - log.error("Container couldn't be started: ", e); - return false; - } - container.start(); - log.info("Custom KakfaListenerContainer STARTED..."); - return true; + private ConsumerFactory createConsumerFactory() { + Map props = kafkaProperties.buildConsumerProperties(); - } + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000"); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, batchSize); - public boolean pauseContainer() { - KafkaMessageListenerContainer container = null; - try { - container = container(); - } catch (Exception e) { - log.error("Container couldn't be started: ", e); - return false; - } - container.stop(); - log.info("Custom KakfaListenerContainer STOPPED..."); - return true; + JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class, false); + ErrorHandlingDeserializer errorHandlingDeserializer = new ErrorHandlingDeserializer<>(jsonDeserializer); + + return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), errorHandlingDeserializer); } - public boolean resumeContainer() { - KafkaMessageListenerContainer container = null; - try { - container = container(); - } catch (Exception e) { - log.error("Container couldn't be started: ", e); - return false; + @PreDestroy + public void shutdown() { + if (batchContainer != null) { + try { + batchContainer.stop(); + log.info("Stopped batch container"); + } catch (Exception e) { + log.error("Error stopping batch container", e); + } } - container.start(); - log.info("Custom KakfaListenerContainer STARTED..."); - - return true; } - -} - +} \ No newline at end of file diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java index 9daf2422baa..d27bfe0bd62 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java @@ -1,8 +1,9 @@ package org.egov.infra.persist.consumer; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.egov.infra.persist.service.PersistService; @@ -16,6 +17,8 @@ import org.springframework.stereotype.Service; import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; @Service @@ -40,56 +43,135 @@ public class PersisterBatchListner implements BatchMessageListener> dataList) { - long startTime = System.currentTimeMillis(); - int totalRecordsSuccess = 0; - int totalRecordsFailed = 0; - Map> topicTorcvDataList = new HashMap<>(); + // Step 1: Group messages by topic (preserving order within each topic) + Map> topicToDataList = new LinkedHashMap<>(); - dataList.forEach(data -> { - try { - if(!topicTorcvDataList.containsKey(data.topic())){ - List rcvDataList= new LinkedList<>(); - rcvDataList.add(objectMapper.writeValueAsString(data.value())); - topicTorcvDataList.put(data.topic(),rcvDataList); - } - else { - topicTorcvDataList.get(data.topic()).add(objectMapper.writeValueAsString(data.value())); - } + dataList.forEach(data -> { + try { + String jsonValue = objectMapper.writeValueAsString(data.value()); + topicToDataList.computeIfAbsent(data.topic(), k -> new ArrayList<>()).add(jsonValue); + } catch (JsonProcessingException e) { + log.error("Failed to serialize incoming message", e); + pushToErrorQueue(data.topic(), data.value(), e); + } + }); + + // Step 2: Process topics + int[] results = processTopics(topicToDataList); + + long timeTaken = System.currentTimeMillis() - startTime; + log.info("Batch processed - success: {}, failed: {}, topics: {}, time: {} ms", + results[0], results[1], topicToDataList.size(), timeTaken); + } + + /** + * Process topics - directly for single topic, parallel for multiple topics. + */ + private int[] processTopics(Map> topicToDataList) { + // Optimization: Direct processing for single topic (avoids thread pool overhead) + if (topicToDataList.size() == 1) { + Map.Entry> entry = topicToDataList.entrySet().iterator().next(); + return processSingleTopic(entry.getKey(), entry.getValue()); + } + + // Parallel processing for multiple topics + AtomicInteger totalRecordsSuccess = new AtomicInteger(0); + AtomicInteger totalRecordsFailed = new AtomicInteger(0); + + // Capture MDC context from parent thread + Map mdcContext = MDC.getCopyOfContextMap(); + + List> futures = new ArrayList<>(); + + for (Map.Entry> entry : topicToDataList.entrySet()) { + String topic = entry.getKey(); + List messages = entry.getValue(); + + CompletableFuture future = CompletableFuture.runAsync(() -> { + if (mdcContext != null) { + MDC.setContextMap(mdcContext); } - catch (JsonProcessingException e) { - log.error("Failed to serialize incoming message", e); - pushToErrorQueue(data.topic(), data.value(), e); + try { + int[] result = processSingleTopic(topic, messages); + totalRecordsSuccess.addAndGet(result[0]); + totalRecordsFailed.addAndGet(result[1]); + } finally { + MDC.clear(); } - }); + }, topicProcessorExecutor); - for(Map.Entry> entry : topicTorcvDataList.entrySet()){ - try { - persistService.persist(entry.getKey(),entry.getValue()); - if(!entry.getKey().equalsIgnoreCase(persistAuditKafkaTopic)){ - Map producerRecord = new HashMap<>(); - producerRecord.put("topic", entry.getKey()); - producerRecord.put("value", entry.getValue()); - kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); - } - totalRecordsSuccess += entry.getValue().size(); - } catch (Exception e) { - log.error("Error while persisting message from topic: {}", entry.getKey(), e); - entry.getValue().forEach(data -> { - pushToErrorQueue(entry.getKey(), data, e); - }); - totalRecordsFailed += entry.getValue().size(); - } + futures.add(future); } - long timeTaken = System.currentTimeMillis() - startTime; - log.info("Total messages persisted successfully: {}, failed: {}, time taken: {} ms", totalRecordsSuccess, totalRecordsFailed, timeTaken); + // Wait for all topics to complete + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } catch (CompletionException e) { + log.error("Error during parallel topic processing", e.getCause()); + } + return new int[]{totalRecordsSuccess.get(), totalRecordsFailed.get()}; + } + + /** + * Process a single topic: persist messages and send audit. + */ + private int[] processSingleTopic(String topic, List messages) { + if (messages.isEmpty()) { + return new int[]{0, 0}; + } + + try { + persistService.persist(topic, messages); + + // Send to audit topic if not audit topic itself + if (!topic.equalsIgnoreCase(persistAuditKafkaTopic)) { + Map producerRecord = new HashMap<>(); + producerRecord.put("topic", topic); + producerRecord.put("value", messages); + kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + } + + log.debug("Successfully processed {} messages from topic: {}", messages.size(), topic); + return new int[]{messages.size(), 0}; + + } catch (Exception e) { + log.error("Error while persisting messages from topic: {}", topic, e); + messages.forEach(data -> pushToErrorQueue(topic, data, e)); + return new int[]{0, messages.size()}; + } } private void pushToErrorQueue(String topic, Object body, Exception e) { @@ -109,5 +191,4 @@ private void pushToErrorQueue(String topic, Object body, Exception e) { log.error("Failed to push message to error queue for topic: {}", topic, ex); } } - -} +} \ No newline at end of file diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java index 2d00d4fd48e..011ac2b00ef 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java @@ -26,7 +26,6 @@ import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.util.StringUtils; import java.util.*; @@ -64,13 +63,13 @@ public class PersisterConsumerConfig { @Value("${persister.bulk.enabled:false}") private Boolean batchPersisterEnabled; - @Value("${persister.custom.executor.maxPoolSize}") + @Value("${persister.custom.executor.max-pool-size}") private Integer maxPoolSize; @Value("${persister.custom.executor.enabled}") private Boolean customExecutorEnabled; - @Value("${persister.deadLetter.reprocess.enabled}") + @Value("${persister.dead-letter.reprocess.enabled}") private Boolean deadLetterReprocessEnabled; @Value("${tracer.errorsTopic}") @@ -89,7 +88,7 @@ public void setTopics() { log.info("Configured batch topics from property: {}", configuredBatchTopics); } - // Add topics that do NOT contain "-batch" AND are NOT in the configured batch list + // Add topics that do NOT contain "-batch" AND are NOT in configured batch list topicMap.getTopicMap().keySet().forEach(topic -> { if (!topic.contains("-batch") && !configuredBatchTopics.contains(topic)) { topics.add(topic); @@ -108,7 +107,7 @@ public ConsumerFactory consumerFactory() { props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000"); - JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class,false); + JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class,false); ErrorHandlingDeserializer errorHandlingDeserializer = new ErrorHandlingDeserializer<>(jsonDeserializer); @@ -120,7 +119,7 @@ public ConsumerFactory consumerFactory() { public KafkaListenerContainerFactory> kafkaListenerContainerFactory() { ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); - factory.getContainerProperties(); + factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD); factory.setConcurrency(3); factory.getContainerProperties().setPollTimeout(30000); factory.setCommonErrorHandler(kafkaConsumerErrorHandler); @@ -133,9 +132,6 @@ public KafkaListenerContainerFactory container() throws Exception { ContainerProperties properties = new ContainerProperties(this.topics.toArray(new String[topics.size()])); - // set more properties - // properties.setPauseEnabled(true); - // properties.setPauseAfter(0); properties.setMessageListener(indexerMessageListener); if (customExecutorEnabled) { ExecutorService executorService = Executors.newFixedThreadPool(maxPoolSize); diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index 887ee5c1ce1..676d4cd2fda 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -41,7 +41,7 @@ public class PersisterMessageListener implements MessageListener @Value("${audit.generate.kafka.topic}") private String auditGenerateKafkaTopic; - @Value("${persister.deadLetter.reprocess.error.topic}") + @Value("${persister.dead-letter.reprocess.error-topic}") private String deadLetterReprocessErrorTopic; @Value("${tracer.errorsTopic}") @@ -58,9 +58,6 @@ public void onMessage(ConsumerRecord data) { String topic = data.topic(); String deadLetterTopic = null; Object body = null; - - - try { if (Objects.equals(topic, tracerErrorsTopic)) { LinkedHashMap message = (LinkedHashMap) data.value(); diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java index a96a3ac839d..d35956cb5ec 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java @@ -68,16 +68,27 @@ public List getRows(List jsonMaps, Object jsonObj, String bas List> dataSource = extractData(baseJsonPath, jsonObj); + if (dataSource == null || dataSource.isEmpty()) { + log.debug("No data found for basePath: {}", baseJsonPath); + return new ArrayList<>(); + } + List rows = new ArrayList<>(); + int nullRecords = 0; + int emptyChildRecords = 0; for (int i = 0; i < dataSource.size(); i++) { LinkedHashMap rawDataRecord = dataSource.get(i); - if (rawDataRecord == null) + if (rawDataRecord == null) { + nullRecords++; continue; + } - if (isChildObjectEmpty(baseJsonPath, rawDataRecord)) + if (isChildObjectEmpty(baseJsonPath, rawDataRecord)) { + emptyChildRecords++; continue; + } List row = new ArrayList<>(); @@ -185,6 +196,12 @@ else if (type.equals(TypeEnum.DATE) & value != null) { } rows.add(row.toArray()); } + + if (nullRecords > 0 || emptyChildRecords > 0) { + log.debug("getRows for basePath '{}': {} rows extracted, {} null records skipped, {} empty child records skipped", + baseJsonPath, rows.size(), nullRecords, emptyChildRecords); + } + return rows; } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java index b2e140901f5..730ccfce9dc 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/service/PersistService.java @@ -52,28 +52,90 @@ public void persist(String topic, String json) { } } + /** + * Optimized batch persist with query-level row aggregation. + * + * Instead of executing batchUpdate per message per QueryMap, + * this aggregates rows across all messages for each QueryMap + * and executes a single batchUpdate per QueryMap. + * + * Order preservation: + * 1. QueryMaps are processed in YAML-defined order (critical for FK dependencies) + * 2. Rows within each QueryMap preserve message order (first message's rows first) + * + * @param topic Kafka topic name + * @param jsons List of JSON messages from batch + */ @Transactional public void persist(String topic, List jsons) { Map> map = topicMap.getTopicMap(); - Map> applicableMappings = new LinkedHashMap<>(); - for (String json : jsons){ + // Step 1: Parse all documents and pair with their applicable mappings + // Using LinkedHashMap to preserve message order + Map> documentToMappings = new LinkedHashMap<>(); + + for (String json : jsons) { Object document = Configuration.defaultConfiguration().jsonProvider().parse(json); - applicableMappings.put(document, filterMappings(map.get(topic), document)); + documentToMappings.put(document, filterMappings(map.get(topic), document)); } - for (Map.Entry> entry : applicableMappings.entrySet()) { - Object jsonObj = entry.getKey(); + // Step 2: Group documents by mapping (using mapping name as key for identity) + // This handles the case where different messages might have different versions + Map> mappingNameToDocuments = new LinkedHashMap<>(); + Map mappingNameToMapping = new LinkedHashMap<>(); + + for (Map.Entry> entry : documentToMappings.entrySet()) { + Object document = entry.getKey(); List mappings = entry.getValue(); for (Mapping mapping : mappings) { - for (QueryMap queryMap : mapping.getQueryMaps()) { + String mappingKey = mapping.getName() + ":" + mapping.getVersion(); + mappingNameToDocuments.computeIfAbsent(mappingKey, k -> new ArrayList<>()).add(document); + mappingNameToMapping.putIfAbsent(mappingKey, mapping); + } + } + + // Step 3: For each mapping, process QueryMaps in order with aggregated rows + for (Map.Entry mappingEntry : mappingNameToMapping.entrySet()) { + String mappingKey = mappingEntry.getKey(); + Mapping mapping = mappingEntry.getValue(); + List documents = mappingNameToDocuments.get(mappingKey); + + log.info("Processing mapping '{}' with {} documents", mappingKey, documents.size()); + + // Process QueryMaps in YAML-defined order (critical for FK/delete-insert order) + for (QueryMap queryMap : mapping.getQueryMaps()) { + String query = queryMap.getQuery(); + String basePath = queryMap.getBasePath(); + List jsonMaps = queryMap.getJsonMaps(); + + // Aggregate rows from all documents for this QueryMap + // Preserves document order: first message's rows appear first + List aggregatedRows = new ArrayList<>(); + int skippedDocuments = 0; + + for (Object document : documents) { + try { + List rows = persistRepository.getRows(jsonMaps, document, basePath); + if (rows.isEmpty()) { + skippedDocuments++; + } + aggregatedRows.addAll(rows); + } catch (Exception e) { + skippedDocuments++; + log.warn("Failed to extract rows for basePath '{}': {}", basePath, e.getMessage()); + } + } - String query = queryMap.getQuery(); - String basePath = queryMap.getBasePath(); - List rows = persistRepository.getRows(queryMap.getJsonMaps(), jsonObj, basePath); - persistRepository.persist(query, rows); + // Single batchUpdate for all aggregated rows + if (!aggregatedRows.isEmpty()) { + log.info("Executing aggregated batch: {} rows for query (skipped {} docs, basePath: {})", + aggregatedRows.size(), skippedDocuments, basePath); + persistRepository.persist(query, aggregatedRows); + } else if (skippedDocuments > 0) { + log.warn("No rows to persist for basePath '{}' - all {} documents were skipped", + basePath, skippedDocuments); } } } diff --git a/core-services/egov-persister/src/main/resources/application.properties b/core-services/egov-persister/src/main/resources/application.properties index e5a83fe6c56..859817d6b6b 100644 --- a/core-services/egov-persister/src/main/resources/application.properties +++ b/core-services/egov-persister/src/main/resources/application.properties @@ -62,11 +62,13 @@ audit.persist.kafka.topic=audit-create audit.generate.kafka.topic=process-audit-records persister.custom.executor.enabled=true -persister.custom.executor.maxPoolSize=10 -persister.custom.executor.batchMaxPoolSize=1 +persister.custom.executor.max-pool-size=10 -persister.deadLetter.reprocess.enabled=true -persister.deadLetter.reprocess.error.topic=egov-persister-deadletter-processed +# Thread pool size for parallel topic processing within a batch +persister.batch.parallel-topic-processing.thread-pool-size=1 + +persister.dead-letter.reprocess.enabled=true +persister.dead-letter.reprocess.error-topic=egov-persister-deadletter-processed otel.traces.exporter=otlp otel.service.name=egov-persister diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfigTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfigTest.java deleted file mode 100644 index becdc4669bb..00000000000 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfigTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.egov.infra.persist.consumer; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -class PersisterBatchConsumerConfigTest { - - - @Test - void testStartContainer() { - - assertFalse((new PersisterBatchConsumerConfig()).startContainer()); - } - - @Test - void testPauseContainer() { - - assertFalse((new PersisterBatchConsumerConfig()).pauseContainer()); - } - - - @Test - void testResumeContainer() { - - assertFalse((new PersisterBatchConsumerConfig()).resumeContainer()); - } -} - From 05f6889a53ee535f63155475da11bd78f38f2d17 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 6 Jul 2026 18:56:06 +0530 Subject: [PATCH 06/24] Persister Changes --- .../configs/pgr-services-audit-persister.yml | 2 +- .../resources/configs/tradelicense-audit.yml | 18 +- core-services/boundary-service/README.md | 180 ++++++++++++- .../digit/config/ApplicationProperties.java | 5 + .../main/java/digit/errors/ErrorCodes.java | 14 ++ .../BoundaryRelationshipRepository.java | 16 ++ .../BoundaryRelationshipRepositoryImpl.java | 37 +++ .../BoundaryEntityQueryBuilder.java | 6 +- .../BoundaryRelationshipQueryBuilder.java | 14 +- .../service/BoundaryRelationshipService.java | 180 ++++++++++++- .../BoundaryRelationshipValidator.java | 31 ++- .../main/java/digit/util/HierarchyUtil.java | 2 +- .../BoundaryRelationshipController.java | 18 ++ .../BulkBoundaryRelationshipRequest.java | 43 ++++ .../BulkBoundaryRelationshipResponse.java | 44 ++++ .../models/FailedBoundaryRelationship.java | 34 +++ .../src/main/resources/application.properties | 24 +- .../src/main/resources/boundary-persister.yml | 6 +- ...__boundary_relationship_search_indexes.sql | 34 +++ .../java/org/egov/EgovPersistApplication.java | 4 +- .../aspectj/TransactionInterceptorAspect.java | 23 -- .../consumer/DbExceptionClassifier.java | 71 ++++++ .../persist/consumer/DbHealthMonitor.java | 58 +++++ .../PersisterBatchConsumerConfig.java | 34 ++- .../consumer/PersisterBatchListner.java | 162 +++++++----- .../consumer/PersisterConsumerConfig.java | 74 +++--- .../consumer/PersisterMessageListener.java | 236 ++++++++++-------- .../consumer/PersisterProducerConfig.java | 48 ++++ .../consumer/StoppingErrorHandler.java | 23 -- .../consumer/TransientPersistException.java | 19 ++ .../src/main/resources/application.properties | 14 +- .../resources/egov-pg-service-persister.yml | 4 +- .../resources/egov-user-event-persister.yml | 2 +- .../resources/hrms-employee-persister.yml | 30 +-- .../src/main/resources/persister.yml | 2 +- .../src/main/resources/pgr.v3.yml | 8 +- .../src/main/resources/property-services.yml | 14 +- .../src/main/resources/pt-drafts.yml | 2 +- .../resources/tl-billing-slab-persister.yml | 2 +- .../main/resources/user-service-persist.yml | 6 +- .../consumer/DbExceptionClassifierTest.java | 60 +++++ .../PersisterMessageListenerTest.java | 158 +++++++++--- .../main/resources/pg-service-persister.yml | 4 +- .../resources/egov-workflow-v2-persister.yml | 4 +- .../src/main/resources/mdms-persister.yml | 4 +- .../resources/service-request-persister.yml | 4 +- 46 files changed, 1410 insertions(+), 368 deletions(-) create mode 100644 core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequest.java create mode 100644 core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipResponse.java create mode 100644 core-services/boundary-service/src/main/java/digit/web/models/FailedBoundaryRelationship.java create mode 100644 core-services/boundary-service/src/main/resources/db/migration/main/V20260616120000__boundary_relationship_search_indexes.sql delete mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java create mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java create mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java delete mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/StoppingErrorHandler.java create mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/TransientPersistException.java create mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java diff --git a/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml b/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml index 4ca25d9eec3..51c31a6de9c 100644 --- a/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml +++ b/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml @@ -14,7 +14,7 @@ serviceMaps: queryMaps: - - query: INSERT INTO eg_pgr_service_v2(id, tenantid, servicecode, servicerequestid, description, accountid, additionaldetails, applicationstatus, source, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pgr_service_v2(id, tenantid, servicecode, servicerequestid, description, accountid, additionaldetails, applicationstatus, source, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, servicerequestid) DO NOTHING; basePath: $.service jsonMaps: - jsonPath: $.service.id diff --git a/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml b/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml index 71ab06e57bb..a083505b1b6 100644 --- a/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml +++ b/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml @@ -13,7 +13,7 @@ serviceMaps: auditAttributeBasePath: $.Licenses.* queryMaps: - - query: INSERT INTO eg_tl_tradelicense( id, accountid,tenantid,tradeName, validfrom,validto,licensetype,applicationNumber, licenseNumber, oldlicensenumber, propertyid, oldpropertyid, applicationdate, commencementdate, financialyear, action, status, createdby, lastmodifiedby, createdtime, lastmodifiedtime, businessservice, applicationtype, workflowcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_tradelicense( id, accountid,tenantid,tradeName, validfrom,validto,licensetype,applicationNumber, licenseNumber, oldlicensenumber, propertyid, oldpropertyid, applicationdate, commencementdate, financialyear, action, status, createdby, lastmodifiedby, createdtime, lastmodifiedtime, businessservice, applicationtype, workflowcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.* jsonMaps: - jsonPath: $.Licenses.*.id @@ -65,7 +65,7 @@ serviceMaps: - jsonPath: $.Licenses.*.workflowCode - - query: INSERT INTO eg_tl_tradelicensedetail( id, surveyno, subownershipcategory, channel, additionaldetail, tradelicenseid,structureType,operationalArea,noOfEmployees,adhocExemption,adhocPenalty,adhocExemptionReason,adhocPenaltyReason, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?); + - query: INSERT INTO eg_tl_tradelicensedetail( id, surveyno, subownershipcategory, channel, additionaldetail, tradelicenseid,structureType,operationalArea,noOfEmployees,adhocExemption,adhocPenalty,adhocExemptionReason,adhocPenaltyReason, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.id @@ -106,7 +106,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_tradeunit( id, tenantid,active, tradetype, uom, uomvalue, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_tradeunit( id, tenantid,active, tradetype, uom, uomvalue, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.tradeUnits.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.tradeUnits.*.id @@ -133,7 +133,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_accessory( id, tenantid,active, accessoryCategory, uom, uomvalue, count, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_accessory( id, tenantid,active, accessoryCategory, uom, uomvalue, count, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.accessories.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.accessories.*.id @@ -161,7 +161,7 @@ serviceMaps: - jsonPath: $.Licenses[*][?({id} in @.tradeLicenseDetail.accessories[*].id)].auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_owner( id,tenantid,active,institutionid, tradelicensedetailid, isprimaryowner, ownertype, ownershippercentage, relationship, createdby,lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_owner( id,tenantid,active,institutionid, tradelicensedetailid, isprimaryowner, ownertype, ownershippercentage, relationship, createdby,lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id, tradelicensedetailid) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.owners.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.owners.*.uuid @@ -192,7 +192,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_document_owner(id,tenantId,userid,active, tradeLicenseDetailId, documenttype, fileStoreId,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ? ,? ,?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_document_owner(id,tenantId,userid,active, tradeLicenseDetailId, documenttype, fileStoreId,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ? ,? ,?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.owners.*.documents.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.owners.*.documents.*.id @@ -221,7 +221,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_applicationdocument( id, tenantid, active, documenttype, tradecategorydetail, filestoreid, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_applicationdocument( id, tenantid, active, documenttype, tradecategorydetail, filestoreid, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.applicationDocuments.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.applicationDocuments.*.id @@ -247,7 +247,7 @@ serviceMaps: - jsonPath: $.Licenses[*][?({id} in @.tradeLicenseDetail.applicationDocuments[*].id)].auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_address( id, tenantid, doorno,street,buildingName, latitude,longitude, addressid, addressnumber,locality, type, addressline1, addressline2, landmark, city, pincode, detail, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_address( id, tenantid, doorno,street,buildingName, latitude,longitude, addressid, addressnumber,locality, type, addressline1, addressline2, landmark, city, pincode, detail, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.address jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.address.id @@ -295,7 +295,7 @@ serviceMaps: - jsonPath: $.Licenses.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_institution(tenantId,active,id,instituionName,contactNo,organisationRegistrationNo,address, tradelicensedetailid, name, type,designation, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ; + - query: INSERT INTO eg_tl_institution(tenantId,active,id,instituionName,contactNo,organisationRegistrationNo,address, tradelicensedetailid, name, type,designation, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Licenses.*.tradeLicenseDetail.institution jsonMaps: diff --git a/core-services/boundary-service/README.md b/core-services/boundary-service/README.md index a2e8a9f7b84..614a4952835 100644 --- a/core-services/boundary-service/README.md +++ b/core-services/boundary-service/README.md @@ -1,18 +1,176 @@ -# Swagger generated server +# Boundary Service -Spring Boot Server +Spring Boot service that manages a tenant's administrative hierarchy — boundary **entities**, boundary **hierarchy definitions**, and the boundary **relationships** that record, for each boundary, its type, parent, and materialized path within a hierarchy (country → province → district → … → village). +- **Java:** 17 +- **Framework:** Spring Boot 3.2.2 +- **Datastore:** PostgreSQL (Flyway migrations under `src/main/resources/db/migration/main`) +- **Messaging:** Kafka. Every write (entity/hierarchy/relationship create & update, single **and** bulk) is published to a topic and written to PostgreSQL by `egov-persister`. boundary-service itself performs **no direct DB write** — it only reads (for validation/search) and publishes. There is no Kafka consumer in this service. +- **Context path:** `/boundary-service`  •  **Default port:** `8081` -## Overview -This server was generated by the [swagger-codegen](https://github.com/swagger-api/swagger-codegen) project. -By using the [OpenAPI-Spec](https://github.com/swagger-api/swagger-core), you can easily generate a server stub. -This is an example of building a swagger-enabled server in Java using the SpringBoot framework. +--- -The underlying library integrating swagger to SpringBoot is [springfox](https://github.com/springfox/springfox) +## API surface -Start your server as an simple java application +All endpoints are `POST`. Base URL: `http://:8081/boundary-service`. -You can view the api documentation in swagger-ui by pointing to -http://localhost:8080/ +| Resource | Path | Purpose | Write model | +| --- | --- | --- | --- | +| Hierarchy definition | `/boundary-hierarchy-definition/_create` | Define the boundary-type order for a hierarchy | async (Kafka → persister) | +| Hierarchy definition | `/boundary-hierarchy-definition/_search` | Search hierarchy definitions | — | +| Boundary entity | `/boundary/_create` | Create boundary entities (geometry) | async (Kafka → persister) | +| Boundary entity | `/boundary/_search` | Search boundary entities | — | +| Boundary entity | `/boundary/_update` | Update boundary entities | async (Kafka → persister) | +| Boundary relationship | `/boundary-relationships/_create` | Create a **single** relationship | validate → publish → `202 Accepted` | +| Boundary relationship | **`/boundary-relationships/bulk/_create`** | **Create relationships in bulk** | validate+enrich **synchronously**, publish → `200 OK` | +| Boundary relationship | `/boundary-relationships/_search` | Search the relationship tree | — | +| Boundary relationship | `/boundary-relationships/_update` | Update a relationship's parent | async (Kafka → persister) | -Change default port value in application.properties \ No newline at end of file +### Single vs. bulk relationship create + +Both paths validate + enrich a relationship (assign `id`, audit details, and the ancestral materialized path) and then **publish it to the `save-boundary-relationship` topic**, which `egov-persister` writes with an idempotent `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`. Neither path writes to the database directly. + +- The **single** `_create` validates one record and publishes it, returning `202 Accepted` (acceptance, not a committed write). +- The **bulk** `_create` validates + enriches every record in the request **synchronously**, publishes the valid ones (one message per record), and returns `200 OK` with a **per-record outcome** so the caller learns immediately which records were accepted and which failed validation, and with what reason. + +See [`docs/Bulk-Boundary-Relationship-Creation-Design.docx`](docs/Bulk-Boundary-Relationship-Creation-Design.docx) for the design rationale and [`docs/Bulk-Boundary-Relationship-Flow.docx`](docs/Bulk-Boundary-Relationship-Flow.docx) for the end-to-end flow. + +--- + +## Bulk Boundary Relationship Creation + +`POST /boundary-relationships/bulk/_create` + +Creates many relationships in one request with a **per-record outcome**. Validation and enrichment happen synchronously on the request thread **before anything is placed on Kafka**; each valid record is then published to `save-boundary-relationship` for `egov-persister` to write. Individual record failures do **not** fail the whole request. + +### Semantics + +- **Request guards.** `RequestInfo.userInfo` must be present, and the request must carry `1 … boundary.bulk.max.size` (default `100`) relationships. These are enforced in-service (bean validation is not active in this deployment) and return a structured `400` (`BULK_REQUEST_INFO_MISSING` / `BULK_REQUEST_EMPTY` / `BULK_REQUEST_SIZE_EXCEEDED`). +- **Per-record validation.** Each record runs the same business rules as the single create (boundary entity exists, no duplicate, parent exists, correct hierarchy level; `code`/`tenantId`/`hierarchyType` must not contain the reserved `|` path delimiter). A record that fails validation is reported in `failedBoundaryRelationships`; the rest continue. +- **Persistence via egov-persister.** Validated + enriched records are published, **one message per record**, to `save-boundary-relationship`. The publish is blocking (it returns once the broker has accepted each record). `egov-persister` writes each with `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`, so redelivery / re-submission is a safe no-op and one un-insertable record never fails the others. A record accepted here is reported in `successfulBoundaryRelationships` ("accepted for persistence", not "already committed"). +- **Transient failures are retryable, not fatal.** A parent/entity not yet persisted, a transient DB error, or a publish failure (broker unreachable within `max.block.ms`) is reported with a retryable code (`PARENT_NOT_FOUND`, `BOUNDARY_ENTITY_DOES_NOT_EXIST`, `BULK_RELATIONSHIP_PERSIST_TRANSIENT`) rather than aborting; the caller retries only those records. +- **Intra-request de-duplication.** Two records with the same `(tenantId, hierarchyType, code)` in one request are rejected (`DUPLICATE_RECORD_IN_REQUEST`). + +### Caller contract + +The endpoint expects each batch to be a set of **siblings whose parent is already persisted**: + +1. Create the hierarchy **top-down**, one level at a time. Confirm an upper (single-create) level is persisted before creating its children. +2. For high-cardinality lower levels (locality, village), split into batches of ≤ `boundary.bulk.max.size` (100) where every record in a batch shares one parent, and submit them with bounded parallelism. +3. Treat the response per record: `DUPLICATE_RECORD` is an idempotent success; the retryable codes above should be retried (a bulk-level parent may still be committing — the persister writes asynchronously); any other code is a permanent data error. Re-submission targets only the missing records and is idempotent. + +### Scaling & reliability + +- **Horizontal scale** is by running more boundary-service replicas behind the load balancer (a stateless HTTP path) and by the caller submitting chunks concurrently — there is no Kafka consumer group to size and no per-partition head-of-line blocking. +- **Idempotency** comes from `INSERT … ON CONFLICT (tenantid, code, hierarchytype) DO NOTHING`, so a parent that is briefly late simply causes its children to be retried by the caller until it lands; ordering across levels is *not* required. +- **DB-write reliability** (per-record isolation of an un-insertable row, transient-DB retry, dead-lettering) is owned by `egov-persister`. Because it is one message per record, a bad record is isolated on its own regardless of whether the persister runs its normal listener or the optional batch listener. + +### Request + +```json +{ + "RequestInfo": { "apiId": "boundary", "ver": "1.0", "msgId": "...", "userInfo": { } }, + "BoundaryRelationships": [ + { "tenantId": "pg", "code": "VILLAGE_001", "hierarchyType": "ADMIN", "boundaryType": "Village", "parent": "LOCALITY_01" }, + { "tenantId": "pg", "code": "VILLAGE_002", "hierarchyType": "ADMIN", "boundaryType": "Village", "parent": "LOCALITY_01" } + ] +} +``` + +| Field | Required | Notes | +| --- | --- | --- | +| `code` | yes | Boundary code; a matching boundary **entity** must already exist. Must not contain `\|`. | +| `tenantId` | yes | Tenant. Must not contain `\|`. | +| `hierarchyType` | yes | Must have a hierarchy definition. Must not contain `\|`. | +| `boundaryType` | yes | Must be part of the hierarchy definition. | +| `parent` | no | Parent boundary code; required for every non-root level and must already be persisted. Omit only for the root level. | + +### Response — `200 OK` + +```json +{ + "ResponseInfo": { "status": "successful" }, + "successfulBoundaryRelationships": [ + { + "id": "a1b2...-uuid", + "tenantId": "pg", + "code": "VILLAGE_001", + "hierarchyType": "ADMIN", + "boundaryType": "Village", + "parent": "LOCALITY_01", + "auditDetails": { "createdBy": "...", "createdTime": 1718600000000, "lastModifiedBy": "...", "lastModifiedTime": 1718600000000 } + } + ], + "failedBoundaryRelationships": [ + { + "boundaryRelationship": { "tenantId": "pg", "code": "VILLAGE_002", "hierarchyType": "ADMIN", "boundaryType": "Village", "parent": "LOCALITY_99" }, + "errorCode": "PARENT_NOT_FOUND", + "errorMessage": "Parent entity for current boundary relationship does not exist." + } + ] +} +``` + +A partial outcome still returns `200 OK`; the caller inspects the two lists. `successfulBoundaryRelationships` is the set accepted for (idempotent, asynchronous) persistence; `failedBoundaryRelationships` is the work to retry or escalate. A whole-request rejection (bad envelope) returns a structured `400`. + +### Per-record error codes + +| `errorCode` | Meaning | Caller action | +| --- | --- | --- | +| `BOUNDARY_ENTITY_DOES_NOT_EXIST` | No boundary entity exists for the given `code` + `tenantId`. | retry | +| `PARENT_NOT_FOUND` | The referenced `parent` relationship is not (yet) persisted. | retry | +| `BULK_RELATIONSHIP_PERSIST_TRANSIENT` | A transient DB error during validation reads, or a publish failure. | retry | +| `DUPLICATE_RECORD` | A relationship with this `code` already exists. | idempotent success | +| `DUPLICATE_RECORD_IN_REQUEST` | The same `(tenantId, hierarchyType, code)` appears more than once in this request. | fix input | +| `BOUNDARY_TYPE_ERROR` | `boundaryType` is not part of the hierarchy definition. | fix input | +| `HIERARCHY_ERROR` | The record's level is not a direct child of its parent's level (or a non-root record has no parent). | fix input | +| `HIERARCHY_DEFINITION_DOES_NOT_EXIST_ERR` | No hierarchy definition for the `tenantId` + `hierarchyType`. | fix input | +| `HIERARCHY_DEFINITION_INVALID_ERR` | The hierarchy definition is malformed (no root boundary type / no node with an empty parent). | investigate | +| `INVALID_BOUNDARY_CODE` | `code`/`tenantId`/`hierarchyType` contains the reserved `\|` delimiter. | fix input | +| `BULK_RELATIONSHIP_VALIDATION_ERROR` | An unexpected runtime error during per-record validation/enrichment (isolated so the rest of the batch proceeds). | investigate | + +Whole-request `400` codes: `BULK_REQUEST_INFO_MISSING`, `BULK_REQUEST_EMPTY`, `BULK_REQUEST_SIZE_EXCEEDED`. + +--- + +## Build & run + +```bash +# Build +mvn clean install + +# Run (after configuring datasource & kafka in application.properties) +mvn spring-boot:run +# or +java -jar target/boundary-service-1.0.1.jar +``` + +### Key configuration (`src/main/resources/application.properties`) + +| Property | Default | Purpose | +| --- | --- | --- | +| `server.port` | `8081` | HTTP port | +| `spring.datasource.url` | `jdbc:postgresql://localhost:5432/postgres` | PostgreSQL connection (used for validation/search **reads** only) | +| `spring.flyway.enabled` | `false` | Enable to run DB migrations on startup | +| `spring.kafka.bootstrap-servers` | `localhost:9092` | Kafka brokers used by the producer. In DIGIT deployments the common Helm chart injects `SPRING_KAFKA_BOOTSTRAP_SERVERS`. | +| `spring.kafka.producer.properties.max.block.ms` | `15000` | Bounds how long a synchronous publish blocks if the broker is unreachable, so an outage fails fast (as a transient error the caller retries) instead of tying up request threads. | +| `kafka.topics.create.boundary.relationship` | `save-boundary-relationship` | Topic for **both** single and bulk relationship create, consumed by `egov-persister`. | +| `boundary.bulk.max.size` | `100` | Max records accepted by `/bulk/_create` (enforced in-service; keep ≥ the caller's chunk size). | +| `boundary.default.limit` / `boundary.max.default.limit` | `50` / `300` | Search paging defaults | + +> Bulk creation writes through `egov-persister`, not directly to PostgreSQL, so the request path is not gated by the DB connection pool for writes. To make the persister aggregate high-volume creates into batched multi-row inserts, add `save-boundary-relationship` to the persister's `persister.batch.topics` (with `persister.bulk.enabled=true`); this is an **optional throughput optimization** — the topic is otherwise consumed one record at a time by the persister's normal listener, so bulk creation works on any persister deployment with no extra configuration. + +--- + +## Database + +Relationships are stored in `boundary_relationship` (`tenantId, code, hierarchyType` primary key; `ancestralMaterializedPath` holds the `|`-delimited ancestor chain used for subtree search). All writes go through the `egov-persister` mapping in `src/main/resources/boundary-persister.yml`, which uses `INSERT … ON CONFLICT (tenantid, code, hierarchytype) DO NOTHING` so at-least-once redelivery and caller re-submission are idempotent. + +### Search indexes + +Migration `V20260616120000__boundary_relationship_search_indexes.sql` adds two indexes that keep relationship/subtree search off full table scans (which, under concurrent campaign-scale search, were exhausting the connection pool): + +- a **GIN** index on `string_to_array(ancestralmaterializedpath, '|')` — serves the `ARRAY[…] && …` subtree-overlap predicate; +- a `(tenantid, parent)` index — serves the `parent = ?` and root (`parent IS NULL`) branches. + +They are plain (non-`CONCURRENTLY`) `CREATE INDEX IF NOT EXISTS` on purpose — `CONCURRENTLY` hangs inside a Flyway migration. For a very large existing table, build them out-of-band instead. See the migration header for details. diff --git a/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java b/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java index 82ef53dcf36..fbc569aeb7e 100644 --- a/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java +++ b/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java @@ -99,6 +99,11 @@ public class ApplicationProperties { @Value("${kafka.topics.update.boundary.relationship}") private String updateBoundaryRelationshipTopic; + // Upper bound on records accepted by POST /boundary-relationships/bulk/_create. Enforced in the + // service (bean validation is not active in this deployment). Keep in sync with the caller's chunk size. + @Value("${boundary.bulk.max.size:100}") + private Integer bulkCreateMaxSize; + @Value("${boundary.default.offset}") private Integer defaultOffset; diff --git a/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java b/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java index 6418b51f88f..f9efbfa76f0 100644 --- a/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java +++ b/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java @@ -30,5 +30,19 @@ public class ErrorCodes { public static final String MULTIPLE_ROOT_NODES_ERR_MSG = "Hierarchy definition must have only one root node."; public static final String DUPLICATE_BOUNDARY_CODE = "DUPLICATE_BOUNDARY"; public static final String DUPLICATE_BOUNDARY_MSG = "Duplicate boundary entities found in the request."; + public static final String DUPLICATE_RECORD_IN_REQUEST_CODE = "DUPLICATE_RECORD_IN_REQUEST"; + public static final String DUPLICATE_RECORD_IN_REQUEST_MSG = "Duplicate boundary relationship (same tenantId, hierarchyType and code) found within the bulk request."; + public static final String BULK_RELATIONSHIP_VALIDATION_ERROR_CODE = "BULK_RELATIONSHIP_VALIDATION_ERROR"; + public static final String BULK_RELATIONSHIP_VALIDATION_ERROR_MSG = "Boundary relationship could not be validated due to an unexpected error."; + public static final String BULK_RELATIONSHIP_PERSIST_TRANSIENT_CODE = "BULK_RELATIONSHIP_PERSIST_TRANSIENT"; + public static final String BULK_RELATIONSHIP_PERSIST_TRANSIENT_MSG = "Boundary relationship persistence failed transiently and will be retried."; + public static final String BULK_REQUEST_EMPTY_CODE = "BULK_REQUEST_EMPTY"; + public static final String BULK_REQUEST_EMPTY_MSG = "Bulk boundary relationship request must contain at least one relationship."; + public static final String BULK_REQUEST_SIZE_EXCEEDED_CODE = "BULK_REQUEST_SIZE_EXCEEDED"; + public static final String BULK_REQUEST_SIZE_EXCEEDED_MSG = "Bulk boundary relationship request exceeds the maximum allowed size of "; + public static final String BULK_REQUEST_INFO_MISSING_CODE = "BULK_REQUEST_INFO_MISSING"; + public static final String BULK_REQUEST_INFO_MISSING_MSG = "Bulk boundary relationship request is missing RequestInfo.userInfo."; + public static final String INVALID_BOUNDARY_CODE_CODE = "INVALID_BOUNDARY_CODE"; + public static final String INVALID_BOUNDARY_CODE_MSG = "code, tenantId and hierarchyType must not contain the '|' character, which is reserved as the ancestral materialized-path delimiter."; } diff --git a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java index 353c2e1c2d1..e44f1639edd 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java +++ b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java @@ -1,15 +1,31 @@ package digit.repository; +import digit.web.models.BoundaryRelation; import digit.web.models.BoundaryRelationshipDTO; import digit.web.models.BoundaryRelationshipRequest; import digit.web.models.BoundaryRelationshipRequestDTO; import digit.web.models.BoundaryRelationshipSearchCriteria; +import org.egov.common.contract.request.RequestInfo; import java.util.List; public interface BoundaryRelationshipRepository { public void create(BoundaryRelationshipRequest boundaryRelationshipRequest); + /** + * Persists the given validated and enriched boundary relationships through egov-persister (no direct + * DB write): each record is published, one message per record, to the same save-boundary-relationship + * topic the single {@link #create} uses. The publish is blocking (it returns once the broker has + * accepted each record); egov-persister then writes each via an idempotent + * INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING, so redelivery is a safe no-op and + * one un-insertable record never fails the others. Optionally listing that topic in the persister's + * persister.batch.topics lets it aggregate a poll into one multi-row insert for throughput. + * + * @param boundaryRelationships validated and enriched relationships to persist + * @param requestInfo request info propagated onto each published message + */ + public void createBulk(List boundaryRelationships, RequestInfo requestInfo); + public void update(BoundaryRelationshipRequestDTO boundaryRelationshipRequest); public List search(BoundaryRelationshipSearchCriteria boundaryRelationshipSearchCriteria); diff --git a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java index bd78c589827..b8128dfb930 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java +++ b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java @@ -6,9 +6,11 @@ import digit.repository.querybuilder.BoundaryRelationshipQueryBuilder; import digit.repository.rowmapper.BoundaryRelationshipRowMapper; import digit.web.models.*; +import org.egov.common.contract.request.RequestInfo; import org.springframework.beans.BeanUtils; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; @@ -49,6 +51,41 @@ public void create(BoundaryRelationshipRequest boundaryRelationshipRequest) { producer.push(applicationProperties.getCreateBoundaryRelationshipTopic(), boundaryRelationshipRequestDTO); } + /** + * Persists the given validated and enriched boundary relationships through egov-persister rather than + * a direct JDBC write. Each relationship is published as its OWN message to the SAME topic the single + * create uses ({@code save-boundary-relationship}) via {@link #create}, so both paths write identical + * rows through the identical, idempotent + * {@code INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING} mapping. + * + *

Reusing that topic (instead of a dedicated {@code -batch} topic) is deliberate for + * deployment-safety: {@code save-boundary-relationship} is always consumed by the persister's normal + * listener, so bulk creation works on any persister deployment with no extra configuration. Adding + * {@code save-boundary-relationship} to the persister's {@code persister.batch.topics} (with + * {@code persister.bulk.enabled=true}) is a pure, optional throughput optimization: its batch listener + * then aggregates a poll into one multi-row insert. A dedicated {@code -batch} topic, by contrast, is + * dropped by the normal listener and would be silently orphaned if batch mode were not enabled.

+ * + *

One message per record (rather than one message carrying the whole batch) preserves per-record + * isolation on either listener: a single un-insertable record fails/dead-letters on its own without + * affecting the rest. Because the insert is idempotent, at-least-once redelivery is a safe no-op.

+ * + * @param boundaryRelationships validated and enriched relationships to persist + * @param requestInfo request info propagated onto each published message + */ + @Override + public void createBulk(List boundaryRelationships, RequestInfo requestInfo) { + if (CollectionUtils.isEmpty(boundaryRelationships)) + return; + + for (BoundaryRelation boundaryRelationship : boundaryRelationships) { + create(BoundaryRelationshipRequest.builder() + .requestInfo(requestInfo) + .boundaryRelationship(boundaryRelationship) + .build()); + } + } + /** * This method implements boundary relationship interface's update method. In this implementation * it pushes the request to kafka for persister to pick it up and perform update. diff --git a/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryEntityQueryBuilder.java b/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryEntityQueryBuilder.java index f04eaf9a779..6cd43b6f996 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryEntityQueryBuilder.java +++ b/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryEntityQueryBuilder.java @@ -51,8 +51,12 @@ private String buildQuery(BoundarySearchCriteria boundarySearchCriteria , List PostgreSQL "No value + // specified for parameter". Set codes = new HashSet<>(boundarySearchCriteria.getCodes()); + builder.append(" boundary.code IN ( ").append(QueryUtil.createQuery(codes.size())).append(" )"); QueryUtil.addToPreparedStatement(preparedStmtList , codes); } return builder.toString(); diff --git a/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryRelationshipQueryBuilder.java b/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryRelationshipQueryBuilder.java index 2bf04106ecc..ea854c96efe 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryRelationshipQueryBuilder.java +++ b/core-services/boundary-service/src/main/java/digit/repository/querybuilder/BoundaryRelationshipQueryBuilder.java @@ -7,6 +7,7 @@ import org.springframework.util.ObjectUtils; import java.util.HashSet; import java.util.List; +import java.util.Set; @Component public class BoundaryRelationshipQueryBuilder { @@ -52,8 +53,11 @@ private String buildQuery(BoundaryRelationshipSearchCriteria boundaryRelationshi if (!CollectionUtils.isEmpty(boundaryRelationshipSearchCriteria.getCodes())) { QueryUtil.addClauseIfRequired(builder, preparedStmtList); - builder.append(" code IN ( ").append(QueryUtil.createQuery(boundaryRelationshipSearchCriteria.getCodes().size())).append(" )"); - QueryUtil.addToPreparedStatement(preparedStmtList, new HashSet<>(boundaryRelationshipSearchCriteria.getCodes())); + // Deduplicate so the placeholder count matches the bind values (duplicate codes would + // otherwise leave a "?" without a value -> "No value specified for parameter"). + Set codeSet = new HashSet<>(boundaryRelationshipSearchCriteria.getCodes()); + builder.append(" code IN ( ").append(QueryUtil.createQuery(codeSet.size())).append(" )"); + QueryUtil.addToPreparedStatement(preparedStmtList, codeSet); } } @@ -64,9 +68,11 @@ private String buildQuery(BoundaryRelationshipSearchCriteria boundaryRelationshi if(!CollectionUtils.isEmpty(boundaryRelationshipSearchCriteria.getCurrentBoundaryCodes())) { QueryUtil.addClauseIfRequired(builder, preparedStmtList); - builder.append(" ARRAY [ ").append(QueryUtil.createQuery(boundaryRelationshipSearchCriteria.getCurrentBoundaryCodes().size())).append(" ]").append("::text[] "); + // Deduplicate so the placeholder count matches the bind values (see note above). + Set currentCodeSet = new HashSet<>(boundaryRelationshipSearchCriteria.getCurrentBoundaryCodes()); + builder.append(" ARRAY [ ").append(QueryUtil.createQuery(currentCodeSet.size())).append(" ]").append("::text[] "); builder.append(" && string_to_array(ancestralmaterializedpath, '|') "); - QueryUtil.addToPreparedStatement(preparedStmtList, new HashSet<>(boundaryRelationshipSearchCriteria.getCurrentBoundaryCodes())); + QueryUtil.addToPreparedStatement(preparedStmtList, currentCodeSet); } return builder.toString(); diff --git a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java index 2a2b81459d3..666ca629647 100644 --- a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java +++ b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java @@ -1,12 +1,19 @@ package digit.service; +import digit.config.ApplicationProperties; +import digit.errors.ErrorCodes; import digit.repository.BoundaryRelationshipRepository; import digit.service.enrichment.BoundaryRelationshipEnricher; import digit.service.validator.BoundaryRelationshipValidator; import digit.util.HierarchyUtil; import digit.web.models.*; +import lombok.extern.slf4j.Slf4j; import org.egov.common.contract.request.RequestInfo; import org.egov.common.utils.ResponseInfoUtil; +import org.egov.tracer.model.CustomException; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.RecoverableDataAccessException; +import org.springframework.dao.TransientDataAccessException; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -14,6 +21,7 @@ import java.util.stream.Collectors; @Service +@Slf4j public class BoundaryRelationshipService { private BoundaryRelationshipValidator boundaryRelationshipValidator; @@ -24,12 +32,16 @@ public class BoundaryRelationshipService { private HierarchyUtil hierarchyUtil; + private ApplicationProperties applicationProperties; + public BoundaryRelationshipService(BoundaryRelationshipValidator boundaryRelationshipValidator, BoundaryRelationshipEnricher boundaryRelationshipEnricher, - BoundaryRelationshipRepository boundaryRelationshipRepository, HierarchyUtil hierarchyUtil) { + BoundaryRelationshipRepository boundaryRelationshipRepository, HierarchyUtil hierarchyUtil, + ApplicationProperties applicationProperties) { this.boundaryRelationshipValidator = boundaryRelationshipValidator; this.boundaryRelationshipEnricher = boundaryRelationshipEnricher; this.boundaryRelationshipRepository = boundaryRelationshipRepository; this.hierarchyUtil = hierarchyUtil; + this.applicationProperties = applicationProperties; } /** @@ -56,6 +68,158 @@ public BoundaryRelationshipResponse createBoundaryRelationship(BoundaryRelations } + /** + * Request handler for processing bulk boundary relationship create requests. + * + *

Each record is validated and enriched independently (reusing the same business rules as + * the single create), so a business failure on one record does not abort the others. The records + * that pass validation are handed to egov-persister for the actual DB write (see + * {@link BoundaryRelationshipRepository#createBulk}); this service performs no direct DB insert. + * The response reports the outcome of every record: the relationships accepted for persistence + * and, separately, the ones that failed validation/enrichment with a reason for each.

+ * + * @param body bulk create request + * @return per-record success/failure response + */ + public BulkBoundaryRelationshipResponse createBulkBoundaryRelationship(BulkBoundaryRelationshipRequest body) { + + RequestInfo requestInfo = body.getRequestInfo(); + + // Guard the request envelope explicitly: bean validation (@Valid/@Size/@NotNull) is NOT active in + // this service, and this endpoint is the sole entry for bulk creation, so the size/shape limits + // must be enforced here (they are also required for a bounded synchronous validation+enrich pass). + if (requestInfo == null || requestInfo.getUserInfo() == null) { + throw new CustomException(ErrorCodes.BULK_REQUEST_INFO_MISSING_CODE, ErrorCodes.BULK_REQUEST_INFO_MISSING_MSG); + } + if (CollectionUtils.isEmpty(body.getBoundaryRelationships())) { + throw new CustomException(ErrorCodes.BULK_REQUEST_EMPTY_CODE, ErrorCodes.BULK_REQUEST_EMPTY_MSG); + } + if (body.getBoundaryRelationships().size() > applicationProperties.getBulkCreateMaxSize()) { + throw new CustomException(ErrorCodes.BULK_REQUEST_SIZE_EXCEEDED_CODE, + ErrorCodes.BULK_REQUEST_SIZE_EXCEEDED_MSG + applicationProperties.getBulkCreateMaxSize()); + } + + List validatedRelationships = new ArrayList<>(); + List failedRelationships = new ArrayList<>(); + + // Track records seen in this batch to reject intra-batch duplicates. The per-record + // duplicate check queries the database, which cannot see other records in the same request; + // without this, two identical records would both validate and be published for the same + // natural key, giving the caller no clear per-record duplicate signal and asking the persister + // to insert the same (tenantId, code, hierarchyType) twice in one batch. + Set seenKeysInBatch = new HashSet<>(); + + for (BoundaryRelation boundaryRelationship : body.getBoundaryRelationships()) { + try { + String key = buildUniquenessKey(boundaryRelationship); + if (seenKeysInBatch.contains(key)) { + throw new CustomException(ErrorCodes.DUPLICATE_RECORD_IN_REQUEST_CODE, ErrorCodes.DUPLICATE_RECORD_IN_REQUEST_MSG); + } + + // Reuse the existing single-record validation and enrichment so bulk and single + // create share identical business rules. + BoundaryRelationshipRequest singleRequest = BoundaryRelationshipRequest.builder() + .requestInfo(requestInfo) + .boundaryRelationship(boundaryRelationship) + .build(); + + String ancestralMaterializedPath = boundaryRelationshipValidator.validateBoundaryRelationshipCreateRequest(singleRequest); + boundaryRelationshipEnricher.enrichBoundaryRelationshipCreateRequest(singleRequest, ancestralMaterializedPath); + + seenKeysInBatch.add(key); + validatedRelationships.add(singleRequest.getBoundaryRelationship()); + } catch (CustomException e) { + failedRelationships.add(FailedBoundaryRelationship.builder() + .boundaryRelationship(boundaryRelationship) + .errorCode(e.getCode()) + .errorMessage(e.getMessage()) + .build()); + } catch (TransientDataAccessException | RecoverableDataAccessException | DataAccessResourceFailureException e) { + // The validation pass issues several JDBC reads per record. A DB blip or Hikari pool + // exhaustion surfaces as CannotGetJdbcConnectionException, which extends + // DataAccessResourceFailureException (a NON-transient marker in Spring's hierarchy) — so it + // must be caught explicitly here alongside the transient/recoverable markers. Classifying + // it transient lets the caller retry (the reads are stateless and the persister insert is + // idempotent) instead of the whole campaign aborting on a momentary DB saturation. + failedRelationships.add(FailedBoundaryRelationship.builder() + .boundaryRelationship(boundaryRelationship) + .errorCode(ErrorCodes.BULK_RELATIONSHIP_PERSIST_TRANSIENT_CODE) + .errorMessage(withCause(ErrorCodes.BULK_RELATIONSHIP_PERSIST_TRANSIENT_MSG, e)) + .build()); + } catch (Exception e) { + // A non-business RuntimeException (e.g. null userInfo during enrichment, a malformed + // hierarchy definition) must not unwind the whole job; record it as a per-record + // failure so the remaining records still proceed. + failedRelationships.add(FailedBoundaryRelationship.builder() + .boundaryRelationship(boundaryRelationship) + .errorCode(ErrorCodes.BULK_RELATIONSHIP_VALIDATION_ERROR_CODE) + .errorMessage(withCause(ErrorCodes.BULK_RELATIONSHIP_VALIDATION_ERROR_MSG, e)) + .build()); + } + } + + // Persistence is delegated to egov-persister (no direct DB write from this service): each + // validated + enriched record is published, one message per record, to the save-boundary-relationship + // topic, which egov-persister writes via an idempotent INSERT ... ON CONFLICT DO NOTHING. Publishing + // is blocking (CustomKafkaTemplate.send().get()); a publish failure (broker unreachable within + // max.block.ms, serialization) is reported transient so the caller retries the affected records — + // re-publishing is a safe no-op because the insert is idempotent. One message per record preserves + // per-record isolation on the persister side regardless of whether it runs the normal or the + // (optional) batch listener. + List successfulRelationships = validatedRelationships; + if (!CollectionUtils.isEmpty(validatedRelationships)) { + try { + boundaryRelationshipRepository.createBulk(validatedRelationships, requestInfo); + } catch (Exception e) { + successfulRelationships = new ArrayList<>(); + markPersistFailure(validatedRelationships, failedRelationships, ErrorCodes.BULK_RELATIONSHIP_PERSIST_TRANSIENT_CODE, withCause(ErrorCodes.BULK_RELATIONSHIP_PERSIST_TRANSIENT_MSG, e)); + } + } + + return BulkBoundaryRelationshipResponse.builder() + .responseInfo(ResponseInfoUtil.createResponseInfoFromRequestInfo(requestInfo, Boolean.TRUE)) + .successfulBoundaryRelationships(successfulRelationships) + .failedBoundaryRelationships(failedRelationships) + .build(); + } + + /** + * Builds the natural-key string (tenantId, hierarchyType, code) used to detect duplicate + * relationships within a single bulk request. Mirrors the table's primary key. + */ + private String buildUniquenessKey(BoundaryRelation boundaryRelationship) { + return boundaryRelationship.getTenantId() + "|" + + boundaryRelationship.getHierarchyType() + "|" + + boundaryRelationship.getCode(); + } + + /** + * Records the whole validated set as failed with the given (transient) persistence error code + * when publishing the validated set to the batch-persister topic failed. + */ + private void markPersistFailure(List relationships, List failures, String errorCode, String errorMessage) { + for (BoundaryRelation relationship : relationships) { + failures.add(FailedBoundaryRelationship.builder() + .boundaryRelationship(relationship) + .errorCode(errorCode) + .errorMessage(errorMessage) + .build()); + } + } + + /** + * Returns the stable, caller-facing message for a failure and logs the underlying cause separately. + * The concrete exception text is deliberately NOT folded into the returned message: that string is + * surfaced in the HTTP response payload and republished to the Kafka error topic, so raw + * SQL/driver/internal details must not leak into it. + */ + private String withCause(String message, Throwable cause) { + if (cause != null) { + log.warn("{} (cause: {})", message, cause.toString(), cause); + } + return message; + } + /** * Request handler for processing boundary relationship search requests. * @param boundaryRelationshipSearchCriteria @@ -122,10 +286,20 @@ private List getParentBoundaries(List allAncestorCodes = boundaries.stream() - .map(dto -> dto.getAncestralMaterializedPath().split("\\|")) - .flatMap(Arrays::stream) + .map(BoundaryRelationshipDTO::getAncestralMaterializedPath) + .filter(path -> path != null && !path.isEmpty()) + .flatMap(path -> Arrays.stream(path.split("\\|"))) .collect(Collectors.toSet()); + // Root nodes have an empty materialized path, so they contribute no ancestor codes. If NONE of + // the matched boundaries has an ancestor, there are no parents to fetch — return early. Passing + // an empty codes list to search would otherwise cause the query builder to drop the + // `code IN (...)` predicate and scan the entire tenant/hierarchy (a full-table read on a public + // endpoint that would also return the whole tree as bogus parents). + if (allAncestorCodes.isEmpty()) { + return parentBoundaries; + } + parentBoundaries = boundaryRelationshipRepository.search(BoundaryRelationshipSearchCriteria.builder() .tenantId(boundaryRelationshipSearchCriteria.getTenantId()) .hierarchyType(boundaryRelationshipSearchCriteria.getHierarchyType()) diff --git a/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryRelationshipValidator.java b/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryRelationshipValidator.java index b5c3a15e888..d2125fe41c6 100644 --- a/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryRelationshipValidator.java +++ b/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryRelationshipValidator.java @@ -1,5 +1,6 @@ package digit.service.validator; +import digit.errors.ErrorCodes; import digit.repository.BoundaryRelationshipRepository; import digit.repository.BoundaryRepository; import digit.util.HierarchyUtil; @@ -35,6 +36,11 @@ public BoundaryRelationshipValidator(BoundaryRelationshipRepository boundaryRela * @return */ public String validateBoundaryRelationshipCreateRequest(BoundaryRelationshipRequest body) { + // Reject the reserved materialized-path delimiter in the key fields: '|' joins the ancestral + // path (and the bulk dedup key), so a '|' inside code/tenantId/hierarchyType would corrupt + // subtree search and produce false dedup collisions. Applies to both single and bulk create. + validateKeyFieldsHaveNoDelimiter(body); + // Check if boundary entity exists validateIfBoundaryEntityExists(body); @@ -120,6 +126,24 @@ private BoundaryRelationshipDTO validateExistence(BoundaryRelationshipRequest bo return boundaryRelationshipDTOList.get(0); } + /** + * Rejects the reserved '|' materialized-path delimiter in the natural-key fields (code, tenantId, + * hierarchyType). A '|' in any of these would split into bogus tokens in the ancestral path and the + * GIN-overlap subtree search, and collide in the bulk dedup key. + */ + private void validateKeyFieldsHaveNoDelimiter(BoundaryRelationshipRequest body) { + BoundaryRelation boundaryRelationship = body.getBoundaryRelationship(); + if (containsPathDelimiter(boundaryRelationship.getCode()) + || containsPathDelimiter(boundaryRelationship.getTenantId()) + || containsPathDelimiter(boundaryRelationship.getHierarchyType())) { + throw new CustomException(ErrorCodes.INVALID_BOUNDARY_CODE_CODE, ErrorCodes.INVALID_BOUNDARY_CODE_MSG); + } + } + + private boolean containsPathDelimiter(String value) { + return value != null && value.contains("|"); + } + /** * This method checks if the given boundary relationship already exists. * @param body @@ -187,7 +211,12 @@ private void validateRelationshipForProperHierarchy(BoundaryRelationshipRequest throw new CustomException("HIERARCHY_ERROR", "Boundary relationship without defined parent should have root boundary hierarchy type."); } } else{ - if(!body.getBoundaryRelationship().getBoundaryType().equals(hierarchyOrder.get(hierarchyOrder.indexOf(parentBoundaryType) + 1))) { + // Guard the index lookup: if the parent's boundary type is not in the hierarchy or is the + // leaf level, indexOf(...)+1 would otherwise throw IndexOutOfBounds (a plain RuntimeException + // that would escape the per-record bulk handling). Surface it as a recordable CustomException. + int parentIndex = hierarchyOrder.indexOf(parentBoundaryType); + if(parentIndex < 0 || parentIndex + 1 >= hierarchyOrder.size() + || !body.getBoundaryRelationship().getBoundaryType().equals(hierarchyOrder.get(parentIndex + 1))) { throw new CustomException("HIERARCHY_ERROR", "Hierarchy of child should be the direct descendant of parent's boundary hierarchy type."); } } diff --git a/core-services/boundary-service/src/main/java/digit/util/HierarchyUtil.java b/core-services/boundary-service/src/main/java/digit/util/HierarchyUtil.java index fec2ed5214d..bfa5bc72ddd 100644 --- a/core-services/boundary-service/src/main/java/digit/util/HierarchyUtil.java +++ b/core-services/boundary-service/src/main/java/digit/util/HierarchyUtil.java @@ -58,7 +58,7 @@ public List getHierarchyOrder(String tenantId, String hierarchyType) { .stream() .filter(hierarchyNode -> ObjectUtils.isEmpty(hierarchyNode.getParentBoundaryType())) .findFirst() - .get() + .orElseThrow(() -> new CustomException("HIERARCHY_DEFINITION_INVALID_ERR", "Hierarchy definition has no root boundary type.")) .getBoundaryType(); hierarchyOrder.add(rootHierarchyNode); diff --git a/core-services/boundary-service/src/main/java/digit/web/controllers/BoundaryRelationshipController.java b/core-services/boundary-service/src/main/java/digit/web/controllers/BoundaryRelationshipController.java index 836ba8f4007..17025b8c464 100644 --- a/core-services/boundary-service/src/main/java/digit/web/controllers/BoundaryRelationshipController.java +++ b/core-services/boundary-service/src/main/java/digit/web/controllers/BoundaryRelationshipController.java @@ -33,6 +33,24 @@ public ResponseEntity create(@Valid @RequestBody B return new ResponseEntity<>(boundaryRelationshipResponse, HttpStatus.ACCEPTED); } + /** + * Request handler for serving bulk boundary relationship create requests. + * + *

Validates and enriches each relationship synchronously and returns a per-record outcome + * (which records were accepted for persistence, and which failed validation with a reason). The + * actual DB write is performed asynchronously by egov-persister — the valid records are published + * to the idempotent batch-persister topic — so a successful entry means "accepted and will be + * persisted", not "already committed".

+ * + * @param body bulk create request + * @return per-record success/failure response + */ + @RequestMapping(value = "/bulk/_create", method = RequestMethod.POST) + public ResponseEntity bulkCreate(@Valid @RequestBody BulkBoundaryRelationshipRequest body) { + BulkBoundaryRelationshipResponse bulkBoundaryRelationshipResponse = boundaryRelationshipService.createBulkBoundaryRelationship(body); + return new ResponseEntity<>(bulkBoundaryRelationshipResponse, HttpStatus.OK); + } + /** * Request handler for serving boundary relationships search request. * @param boundaryRelationshipSearchCriteria diff --git a/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequest.java b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequest.java new file mode 100644 index 00000000000..2b2c9a33d05 --- /dev/null +++ b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequest.java @@ -0,0 +1,43 @@ +package digit.web.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.egov.common.contract.request.RequestInfo; +import org.springframework.validation.annotation.Validated; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import lombok.Data; +import lombok.Builder; + +import java.util.List; + +/** + * Request payload for creating boundary relationships in bulk. + * + *

Each record in {@code boundaryRelationships} is expected to be a sibling at the same + * level whose parent has already been persisted (callers batch children under a single, + * already-persisted parent). The batch is capped to keep a single request bounded; callers + * are responsible for chunking larger levels into multiple requests.

+ */ +@Validated +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class BulkBoundaryRelationshipRequest { + + @JsonProperty("RequestInfo") + @Valid + private RequestInfo requestInfo = null; + + @JsonProperty("BoundaryRelationships") + @NotNull + @Valid + @Size(min = 1, max = 100) + private List boundaryRelationships = null; + +} diff --git a/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipResponse.java b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipResponse.java new file mode 100644 index 00000000000..fa0a1063fd1 --- /dev/null +++ b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipResponse.java @@ -0,0 +1,44 @@ +package digit.web.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.egov.common.contract.response.ResponseInfo; +import org.springframework.validation.annotation.Validated; + +import jakarta.validation.Valid; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import lombok.Data; +import lombok.Builder; + +import java.util.List; + +/** + * Response for a bulk boundary relationship create request. + * + *

Reports the outcome of each record explicitly: {@code successfulBoundaryRelationships} + * holds the records that passed validation/enrichment and were accepted for persistence (handed to + * egov-persister, which writes them idempotently and asynchronously), while + * {@code failedBoundaryRelationships} holds the records that could not be accepted, each with a + * reason. The whole request does not fail because of individual record failures.

+ */ +@Validated +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class BulkBoundaryRelationshipResponse { + + @JsonProperty("ResponseInfo") + @Valid + private ResponseInfo responseInfo = null; + + @JsonProperty("successfulBoundaryRelationships") + @Valid + private List successfulBoundaryRelationships = null; + + @JsonProperty("failedBoundaryRelationships") + @Valid + private List failedBoundaryRelationships = null; + +} diff --git a/core-services/boundary-service/src/main/java/digit/web/models/FailedBoundaryRelationship.java b/core-services/boundary-service/src/main/java/digit/web/models/FailedBoundaryRelationship.java new file mode 100644 index 00000000000..356e59f182f --- /dev/null +++ b/core-services/boundary-service/src/main/java/digit/web/models/FailedBoundaryRelationship.java @@ -0,0 +1,34 @@ +package digit.web.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.springframework.validation.annotation.Validated; + +import jakarta.validation.Valid; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import lombok.Data; +import lombok.Builder; + +/** + * A single boundary relationship that could not be created during a bulk request, paired with + * the reason it failed so the caller can act on it (for example, by idempotent resubmission). + */ +@Validated +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class FailedBoundaryRelationship { + + @JsonProperty("boundaryRelationship") + @Valid + private BoundaryRelation boundaryRelationship = null; + + @JsonProperty("errorCode") + private String errorCode = null; + + @JsonProperty("errorMessage") + private String errorMessage = null; + +} diff --git a/core-services/boundary-service/src/main/resources/application.properties b/core-services/boundary-service/src/main/resources/application.properties index 11517b8da9f..010a62b7291 100644 --- a/core-services/boundary-service/src/main/resources/application.properties +++ b/core-services/boundary-service/src/main/resources/application.properties @@ -31,15 +31,16 @@ spring.kafka.listener.missing-topics-fatal=false spring.kafka.consumer.properties.spring.json.use.type.headers=false # KAFKA CONSUMER CONFIGURATIONS -kafka.consumer.config.auto_commit=true -kafka.consumer.config.auto_commit_interval=100 -kafka.consumer.config.session_timeout=15000 -kafka.consumer.config.auto_offset_reset=earliest +spring.kafka.consumer.enable-auto-commit=false # KAFKA PRODUCER CONFIGURATIONS -kafka.producer.config.retries_config=0 -kafka.producer.config.batch_size_config=16384 -kafka.producer.config.linger_ms_config=1 -kafka.producer.config.buffer_memory_config=33554432 +spring.kafka.producer.retries=3 +spring.kafka.producer.batch-size=16384 +spring.kafka.producer.properties.linger.ms=1 +spring.kafka.producer.buffer-memory=33554432 +# Bound how long a publish blocks when the broker is unreachable / the buffer is full. The bulk/single +# create endpoints publish synchronously on the request thread, so a broker outage must fail fast (surfaced +# as a transient error the caller retries) rather than tie up request threads for the 60s Kafka default. +spring.kafka.producer.properties.max.block.ms=15000 #Localization config egov.localization.host=https://dev.digit.org @@ -87,11 +88,18 @@ kafka.topics.create.boundary = create-boundary-entity kafka.topics.update.boundary = update-boundary-entity kafka.topics.create.boundary.hierarchy = save-boundary-hierarchy-definition kafka.topics.update.boundary.hierarchy = update-boundary-hierarchy-definition +# Both single AND bulk relationship creates publish (one message per record) to this topic; the /bulk/_create +# endpoint validates + enriches each record synchronously first, so nothing un-validated is ever placed on +# Kafka. egov-persister writes each via an idempotent INSERT ... ON CONFLICT DO NOTHING (see boundary-persister.yml). +# Adding this topic to the persister's `persister.batch.topics` (with persister.bulk.enabled=true) is an +# optional throughput optimization that lets the persister aggregate a poll into one multi-row insert. kafka.topics.create.boundary.relationship = save-boundary-relationship kafka.topics.update.boundary.relationship = update-boundary-relationship boundary.default.offset=0 boundary.default.limit=50 boundary.max.default.limit=300 +# Max records accepted by POST /boundary-relationships/bulk/_create (enforced in-service; keep >= caller chunk size). +boundary.bulk.max.size=100 otel.traces.exporter=otlp otel.service.name=boundary-service diff --git a/core-services/boundary-service/src/main/resources/boundary-persister.yml b/core-services/boundary-service/src/main/resources/boundary-persister.yml index b40ae9b17bb..57522b2db73 100644 --- a/core-services/boundary-service/src/main/resources/boundary-persister.yml +++ b/core-services/boundary-service/src/main/resources/boundary-persister.yml @@ -6,7 +6,7 @@ serviceMaps: fromTopic: create-boundary-entity isTransaction: true queryMaps: - - query: INSERT INTO boundary (id, tenantId, code, geometry, additionalDetails, createdBy, lastModifiedBy, createdTime, lastModifiedTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO boundary (id, tenantId, code, geometry, additionalDetails, createdBy, lastModifiedBy, createdTime, lastModifiedTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (code, tenantId) DO NOTHING; basePath: $.Boundary.* jsonMaps: - jsonPath: $.Boundary.*.id @@ -54,7 +54,7 @@ serviceMaps: fromTopic: save-boundary-hierarchy-definition isTransaction: true queryMaps: - - query: INSERT INTO boundary_hierarchy (id, tenantId, hierarchyType, boundaryHierarchy, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO boundary_hierarchy (id, tenantId, hierarchyType, boundaryHierarchy, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantId, hierarchyType) DO NOTHING; basePath: $.BoundaryHierarchy jsonMaps: - jsonPath: $.BoundaryHierarchy.id @@ -79,7 +79,7 @@ serviceMaps: fromTopic: save-boundary-relationship isTransaction: true queryMaps: - - query: INSERT INTO boundary_relationship (id, tenantId, code, hierarchyType, boundaryType, parent, ancestralMaterializedPath, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO boundary_relationship (id, tenantId, code, hierarchyType, boundaryType, parent, ancestralMaterializedPath, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING; basePath: $.BoundaryRelationship jsonMaps: - jsonPath: $.BoundaryRelationship.id diff --git a/core-services/boundary-service/src/main/resources/db/migration/main/V20260616120000__boundary_relationship_search_indexes.sql b/core-services/boundary-service/src/main/resources/db/migration/main/V20260616120000__boundary_relationship_search_indexes.sql new file mode 100644 index 00000000000..83cf9d06a95 --- /dev/null +++ b/core-services/boundary-service/src/main/resources/db/migration/main/V20260616120000__boundary_relationship_search_indexes.sql @@ -0,0 +1,34 @@ +-- Indexes to make boundary-relationship subtree search index-accelerated instead of +-- sequentially scanning the whole boundary_relationship table on every search. +-- +-- Why: BoundaryRelationshipService.getBoundaryRelationships() resolves a subtree +-- (includeChildren=true) with the array-overlap predicate built in BoundaryRelationshipQueryBuilder: +-- ... AND ARRAY[...]::text[] && string_to_array(ancestralmaterializedpath, '|') +-- That predicate is non-sargable, and the table had only the PK + unique(id), so every search did +-- a full Seq Scan, holding a DB connection for its duration. Under campaign-scale concurrent search +-- load this exhausts the Hikari pool (CannotGetJdbcConnectionException). +-- +-- Validated on a 50k-row hierarchy (PostgreSQL 16): the GIN index turns the plan from a full +-- Seq Scan (~20 ms, scans all rows) into a Bitmap Index Scan (~2 ms). Measured with pgbench +-- (randomized subtree anchor, 1-48 concurrent clients on a 12-core box): ~10x higher search +-- throughput and ~10x lower latency under sustained concurrent load (at 48 clients, mean latency +-- ~498 ms -> ~51 ms; ~96 -> ~950 tps). The index removes only the scan cost; both plans still sort +-- and fetch the matched subtree rows, so the end-to-end gain is ~10x rather than the scan-node ratio. +-- +-- 1) GIN index on the materialized-path token array -> serves the && overlap above. +-- 2) (tenantid, parent) -> serves the parent = ? and parent IS NULL (root) search branches. +-- +-- NOTE: these are plain (non-CONCURRENTLY) CREATE INDEX statements on purpose. CREATE INDEX +-- CONCURRENTLY must NOT be used inside a Flyway migration here: even with +-- executeInTransaction=false, Flyway keeps its schema-history connection open in a transaction, +-- and CONCURRENTLY blocks waiting for that concurrent transaction to finish -> the migration hangs +-- indefinitely (verified against Flyway with this exact migration). A plain CREATE INDEX builds in +-- well under a second on this table under a brief, read-allowing ShareLock, which is acceptable for +-- this low-write table. For a very large table or a strict zero-write-downtime requirement, build +-- these indexes CONCURRENTLY out-of-band (manual DBA step) instead of via this migration. + +CREATE INDEX IF NOT EXISTS idx_boundary_relationship_amp_gin + ON boundary_relationship USING GIN (string_to_array(ancestralmaterializedpath, '|')); + +CREATE INDEX IF NOT EXISTS idx_boundary_relationship_tenant_parent + ON boundary_relationship (tenantid, parent); diff --git a/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java b/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java index b9a5ba85e98..72ff50e0cd0 100644 --- a/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java +++ b/core-services/egov-persister/src/main/java/org/egov/EgovPersistApplication.java @@ -15,9 +15,9 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.*; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; +import org.springframework.scheduling.annotation.EnableScheduling; import jakarta.annotation.PostConstruct; import java.io.File; @@ -25,8 +25,8 @@ import java.io.InputStream; import java.util.*; -@EnableAspectJAutoProxy @SpringBootApplication +@EnableScheduling @Slf4j public class EgovPersistApplication { diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java deleted file mode 100644 index 7d62a02c0d5..00000000000 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/aspectj/TransactionInterceptorAspect.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.egov.infra.persist.aspectj; - -import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.springframework.kafka.listener.ListenerExecutionFailedException; -import org.springframework.stereotype.Component; - -@Slf4j -@Aspect -@Component -public class TransactionInterceptorAspect { - - @Around("@annotation(org.springframework.transaction.annotation.Transactional)") - public Object aroundTransactional(ProceedingJoinPoint joinPoint) throws Throwable { - try { - return joinPoint.proceed(); - } catch (Exception e) { - throw new ListenerExecutionFailedException(e.getMessage(), e); - } - } -} diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java new file mode 100644 index 00000000000..a9fe95fdc9b --- /dev/null +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java @@ -0,0 +1,71 @@ +package org.egov.infra.persist.consumer; + +import java.sql.SQLException; + +/** + * Classifies a persistence failure so the consumer can route it correctly. + * + *

Classification is by PostgreSQL SQLSTATE (and exception cause chain) rather than Spring's + * marker interfaces, because connection-acquisition failures (e.g. CannotGetJdbcConnectionException) + * are filed on Spring's NON-transient branch and would otherwise be misclassified as permanent.

+ * + *
    + *
  • BENIGN — unique_violation (23505): the row already exists; under at-least-once redelivery + * or DLQ replay this is an idempotent success, not a failure.
    • + *
  • TRANSIENT — connection / serialization / deadlock / resource failures: retrying may succeed.
    • + *
  • PERMANENT — constraint / data / grammar errors: retrying will always fail the same way.
    • + *
+ */ +public final class DbExceptionClassifier { + + public enum Kind { BENIGN, TRANSIENT, PERMANENT } + + private DbExceptionClassifier() {} + + public static Kind classify(Throwable t) { + String sqlState = sqlState(t); + if ("23505".equals(sqlState)) { + return Kind.BENIGN; // unique_violation + } + if (sqlState != null && ( + sqlState.startsWith("08") // connection exception + || sqlState.startsWith("57") // operator intervention (e.g. admin shutdown, query cancel) + || "40001".equals(sqlState) // serialization_failure + || "40P01".equals(sqlState) // deadlock_detected + || "53300".equals(sqlState) // too_many_connections + || "55P03".equals(sqlState))) { // lock_not_available + return Kind.TRANSIENT; + } + // Connection-acquisition / transient failures may not carry a SQLState on the chain. + for (Throwable c = t; c != null; c = c.getCause()) { + String n = c.getClass().getName(); + if (n.contains("CannotGetJdbcConnection") + || n.contains("DataAccessResourceFailure") + || n.contains("QueryTimeout") + || n.contains("TransientDataAccess") + || n.contains("ConcurrencyFailure") + || n.contains("CannotAcquireLock") + || n.contains("DeadlockLoserDataAccess") + || n.contains("RecoverableDataAccess")) { + return Kind.TRANSIENT; + } + } + return Kind.PERMANENT; + } + + public static boolean isBenign(Throwable t) { + return classify(t) == Kind.BENIGN; + } + + private static String sqlState(Throwable t) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (c instanceof SQLException) { + String s = ((SQLException) c).getSQLState(); + if (s != null) { + return s; + } + } + } + return null; + } +} diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java new file mode 100644 index 00000000000..a6f4692e8f2 --- /dev/null +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java @@ -0,0 +1,58 @@ +package org.egov.infra.persist.consumer; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Pauses the persister consumer while the datasource is unreachable and resumes it once the datasource + * recovers — the "pause-on-DB-health" backstop for transient failures. + * + *

Rationale: on a transient DB failure the listener rethrows and the container error handler retries + * the record in place (never parking a good record). Left alone that would spin retries against a dead + * DB. This monitor stops the consumer from pulling new work while the DB is down, so the pod waits + * quietly instead of hammering; when the DB is back it resumes and the un-committed records are + * re-delivered and persisted. Consumption is gated on real DB health, not a timer.

+ */ +@Component +@Slf4j +public class DbHealthMonitor { + + private final JdbcTemplate jdbcTemplate; + private final PersisterConsumerConfig consumerConfig; + + /** Tracks whether we have paused the consumer, so pause/resume fire only on a health transition. */ + private volatile boolean pausedForDb = false; + + @Autowired + public DbHealthMonitor(JdbcTemplate jdbcTemplate, PersisterConsumerConfig consumerConfig) { + this.jdbcTemplate = jdbcTemplate; + this.consumerConfig = consumerConfig; + } + + @Scheduled(fixedDelayString = "${persister.db-health.check-interval-ms:5000}") + public void checkDatasource() { + boolean healthy = isHealthy(); + if (!healthy && !pausedForDb) { + log.warn("Datasource health check FAILED - pausing persister consumer until it recovers"); + consumerConfig.pauseContainer(); + pausedForDb = true; + } else if (healthy && pausedForDb) { + log.info("Datasource health check RECOVERED - resuming persister consumer"); + consumerConfig.resumeContainer(); + pausedForDb = false; + } + } + + private boolean isHealthy() { + try { + jdbcTemplate.queryForObject("SELECT 1", Integer.class); + return true; + } catch (Exception e) { + log.debug("Datasource probe failed: {}", e.getMessage()); + return false; + } + } +} diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java index 24794892e57..e6b55c36bec 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchConsumerConfig.java @@ -7,7 +7,6 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.egov.infra.persist.web.contract.TopicMap; -import org.egov.tracer.KafkaConsumerErrorHandler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -19,6 +18,7 @@ import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.listener.BatchMessageListener; import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.DefaultErrorHandler; import org.springframework.kafka.listener.KafkaMessageListenerContainer; import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer; import org.springframework.kafka.support.serializer.JsonDeserializer; @@ -37,9 +37,6 @@ matchIfMissing = false) public class PersisterBatchConsumerConfig { - /*@Autowired - private StoppingErrorHandler stoppingErrorHandler;*/ - @Autowired private BatchMessageListener batchMessageListener; @@ -50,7 +47,7 @@ public class PersisterBatchConsumerConfig { private KafkaProperties kafkaProperties; @Autowired - private KafkaConsumerErrorHandler kafkaConsumerErrorHandler; + private DefaultErrorHandler persisterErrorHandler; @Value("${persister.batch.size}") private Integer batchSize; @@ -58,6 +55,15 @@ public class PersisterBatchConsumerConfig { @Value("${persister.batch.topics:}") private String batchTopicsConfig; + @Value("${persister.kafka.partition.assignment.strategy:org.apache.kafka.clients.consumer.CooperativeStickyAssignor,org.apache.kafka.clients.consumer.RangeAssignor}") + private String partitionAssignmentStrategy; + + @Value("${persister.kafka.group.instance.id:}") + private String groupInstanceId; + + @Value("${persister.kafka.session.timeout.ms:}") + private String sessionTimeoutMsOverride; + @Getter private Set batchTopics = new HashSet<>(); @@ -104,14 +110,18 @@ private void createBatchContainer() { properties.setAckMode(ContainerProperties.AckMode.BATCH); batchContainer = new KafkaMessageListenerContainer<>(createConsumerFactory(), properties); - batchContainer.setCommonErrorHandler(kafkaConsumerErrorHandler); + batchContainer.setCommonErrorHandler(persisterErrorHandler); batchContainer.setBeanName("batchContainer"); batchContainer.start(); log.info("Started batch container for {} topics: {}", batchTopics.size(), batchTopics); } catch (Exception e) { - log.error("Failed to create batch container", e); + // Fail loud (consistent with EgovPersistApplication.loadConfigs): a persister that silently + // starts with no batch consumer accrues unbounded Kafka lag with no signal - the exact RCA + // failure mode. Abort startup so the orchestrator restarts the pod instead of running blind. + log.error("Failed to create batch container - aborting startup", e); + throw new IllegalStateException("Failed to start batch container for topics: " + batchTopics, e); } } @@ -119,9 +129,17 @@ private ConsumerFactory createConsumerFactory() { Map props = kafkaProperties.buildConsumerProperties(); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); - props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000"); + props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, + StringUtils.hasText(sessionTimeoutMsOverride) ? sessionTimeoutMsOverride : "30000"); props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, batchSize); + // Cooperative rebalancing + optional static membership — same rationale and rolling-deploy + // safety as PersisterConsumerConfig; "-batch" suffix because both containers share group.id. + props.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partitionAssignmentStrategy); + if (StringUtils.hasText(groupInstanceId)) { + props.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, groupInstanceId + "-batch"); + } + JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class, false); ErrorHandlingDeserializer errorHandlingDeserializer = new ErrorHandlingDeserializer<>(jsonDeserializer); diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java index d27bfe0bd62..28b07056ce0 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java @@ -8,8 +8,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; -import org.egov.tracer.kafka.ErrorQueueProducer; -import org.egov.tracer.model.ErrorQueueContract; import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -34,15 +32,15 @@ public class PersisterBatchListner implements BatchMessageListener> dataList) { long startTime = System.currentTimeMillis(); - // Step 1: Group messages by topic (preserving order within each topic) + // Group messages by topic, preserving arrival order within each topic. Map> topicToDataList = new LinkedHashMap<>(); - dataList.forEach(data -> { try { String jsonValue = objectMapper.writeValueAsString(data.value()); topicToDataList.computeIfAbsent(data.topic(), k -> new ArrayList<>()).add(jsonValue); } catch (JsonProcessingException e) { - log.error("Failed to serialize incoming message", e); - pushToErrorQueue(data.topic(), data.value(), e); + // Unserializable / poison record: it can never reach persist, so dead-letter it directly. + log.error("Failed to serialize incoming message from topic {}", data.topic(), e); + sendToDlq(data.topic(), data.value(), 0, e); } }); - // Step 2: Process topics int[] results = processTopics(topicToDataList); long timeTaken = System.currentTimeMillis() - startTime; - log.info("Batch processed - success: {}, failed: {}, topics: {}, time: {} ms", + log.info("Batch processed - persisted: {}, dead-lettered: {}, topics: {}, time: {} ms", results[0], results[1], topicToDataList.size(), timeTaken); } - /** - * Process topics - directly for single topic, parallel for multiple topics. - */ private int[] processTopics(Map> topicToDataList) { - // Optimization: Direct processing for single topic (avoids thread pool overhead) + // Single topic: process directly (avoids thread-pool hand-off overhead). if (topicToDataList.size() == 1) { Map.Entry> entry = topicToDataList.entrySet().iterator().next(); return processSingleTopic(entry.getKey(), entry.getValue()); } - // Parallel processing for multiple topics - AtomicInteger totalRecordsSuccess = new AtomicInteger(0); - AtomicInteger totalRecordsFailed = new AtomicInteger(0); - - // Capture MDC context from parent thread + AtomicInteger persisted = new AtomicInteger(0); + AtomicInteger failed = new AtomicInteger(0); Map mdcContext = MDC.getCopyOfContextMap(); - List> futures = new ArrayList<>(); for (Map.Entry> entry : topicToDataList.entrySet()) { String topic = entry.getKey(); List messages = entry.getValue(); - - CompletableFuture future = CompletableFuture.runAsync(() -> { + futures.add(CompletableFuture.runAsync(() -> { if (mdcContext != null) { MDC.setContextMap(mdcContext); } try { - int[] result = processSingleTopic(topic, messages); - totalRecordsSuccess.addAndGet(result[0]); - totalRecordsFailed.addAndGet(result[1]); + int[] r = processSingleTopic(topic, messages); + persisted.addAndGet(r[0]); + failed.addAndGet(r[1]); } finally { MDC.clear(); } - }, topicProcessorExecutor); - - futures.add(future); + }, topicProcessorExecutor)); } - // Wait for all topics to complete try { CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } catch (CompletionException e) { - log.error("Error during parallel topic processing", e.getCause()); + // A topic failed to dead-letter durably -> surface it so the offset is NOT committed and + // the whole poll is redelivered (at-least-once). Re-persist of committed rows is a no-op + // under idempotent (ON CONFLICT) writes. + Throwable cause = e.getCause(); + log.error("Parallel topic processing failed; batch will be redelivered", cause); + throw (cause instanceof RuntimeException) ? (RuntimeException) cause : new IllegalStateException(cause); } - - return new int[]{totalRecordsSuccess.get(), totalRecordsFailed.get()}; + return new int[]{persisted.get(), failed.get()}; } /** - * Process a single topic: persist messages and send audit. + * Persist one topic's batch. + * + *

Fast path = a single aggregated batch insert. On failure the cause is classified:

+ *
    + *
  • TRANSIENT (DB/infra unavailable): the whole batch is dead-lettered for bounded retry; + * we do not thrash per-record while the DB is down.
    • + *
  • otherwise: fall back to per-record persistence so the good records commit and only the + * offending record(s) are dead-lettered (R1 — one bad record must not fail the others). + * A row that is already present comes back BENIGN and is counted as success.
    • + *
*/ private int[] processSingleTopic(String topic, List messages) { if (messages.isEmpty()) { return new int[]{0, 0}; } + List persisted; + int failed; try { persistService.persist(topic, messages); - - // Send to audit topic if not audit topic itself - if (!topic.equalsIgnoreCase(persistAuditKafkaTopic)) { - Map producerRecord = new HashMap<>(); - producerRecord.put("topic", topic); - producerRecord.put("value", messages); - kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + persisted = messages; + failed = 0; + } catch (Exception e) { + if (DbExceptionClassifier.classify(e) == DbExceptionClassifier.Kind.TRANSIENT) { + // DB/infra down: the whole batch is good, so don't dead-letter/park it. Rethrow so the + // offset is NOT committed and the batch is retried in place once the DB recovers. + log.warn("Transient failure persisting batch from topic {} ({} records) - retrying whole batch (offset not committed)", + topic, messages.size(), e); + throw new TransientPersistException("Transient DB failure persisting batch for topic " + topic, e); + } + log.warn("Batch persist failed for topic {} ({}); isolating per record", topic, e.getMessage()); + persisted = new ArrayList<>(); + failed = 0; + for (String message : messages) { + try { + persistService.persist(topic, Collections.singletonList(message)); + persisted.add(message); + } catch (Exception ex) { + DbExceptionClassifier.Kind kind = DbExceptionClassifier.classify(ex); + if (kind == DbExceptionClassifier.Kind.BENIGN) { + persisted.add(message); // already present (duplicate) -> idempotent success + } else if (kind == DbExceptionClassifier.Kind.TRANSIENT) { + // DB went down partway through isolation: abort and retry the whole batch rather + // than parking the remaining good records as if they were poison. + throw new TransientPersistException("Transient DB failure isolating record for topic " + topic, ex); + } else { + sendToDlq(topic, message, 0, ex); // isolate only the offending (bad-data) record + failed++; + } + } } + } - log.debug("Successfully processed {} messages from topic: {}", messages.size(), topic); - return new int[]{messages.size(), 0}; + if (!persisted.isEmpty()) { + sendAudit(topic, persisted); + } + log.debug("Topic {}: persisted {}, dead-lettered {}", topic, persisted.size(), failed); + return new int[]{persisted.size(), failed}; + } + /** Best-effort audit AFTER a committed persist; its failure must never dead-letter committed records. */ + private void sendAudit(String topic, List messages) { + if (topic.equalsIgnoreCase(persistAuditKafkaTopic)) { + return; + } + try { + Map producerRecord = new HashMap<>(); + producerRecord.put("topic", topic); + producerRecord.put("value", messages); + kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); } catch (Exception e) { - log.error("Error while persisting messages from topic: {}", topic, e); - messages.forEach(data -> pushToErrorQueue(topic, data, e)); - return new int[]{0, messages.size()}; + log.error("Audit send failed for topic {} ({} records already persisted; NOT dead-lettered)", + topic, messages.size(), e); } } - private void pushToErrorQueue(String topic, Object body, Exception e) { + /** + * Durably (awaited) publish a failed record to the dead-letter topic. CustomKafkaTemplate.send + * blocks on the broker ack, so a failure throws rather than being silently dropped; we rethrow so + * the offset is not committed and the record is redelivered (at-least-once). + */ + private void sendToDlq(String topic, Object body, int attempts, Exception cause) { + Map dlq = new HashMap<>(); + dlq.put("source", topic); + dlq.put("body", body); + dlq.put("attempts", attempts); + dlq.put("ts", System.currentTimeMillis()); + dlq.put("message", cause.getMessage()); + dlq.put("correlationId", MDC.get(CORRELATION_ID_MDC)); try { - ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() - .id(UUID.randomUUID().toString()) - .source(topic) - .body(body) - .ts(System.currentTimeMillis()) - .message(e.getMessage()) - .exception(Arrays.asList(e.getStackTrace())) - .correlationId(MDC.get(CORRELATION_ID_MDC)) - .build(); - errorQueueProducer.sendMessage(errorQueueContract); - log.info("Message pushed to error queue for topic: {}", topic); + kafkaTemplate.send(deadLetterTopic, dlq); + log.info("Dead-lettered record from topic {} (attempts={})", topic, attempts); } catch (Exception ex) { - log.error("Failed to push message to error queue for topic: {}", topic, ex); + log.error("Failed to dead-letter record from topic {}", topic, ex); + throw new IllegalStateException("Dead-letter publish failed for topic " + topic, ex); } } -} \ No newline at end of file +} diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java index 011ac2b00ef..7059c870806 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterConsumerConfig.java @@ -5,7 +5,6 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.egov.infra.persist.web.contract.TopicMap; -import org.egov.tracer.KafkaConsumerErrorHandler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.context.annotation.Bean; @@ -19,6 +18,7 @@ import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.listener.DefaultErrorHandler; import org.springframework.kafka.listener.KafkaMessageListenerContainer; import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer; import org.springframework.kafka.support.serializer.JsonDeserializer; @@ -40,9 +40,6 @@ @Slf4j public class PersisterConsumerConfig { - /* @Autowired - private StoppingErrorHandler stoppingErrorHandler;*/ - @Autowired private PersisterMessageListener indexerMessageListener; @@ -53,7 +50,7 @@ public class PersisterConsumerConfig { private KafkaProperties kafkaProperties; @Autowired - private KafkaConsumerErrorHandler kafkaConsumerErrorHandler; + private DefaultErrorHandler persisterErrorHandler; private Set topics = new HashSet<>(); @@ -75,8 +72,21 @@ public class PersisterConsumerConfig { @Value("${tracer.errorsTopic}") private String deadLetterErrorTopic; + @Value("${persister.kafka.partition.assignment.strategy:org.apache.kafka.clients.consumer.CooperativeStickyAssignor,org.apache.kafka.clients.consumer.RangeAssignor}") + private String partitionAssignmentStrategy; + + @Value("${persister.kafka.group.instance.id:}") + private String groupInstanceId; + + @Value("${persister.kafka.session.timeout.ms:}") + private String sessionTimeoutMsOverride; + private Set configuredBatchTopics = new HashSet<>(); + // The single started container, held so the DB-health monitor can pause/resume the LIVE consumer + // (not rebuild a fresh one). Assigned once in startContainer(). + private KafkaMessageListenerContainer listenerContainer; + @PostConstruct public void setTopics() { // Parse configured batch topics from property @@ -104,8 +114,23 @@ public void setTopics() { public ConsumerFactory consumerFactory() { Map props = kafkaProperties.buildConsumerProperties(); - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); - props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000"); + // Honour spring.kafka.consumer.enable-auto-commit=false: the container commits the offset only + // after the listener returns (record persisted, or durably dead-lettered) — never on a timer. + props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, + StringUtils.hasText(sessionTimeoutMsOverride) ? sessionTimeoutMsOverride : "15000"); + + // Cooperative rebalancing: only the partitions that actually move are revoked, so the other + // members keep consuming straight through a rebalance instead of stopping the world. Listing + // RangeAssignor second keeps the group on the eager protocol until every member in the group + // runs this build, which makes a single rolling deploy safe. + props.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partitionAssignmentStrategy); + + // Static membership: a member that restarts and returns within the session timeout reclaims + // its partitions with no rebalance at all. Suffixed per container type — the single and batch + // containers share group.id, and duplicate instance ids fence each other out of the group. + if (StringUtils.hasText(groupInstanceId)) { + props.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, groupInstanceId + "-single"); + } JsonDeserializer jsonDeserializer = new JsonDeserializer<>(Object.class,false); @@ -122,7 +147,7 @@ public KafkaListenerContainerFactory container() throws Exception { ContainerProperties properties = new ContainerProperties(this.topics.toArray(new String[topics.size()])); properties.setMessageListener(indexerMessageListener); + // Per-record manual ack: offset advances only after the record is durably handled. + properties.setAckMode(ContainerProperties.AckMode.RECORD); if (customExecutorEnabled) { ExecutorService executorService = Executors.newFixedThreadPool(maxPoolSize); AsyncTaskExecutor taskExecutor = new ConcurrentTaskExecutor(executorService); @@ -142,50 +169,41 @@ public KafkaMessageListenerContainer container() throws Exceptio log.info("Custom KafkaListenerContainer built..."); KafkaMessageListenerContainer container = new KafkaMessageListenerContainer<>(consumerFactory(), properties); - container.setCommonErrorHandler(kafkaConsumerErrorHandler); + container.setCommonErrorHandler(persisterErrorHandler); return container; } @Bean public boolean startContainer(){ - KafkaMessageListenerContainer container = null; try { - container = container(); + listenerContainer = container(); } catch (Exception e) { log.error("Container couldn't be started: ",e); return false; } - container.start(); + listenerContainer.start(); log.info("Custom KakfaListenerContainer STARTED..."); return true; } + /** Pause consumption on the LIVE container (keeps partition assignment; no rebalance). Idempotent. */ public boolean pauseContainer(){ - KafkaMessageListenerContainer container = null; - try { - container = container(); - } catch (Exception e) { - log.error("Container couldn't be started: ",e); + if (listenerContainer == null || !listenerContainer.isRunning() || listenerContainer.isContainerPaused()) { return false; } - container.stop(); - log.info("Custom KakfaListenerContainer STOPPED..."); - + listenerContainer.pause(); + log.warn("Persister consumer PAUSED (datasource unavailable)"); return true; } + /** Resume consumption on the LIVE container. Idempotent. */ public boolean resumeContainer(){ - KafkaMessageListenerContainer container = null; - try { - container = container(); - } catch (Exception e) { - log.error("Container couldn't be started: ",e); + if (listenerContainer == null || !listenerContainer.isContainerPaused()) { return false; } - container.start(); - log.info("Custom KakfaListenerContainer STARTED..."); - + listenerContainer.resume(); + log.info("Persister consumer RESUMED (datasource healthy)"); return true; } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index 676d4cd2fda..782003dd6f7 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -1,132 +1,160 @@ package org.egov.infra.persist.consumer; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.errors.SerializationException; import org.egov.infra.persist.service.PersistService; import org.egov.tracer.kafka.CustomKafkaTemplate; -import org.egov.tracer.kafka.ErrorQueueProducer; -import org.egov.tracer.model.ErrorQueueContract; import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.listener.MessageListener; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.extern.slf4j.Slf4j; - import java.util.*; import static org.egov.tracer.constants.TracerConstants.CORRELATION_ID_MDC; +/** + * Single-record listener. Handles both: + * - normal topics: persist, then best-effort audit; + * - the dead-letter topic (when reprocess is enabled): unwrap the dead-letter envelope and re-persist + * one-by-one, with bounded retry (attempt counter) before routing to a terminal parking topic so a + * poison record can never loop forever (R3 + R4). + */ @Service @Slf4j public class PersisterMessageListener implements MessageListener { - - @Autowired - private PersistService persistService; - @Autowired - private ObjectMapper objectMapper; + @Autowired + private PersistService persistService; - @Autowired - private CustomKafkaTemplate kafkaTemplate; + @Autowired + private ObjectMapper objectMapper; - @Value("${audit.persist.kafka.topic}") - private String persistAuditKafkaTopic; + @Autowired + private CustomKafkaTemplate kafkaTemplate; - @Value("${audit.generate.kafka.topic}") - private String auditGenerateKafkaTopic; + @Value("${audit.persist.kafka.topic}") + private String persistAuditKafkaTopic; - @Value("${persister.dead-letter.reprocess.error-topic}") - private String deadLetterReprocessErrorTopic; + @Value("${audit.generate.kafka.topic}") + private String auditGenerateKafkaTopic; - @Value("${tracer.errorsTopic}") - private String tracerErrorsTopic; + @Value("${tracer.errorsTopic}") + private String deadLetterTopic; - @Autowired - private ErrorQueueProducer errorQueueProducer; + @Value("${persister.dead-letter.reprocess.error-topic}") + private String parkingTopic; - @Override - public void onMessage(ConsumerRecord data) { - String rcvData = null; - long startTime = System.currentTimeMillis(); - - String topic = data.topic(); - String deadLetterTopic = null; - Object body = null; - try { - if (Objects.equals(topic, tracerErrorsTopic)) { - LinkedHashMap message = (LinkedHashMap) data.value(); - topic = message.get("source").toString(); - body = message.get("body"); - deadLetterTopic = data.topic(); - } else { - body = data.value(); - } - rcvData = objectMapper.writeValueAsString(body); - persistService.persist(topic, rcvData); - if(!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)){ - Map producerRecord = new HashMap<>(); - producerRecord.put("topic", topic); - producerRecord.put("value", body); - kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); - } - log.info("Message from topic: {} processed successfully in {} ms.", topic, System.currentTimeMillis() - startTime); - } catch (Exception e) { - log.error("Error while persisting message from topic: {}", topic, e); - if(deadLetterTopic == null) { - pushToErrorQueue(topic, body, e); - } else { - sendErrorMessage(deadLetterReprocessErrorTopic, deadLetterTopic, body, e); - } - } - } - - private void pushToErrorQueue(String topic, Object body, Exception e) { - try { - ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() - .id(UUID.randomUUID().toString()) - .source(topic) - .body(body) - .ts(System.currentTimeMillis()) - .message(e.getMessage()) - .exception(Arrays.asList(e.getStackTrace())) - .correlationId(MDC.get(CORRELATION_ID_MDC)) - .build(); - errorQueueProducer.sendMessage(errorQueueContract); - log.info("Message pushed to error queue for topic: {}", topic); - } catch (Exception ex) { - log.error("Failed to push message to error queue for topic: {}", topic, ex); - } - } - - public void sendErrorMessage(String errorTopic, String topic, Object body, Exception ex) { - ErrorQueueContract errorQueueContract = ErrorQueueContract.builder() - .id(UUID.randomUUID().toString()) - .source(topic) - .body(body) - .ts(System.currentTimeMillis()) - .message(ex.getMessage()) - .exception(Arrays.asList(ex.getStackTrace())) - .correlationId(MDC.get(CORRELATION_ID_MDC)) - .build(); - try { - log.info("Sending message to topic - " + errorTopic); - kafkaTemplate.send(errorTopic, errorQueueContract); - } catch (SerializationException serializationException) { - log.info("SerializationException exception occurred while sending exception to error queue"); - try { - kafkaTemplate.send(errorTopic, objectMapper.writeValueAsString(errorQueueContract)); - } catch (JsonProcessingException e) { - log.error("exception occurred while converting ErrorQueueContract to json string", e); - } - } catch (Exception e) { - log.error("exception occurred while sending exception to error queue", e); - } - } + @Value("${persister.dead-letter.max-retries:5}") + private Integer maxRetries; + @Override + @SuppressWarnings("unchecked") + public void onMessage(ConsumerRecord data) { + long startTime = System.currentTimeMillis(); + String topic = data.topic(); + boolean fromDlq = Objects.equals(topic, deadLetterTopic); + Object body = null; + int attempts = 0; + + try { + if (fromDlq) { + // Dead-letter envelope: {source, body, attempts, ...}. Read these BEFORE persisting so a + // malformed envelope still routes to the bounded-retry/park path, not back as a fresh record. + LinkedHashMap message = (LinkedHashMap) data.value(); + topic = String.valueOf(message.get("source")); + body = message.get("body"); + Object a = message.get("attempts"); + // A dead-letter envelope with no numeric attempts (a foreign/raw record) is treated as + // already at the ceiling, so it parks instead of being granted a fresh retry budget. + attempts = (a instanceof Number) ? ((Number) a).intValue() : maxRetries; + } else { + body = data.value(); + } + // Batch-originated envelopes carry body as a pre-serialized JSON String; the single path + // carries a structured object. Re-serialize ONLY the latter, otherwise the String would be + // double-encoded and PersistService would silently extract no rows. + String json = (body instanceof String) ? (String) body : objectMapper.writeValueAsString(body); + persistService.persist(topic, json); + log.info("Message from topic {} persisted in {} ms.", topic, System.currentTimeMillis() - startTime); + } catch (Exception e) { + DbExceptionClassifier.Kind kind = DbExceptionClassifier.classify(e); + if (kind == DbExceptionClassifier.Kind.BENIGN) { + // Row already present (redelivery / DLQ replay) -> idempotent success; never dead-letter it. + log.info("Record for topic {} already present (duplicate) - idempotent success.", topic); + return; + } + if (kind == DbExceptionClassifier.Kind.TRANSIENT) { + // DB/infra momentarily unavailable. The record is GOOD, so it must not be dead-lettered or + // parked: rethrow so the container error handler retries it in place (offset NOT committed) + // until the DB recovers. The DB-health monitor pauses the consumer meanwhile. This holds + // whether the record arrived fresh or from the DLQ - a transient blip never burns the + // poison-retry budget and never strands good data. + log.warn("Transient failure persisting record from topic {} - retrying in place (not dead-lettered)", topic, e); + throw new TransientPersistException("Transient DB failure persisting topic " + topic, e); + } + // PERMANENT (bad data): bounded dead-letter reprocessing, then terminal parking (R3 + R4). + if (!fromDlq) { + // First failure -> dead-letter for bounded reprocessing. + sendToDlq(topic, body, 1, e); + } else if (attempts < maxRetries) { + // Still failing but budget remains -> reprocess once more via the dead-letter topic. + sendToDlq(topic, body, attempts + 1, e); + } else { + // Retry budget exhausted -> terminal parking topic (no further retry -> no infinite loop). + sendToParking(topic, body, attempts, e); + } + return; + } + + // Best-effort audit AFTER a committed persist (skip the audit topic itself). Isolated so its + // failure cannot dead-letter the already-persisted record. + if (!data.topic().equalsIgnoreCase(persistAuditKafkaTopic)) { + try { + Map producerRecord = new HashMap<>(); + producerRecord.put("topic", topic); + producerRecord.put("value", body); + kafkaTemplate.send(auditGenerateKafkaTopic, producerRecord); + } catch (Exception e) { + log.error("Audit send failed for topic {} (record already persisted; NOT dead-lettered)", topic, e); + } + } + } + + /** Awaited dead-letter publish; rethrows on failure so the record is not silently dropped. */ + private void sendToDlq(String topic, Object body, int attempts, Exception cause) { + try { + kafkaTemplate.send(deadLetterTopic, dlqPayload(topic, body, attempts, cause)); + log.info("Dead-lettered record from topic {} (attempts={})", topic, attempts); + } catch (Exception ex) { + log.error("Failed to dead-letter record from topic {}", topic, ex); + throw new IllegalStateException("Dead-letter publish failed for topic " + topic, ex); + } + } + + /** Terminal park after the retry ceiling (or for permanent failures). Best-effort; never loops back. */ + private void sendToParking(String topic, Object body, int attempts, Exception cause) { + try { + kafkaTemplate.send(parkingTopic, dlqPayload(topic, body, attempts, cause)); + log.warn("Parked record from topic {} after {} attempt(s) (terminal, no further retry)", topic, attempts); + } catch (Exception ex) { + // Do NOT swallow: rethrow so the container error handler retries and, if parking stays down, + // seeks back WITHOUT committing the offset -> the record is redelivered, never silently lost. + log.error("Failed to park record from topic {}; offset will not be committed", topic, ex); + throw new IllegalStateException("Parking publish failed for topic " + topic, ex); + } + } + + private Map dlqPayload(String topic, Object body, int attempts, Exception cause) { + Map m = new HashMap<>(); + m.put("source", topic); + m.put("body", body); + m.put("attempts", attempts); + m.put("ts", System.currentTimeMillis()); + m.put("message", cause.getMessage()); + m.put("correlationId", MDC.get(CORRELATION_ID_MDC)); + return m; + } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterProducerConfig.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterProducerConfig.java index 5f523c59ac2..444b09f6911 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterProducerConfig.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterProducerConfig.java @@ -1,13 +1,19 @@ package org.egov.infra.persist.consumer; +import org.egov.tracer.kafka.CustomKafkaTemplate; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.DefaultKafkaProducerFactory; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ConsumerRecordRecoverer; +import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.util.backoff.FixedBackOff; +import java.util.HashMap; import java.util.Map; @Configuration @@ -33,5 +39,47 @@ public Map producerConfigs() { return new KafkaTemplate<>(producerFactory()); } + /** + * Persister-local Kafka error handler used by both consumer containers (replaces the shared tracer + * handler, which committed offsets on a thrown listener exception and so lost the record). + * + *

It fires only when a listener itself throws — which in this design happens only when a durable + * DLQ publish fails. It retries a few times (the broker may recover) and, on exhaustion, durably + * parks the raw record via a blocking send that throws on failure, so the offset is NOT committed + * until the record is parked (no silent loss). Deserialization-poison records are non-retryable by + * default in DefaultErrorHandler and are parked immediately rather than looping.

+ */ + @Bean + public DefaultErrorHandler persisterErrorHandler(CustomKafkaTemplate customKafkaTemplate, + @Value("${persister.dead-letter.reprocess.error-topic}") String parkingTopic) { + ConsumerRecordRecoverer recoverer = (record, ex) -> { + // A transient failure must NEVER be parked (the record is good, the DB was just down): rethrow + // so the container re-seeks and keeps retrying until the DB recovers. In practice the unlimited + // back-off below means a transient failure never reaches the recoverer at all - this is a guard. + if (DbExceptionClassifier.classify(ex) == DbExceptionClassifier.Kind.TRANSIENT) { + throw new IllegalStateException("DB still unavailable - refusing to park transient failure, will retry", ex); + } + // Permanent / poison (incl. undeserialisable records): park a self-describing envelope (source + // topic + error reason), not the bare value, so a terminally-parked record can be triaged/replayed. + // Blocking send -> throws on failure, so a failed park is not committed (DefaultErrorHandler + // re-seeks instead of dropping the record). + Map parked = new HashMap<>(); + parked.put("source", record.topic()); + parked.put("body", record.value()); + parked.put("error", ex == null ? null : ex.toString()); + parked.put("ts", System.currentTimeMillis()); + customKafkaTemplate.send(parkingTopic, parked); + }; + // Default back-off (permanent/poison): a modest ceiling, then park - the poison backstop (R4). + DefaultErrorHandler handler = new DefaultErrorHandler(recoverer, new FixedBackOff(2000L, 5L)); + // Transient failures (DB/infra down): retry in place indefinitely with back-off so a good record is + // never parked for an outage - the offset only advances once it finally persists. Consumption is + // paused by the DB-health monitor while the datasource is down, so this is not a hot spin. + handler.setBackOffFunction((record, ex) -> + DbExceptionClassifier.classify(ex) == DbExceptionClassifier.Kind.TRANSIENT + ? new FixedBackOff(2000L, FixedBackOff.UNLIMITED_ATTEMPTS) + : new FixedBackOff(2000L, 5L)); + return handler; + } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/StoppingErrorHandler.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/StoppingErrorHandler.java deleted file mode 100644 index d4c8e1111fb..00000000000 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/StoppingErrorHandler.java +++ /dev/null @@ -1,23 +0,0 @@ -/* -package org.egov.infra.persist.consumer; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.config.KafkaListenerEndpointRegistry; -import org.springframework.kafka.listener.CommonErrorHandler; -import org.springframework.kafka.listener.ErrorHandler; -import org.springframework.stereotype.Component; - -@Component -public class StoppingErrorHandler implements CommonErrorHandler { - - @Autowired - private KafkaListenerEndpointRegistry kafkaListenerEndpointRegistry; - - @Override - public void handle(Exception thrownException, ConsumerRecord record) { - kafkaListenerEndpointRegistry.stop(); - } - -} -*/ diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/TransientPersistException.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/TransientPersistException.java new file mode 100644 index 00000000000..291f45e2572 --- /dev/null +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/TransientPersistException.java @@ -0,0 +1,19 @@ +package org.egov.infra.persist.consumer; + +/** + * Thrown when a persist fails for a TRANSIENT reason (DB/infra unavailable, deadlock, serialization, + * connection-acquisition, etc. — see {@link DbExceptionClassifier}). + * + *

A transient failure must never dead-letter or park a good record: the data is fine, the + * infrastructure is momentarily not. Escaping the listener with this exception makes the container's + * error handler retry the record IN PLACE (with back-off) without committing the offset, so the record + * is re-attempted until the DB recovers rather than being diverted. The paired DB-health monitor pauses + * consumption while the datasource is down. This is the "retry, don't drop" half of the invariant — + * permanent (bad-data) failures are the ones that go DLQ → bounded retry → parking.

+ */ +public class TransientPersistException extends RuntimeException { + + public TransientPersistException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core-services/egov-persister/src/main/resources/application.properties b/core-services/egov-persister/src/main/resources/application.properties index 859817d6b6b..05a4284f217 100644 --- a/core-services/egov-persister/src/main/resources/application.properties +++ b/core-services/egov-persister/src/main/resources/application.properties @@ -61,7 +61,9 @@ default.version=1.0.0 audit.persist.kafka.topic=audit-create audit.generate.kafka.topic=process-audit-records -persister.custom.executor.enabled=true +# Listener-task-executor on a single KafkaMessageListenerContainer does not parallelise records and +# can interfere with per-record ack ordering; left off. +persister.custom.executor.enabled=false persister.custom.executor.max-pool-size=10 # Thread pool size for parallel topic processing within a batch @@ -69,6 +71,16 @@ persister.batch.parallel-topic-processing.thread-pool-size=1 persister.dead-letter.reprocess.enabled=true persister.dead-letter.reprocess.error-topic=egov-persister-deadletter-processed +# Bounded DLQ retries before a record is moved to the terminal parking topic (R4 - no infinite loop). +persister.dead-letter.max-retries=5 + +# Pause the consumer while the datasource is unreachable and resume when it recovers, so transient +# failures retry in place instead of hammering a dead DB (pause-on-DB-health backstop). +persister.db-health.check-interval-ms=5000 + +# DLQ / parking producer reliability (durable, no-loss publishes). +spring.kafka.producer.acks=all +spring.kafka.producer.properties.enable.idempotence=true otel.traces.exporter=otlp otel.service.name=egov-persister diff --git a/core-services/egov-persister/src/main/resources/egov-pg-service-persister.yml b/core-services/egov-persister/src/main/resources/egov-pg-service-persister.yml index fe548fc071e..2f828efbb6a 100644 --- a/core-services/egov-persister/src/main/resources/egov-pg-service-persister.yml +++ b/core-services/egov-persister/src/main/resources/egov-pg-service-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-pg-txns isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, gateway, "module", order_id, product_info, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, created_time, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, gateway, "module", order_id, product_info, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, created_time, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; basePath: Transaction jsonMaps: @@ -59,7 +59,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_time, last_modified_time) VALUES (?, ?, ?, ?, ?); + - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_time, last_modified_time) VALUES (?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; basePath: TransactionDump jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/egov-user-event-persister.yml b/core-services/egov-persister/src/main/resources/egov-user-event-persister.yml index 8132b8c888a..f9b6023b582 100644 --- a/core-services/egov-persister/src/main/resources/egov-user-event-persister.yml +++ b/core-services/egov-persister/src/main/resources/egov-user-event-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-user-events isTransaction: true queryMaps: - - query: INSERT INTO eg_usrevents_events(tenantid, id, source, eventtype, category, name, postedby, referenceid, description, status, eventdetails, actions, recepient, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_usrevents_events(tenantid, id, source, eventtype, category, name, postedby, referenceid, description, status, eventdetails, actions, recepient, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: events.* jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/hrms-employee-persister.yml b/core-services/egov-persister/src/main/resources/hrms-employee-persister.yml index e3061895ac5..522ada8709f 100644 --- a/core-services/egov-persister/src/main/resources/hrms-employee-persister.yml +++ b/core-services/egov-persister/src/main/resources/hrms-employee-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-hrms-employee isTransaction: true queryMaps: - - query: INSERT INTO eg_hrms_employee(tenantid, id, uuid, code, phone, name, dateOfAppointment, employeestatus, employeetype, active, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_employee(tenantid, id, uuid, code, phone, name, dateOfAppointment, employeestatus, employeetype, active, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.* jsonMaps: @@ -43,7 +43,7 @@ serviceMaps: - - query: INSERT INTO eg_hrms_assignment(tenantid, uuid, position, department, designation, fromdate, todate, govtordernumber, reportingto, isHOD, employeeid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_assignment(tenantid, uuid, position, department, designation, fromdate, todate, govtordernumber, reportingto, isHOD, employeeid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.assignments.* jsonMaps: @@ -80,7 +80,7 @@ serviceMaps: - jsonPath: $.Employees.*.assignments.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_educationaldetails(tenantid, uuid, employeeid, qualification, stream, yearofpassing, university, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_educationaldetails(tenantid, uuid, employeeid, qualification, stream, yearofpassing, university, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.education.* jsonMaps: @@ -111,7 +111,7 @@ serviceMaps: - jsonPath: $.Employees.*.education.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_departmentaltests(tenantid, uuid, employeeid, test, yearofpassing, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_departmentaltests(tenantid, uuid, employeeid, test, yearofpassing, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.tests.* jsonMaps: @@ -138,7 +138,7 @@ serviceMaps: - jsonPath: $.Employees.*.tests.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_empdocuments(tenantid, uuid, employeeid, documentid, documentname, referencetype, referenceid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_empdocuments(tenantid, uuid, employeeid, documentid, documentname, referencetype, referenceid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.documents.* jsonMaps: @@ -167,7 +167,7 @@ serviceMaps: - jsonPath: $.Employees.*.documents.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_servicehistory(tenantid, uuid, employeeid, servicestatus, servicefrom, serviceto, ordernumber, isCurrentPosition, location, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_servicehistory(tenantid, uuid, employeeid, servicestatus, servicefrom, serviceto, ordernumber, isCurrentPosition, location, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.serviceHistory.* jsonMaps: @@ -200,7 +200,7 @@ serviceMaps: - jsonPath: $.Employees.*.serviceHistory.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_jurisdiction (uuid, employeeid, hierarchy, boundarytype, boundary, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_jurisdiction (uuid, employeeid, hierarchy, boundarytype, boundary, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.jurisdictions.* jsonMaps: @@ -241,7 +241,7 @@ serviceMaps: - jsonPath: $.Employees.*.uuid - - query: INSERT INTO eg_hrms_employee(tenantid, id, uuid, code, phone, name, dateOfAppointment, employeestatus, employeetype, active, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_employee(tenantid, id, uuid, code, phone, name, dateOfAppointment, employeestatus, employeetype, active, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.* jsonMaps: @@ -277,7 +277,7 @@ serviceMaps: - - query: INSERT INTO eg_hrms_assignment(tenantid, uuid, position, department, designation, fromdate, todate, govtordernumber, reportingto, isHOD, employeeid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_assignment(tenantid, uuid, position, department, designation, fromdate, todate, govtordernumber, reportingto, isHOD, employeeid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.assignments.* jsonMaps: @@ -314,7 +314,7 @@ serviceMaps: - jsonPath: $.Employees.*.assignments.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_educationaldetails(tenantid, uuid, employeeid, qualification, stream, yearofpassing, university, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_educationaldetails(tenantid, uuid, employeeid, qualification, stream, yearofpassing, university, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.education.* jsonMaps: @@ -345,7 +345,7 @@ serviceMaps: - jsonPath: $.Employees.*.education.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_departmentaltests(tenantid, uuid, employeeid, test, yearofpassing, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_departmentaltests(tenantid, uuid, employeeid, test, yearofpassing, remarks, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.tests.* jsonMaps: @@ -372,7 +372,7 @@ serviceMaps: - jsonPath: $.Employees.*.tests.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_empdocuments(tenantid, uuid, employeeid, documentid, documentname, referencetype, referenceid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_empdocuments(tenantid, uuid, employeeid, documentid, documentname, referencetype, referenceid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.documents.* jsonMaps: @@ -401,7 +401,7 @@ serviceMaps: - jsonPath: $.Employees.*.documents.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_servicehistory(tenantid, uuid, employeeid, servicestatus, servicefrom, serviceto, ordernumber, isCurrentPosition, location, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_servicehistory(tenantid, uuid, employeeid, servicestatus, servicefrom, serviceto, ordernumber, isCurrentPosition, location, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.serviceHistory.* jsonMaps: @@ -434,7 +434,7 @@ serviceMaps: - jsonPath: $.Employees.*.serviceHistory.*.auditDetails.lastModifiedDate - - query: INSERT INTO eg_hrms_jurisdiction (uuid, employeeid, hierarchy, boundarytype, boundary, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_jurisdiction (uuid, employeeid, hierarchy, boundarytype, boundary, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.jurisdictions.* jsonMaps: @@ -461,7 +461,7 @@ serviceMaps: - - query: INSERT INTO eg_hrms_deactivationdetails(uuid, employeeid, reasonfordeactivation, effectivefrom, ordernumber, typeOfDeactivation, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_hrms_deactivationdetails(uuid, employeeid, reasonfordeactivation, effectivefrom, ordernumber, typeOfDeactivation, tenantid, createdby, createddate, lastmodifiedby, lastModifiedDate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid) DO NOTHING; basePath: Employees.*.deactivationDetails.* jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/persister.yml b/core-services/egov-persister/src/main/resources/persister.yml index 66592637c9f..5db79ecfa6d 100644 --- a/core-services/egov-persister/src/main/resources/persister.yml +++ b/core-services/egov-persister/src/main/resources/persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-index-jobs isTransaction: true queryMaps: - - query: INSERT INTO eg_indexer_job(tenantid, jobid, requesterid, typeofjob, oldindex, newindex, jobstatus, totaltimetakeninms, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_indexer_job(tenantid, jobid, requesterid, typeofjob, oldindex, newindex, jobstatus, totaltimetakeninms, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (jobid) DO NOTHING; basePath: job jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/pgr.v3.yml b/core-services/egov-persister/src/main/resources/pgr.v3.yml index 29a1a136a6a..04cd6eded5c 100644 --- a/core-services/egov-persister/src/main/resources/pgr.v3.yml +++ b/core-services/egov-persister/src/main/resources/pgr.v3.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-pgr-service isTransaction: true queryMaps: - - query: INSERT INTO eg_pgr_service(tenantid, servicecode, servicerequestid, description, lat, "long", addressid, address, email, deviceid, accountid, firstname, lastname, phone, attributes, status, source, expectedtime, rating, feedback, landmark, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pgr_service(tenantid, servicecode, servicerequestid, description, lat, "long", addressid, address, email, deviceid, accountid, firstname, lastname, phone, attributes, status, source, expectedtime, rating, feedback, landmark, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, servicerequestid) DO NOTHING; basePath: services.* jsonMaps: @@ -66,7 +66,7 @@ serviceMaps: - jsonPath: $.services.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_pgr_action(uuid, by, "when", action, status, comments, media, assignee, isinternal, tenantid, businesskey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pgr_action(uuid, by, "when", action, status, comments, media, assignee, isinternal, tenantid, businesskey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid, tenantid) DO NOTHING; basePath: actionInfo.* jsonMaps: @@ -96,7 +96,7 @@ serviceMaps: - jsonPath: $.actionInfo.*.businessKey - - query: INSERT INTO eg_pgr_address(uuid, housenoandstreetname, mohalla, landmark, latitude, longitude, city, tenantid, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pgr_address(uuid, housenoandstreetname, mohalla, landmark, latitude, longitude, city, tenantid, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, uuid) DO NOTHING; basePath: services.* jsonMaps: @@ -185,7 +185,7 @@ serviceMaps: - jsonPath: $.services.*.serviceRequestId - - query: INSERT INTO eg_pgr_action(uuid, by, "when", action, status, comments, media, assignee, isinternal, tenantid, businesskey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pgr_action(uuid, by, "when", action, status, comments, media, assignee, isinternal, tenantid, businesskey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (uuid, tenantid) DO NOTHING; basePath: actionInfo.* jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/property-services.yml b/core-services/egov-persister/src/main/resources/property-services.yml index 27a10a26ca5..fa37f670d80 100644 --- a/core-services/egov-persister/src/main/resources/property-services.yml +++ b/core-services/egov-persister/src/main/resources/property-services.yml @@ -7,7 +7,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_pt_property_v2(tenantId,propertyId, status,acknowldgementNumber, oldPropertyId, creationReason, occupancyDate ,createdBy,createdTime,lastModifiedBy,lastModifiedTime ) VALUES (?,?,?,?,?,?,?,?,?,?,?); + - query: INSERT INTO eg_pt_property_v2(tenantId,propertyId, status,acknowldgementNumber, oldPropertyId, creationReason, occupancyDate ,createdBy,createdTime,lastModifiedBy,lastModifiedTime ) VALUES (?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT (propertyid, tenantid) DO NOTHING; basePath: Properties.* jsonMaps: - jsonPath: $.Properties.*.tenantId @@ -33,7 +33,7 @@ serviceMaps: - jsonPath: $.Properties.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_pt_propertydetail_v2(tenantId,assessmentNumber,property,accountId,ownershipCategory,subOwnershipCategory,source,usage,noOfFloors,landArea,buildUpArea,additionalDetails,channel,financialYear,propertyType,propertySubType,usageCategoryMajor,assessmentDate,adhocExemption,adhocPenalty,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?); + - query: INSERT INTO eg_pt_propertydetail_v2(tenantId,assessmentNumber,property,accountId,ownershipCategory,subOwnershipCategory,source,usage,noOfFloors,landArea,buildUpArea,additionalDetails,channel,financialYear,propertyType,propertySubType,usageCategoryMajor,assessmentDate,adhocExemption,adhocPenalty,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT (assessmentnumber) DO NOTHING; basePath: Properties.*.propertyDetails.* jsonMaps: @@ -87,7 +87,7 @@ serviceMaps: - jsonPath: $.Properties.*.propertyDetails.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_pt_owner_v2(tenantId,propertyDetail, userid, isactive,isPrimaryOwner,ownerShipPercentage, ownerType,institutionId,relationship,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pt_owner_v2(tenantId,propertyDetail, userid, isactive,isPrimaryOwner,ownerShipPercentage, ownerType,institutionId,relationship,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (userid, propertydetail) DO NOTHING; basePath: Properties.*.propertyDetails.*.owners.* jsonMaps: @@ -119,7 +119,7 @@ serviceMaps: - - query: INSERT INTO eg_pt_document_propertydetail_v2(tenantId,id, propertydetail, documenttype, fileStore,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pt_document_propertydetail_v2(tenantId,id, propertydetail, documenttype, fileStore,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Properties.*.propertyDetails.*.documents.* jsonMaps: @@ -144,7 +144,7 @@ serviceMaps: - jsonPath: $.Properties[*][?({id} in @.propertyDetails.*.documents[*].id)].auditDetails.lastModifiedTime - - query: INSERT INTO eg_pt_document_owner_v2(tenantId,id, userid, documenttype, fileStore,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pt_document_owner_v2(tenantId,id, userid, documenttype, fileStore,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; basePath: $.Properties.*.propertyDetails.*.owners.*.document jsonMaps: @@ -170,7 +170,7 @@ serviceMaps: - - query: INSERT INTO eg_pt_address_v2(tenantId, id, property,doorNo, latitude,longitude, addressid, addressnumber, type, addressline1, addressline2, landmark, city, pincode, detail,buildingName,street, locality, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pt_address_v2(tenantId, id, property,doorNo, latitude,longitude, addressid, addressnumber, type, addressline1, addressline2, landmark, city, pincode, detail,buildingName,street, locality, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id, property) DO NOTHING; basePath: Properties.*.address jsonMaps: @@ -219,7 +219,7 @@ serviceMaps: - jsonPath: $.Properties.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_pt_unit_v2(tenantId,id,propertyDetail,floorNo,unitType,unitArea,usageCategoryMajor,usageCategoryMinor,usageCategorySubMinor,usageCategoryDetail,occupancyType,occupancyDate,constructionType,constructionSubType,arv,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?); + - query: INSERT INTO eg_pt_unit_v2(tenantId,id,propertyDetail,floorNo,unitType,unitArea,usageCategoryMajor,usageCategoryMinor,usageCategorySubMinor,usageCategoryDetail,occupancyType,occupancyDate,constructionType,constructionSubType,arv,createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT (id) DO NOTHING; basePath: Properties.*.propertyDetails.*.units.* jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/pt-drafts.yml b/core-services/egov-persister/src/main/resources/pt-drafts.yml index e1a802749ca..9e9f4ff951a 100644 --- a/core-services/egov-persister/src/main/resources/pt-drafts.yml +++ b/core-services/egov-persister/src/main/resources/pt-drafts.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-pt-drafts isTransaction: true queryMaps: - - query: INSERT INTO eg_pt_drafts_v2(id, userid, tenantid, draft, createdBy, createdTime, lastModifiedBy, lastModifiedTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pt_drafts_v2(id, userid, tenantid, draft, createdBy, createdTime, lastModifiedBy, lastModifiedTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id, tenantid) DO NOTHING; basePath: draft jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/tl-billing-slab-persister.yml b/core-services/egov-persister/src/main/resources/tl-billing-slab-persister.yml index d9386333697..7df3347abda 100644 --- a/core-services/egov-persister/src/main/resources/tl-billing-slab-persister.yml +++ b/core-services/egov-persister/src/main/resources/tl-billing-slab-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-tl-billingslab isTransaction: true queryMaps: - - query: INSERT INTO eg_tl_billingSlab(id, tenantid, licensetype, structuretype, tradetype, accessorycategory, type, uom, fromUom, toUom, rate, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, cast(? as double precision), cast(? as double precision), ?, ?, ?, ?, ?); + - query: INSERT INTO eg_tl_billingSlab(id, tenantid, licensetype, structuretype, tradetype, accessorycategory, type, uom, fromUom, toUom, rate, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, cast(? as double precision), cast(? as double precision), ?, ?, ?, ?, ?) ON CONFLICT (tenantid, licensetype, structuretype, tradetype, accessorycategory, type, uom, fromuom, touom) DO NOTHING; basePath: billingSlab.* jsonMaps: diff --git a/core-services/egov-persister/src/main/resources/user-service-persist.yml b/core-services/egov-persister/src/main/resources/user-service-persist.yml index 34a37e490e5..6f8f3e1b35d 100644 --- a/core-services/egov-persister/src/main/resources/user-service-persist.yml +++ b/core-services/egov-persister/src/main/resources/user-service-persist.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: user-save-topic isTransaction : true queryMaps: - - query: INSERT INTO eg_user(id, salutation, locale, username, pwdexpirytime, mobilenumber,emailid, name, gender, aadhaarnumber, type, active, accountlocked,tenantid,createdby,lastmodifiedby,createdtime,lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?,?,?,?,?,?); + - query: INSERT INTO eg_user(id, salutation, locale, username, pwdexpirytime, mobilenumber,emailid, name, gender, aadhaarnumber, type, active, accountlocked,tenantid,createdby,lastmodifiedby,createdtime,lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?,?,?,?,?,?) ON CONFLICT (username, type, tenantid) DO NOTHING; basePath: $.users.* jsonMaps: - jsonPath: $.users.*.id @@ -77,7 +77,7 @@ serviceMaps: - jsonPath: $.users.*.userDetails.signature - - query : INSERT INTO eg_user_address(type, address, city, pincode, userid, tenantid) VALUES (?, ?, ?, ?, ?, ?); + - query : INSERT INTO eg_user_address(type, address, city, pincode, userid, tenantid) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (userid, tenantid, type) DO NOTHING; basePath: $.users.*.userDetails.addresses.* jsonMaps: - jsonPath: $.users.*.userDetails.addresses.*.addressType @@ -175,7 +175,7 @@ serviceMaps: - jsonPath: $.users.*.tenantId - - query : INSERT INTO eg_user_address(type, address, city, pincode, userid, tenantid) VALUES (?, ?, ?, ?, ?, ?); + - query : INSERT INTO eg_user_address(type, address, city, pincode, userid, tenantid) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (userid, tenantid, type) DO NOTHING; basePath: $.users.*.userDetails.addresses.* jsonMaps: - jsonPath: $.users.*.userDetails.addresses.*.addressType diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java new file mode 100644 index 00000000000..057dfe898f5 --- /dev/null +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java @@ -0,0 +1,60 @@ +package org.egov.infra.persist.consumer; + +import org.junit.jupiter.api.Test; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.jdbc.CannotGetJdbcConnectionException; + +import java.sql.SQLException; + +import static org.egov.infra.persist.consumer.DbExceptionClassifier.Kind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DbExceptionClassifierTest { + + private static SQLException sqlState(String state) { + return new SQLException("db error", state); + } + + @Test + void uniqueViolationIsBenign() { + assertEquals(Kind.BENIGN, DbExceptionClassifier.classify(sqlState("23505"))); + assertTrue(DbExceptionClassifier.isBenign(sqlState("23505"))); + } + + @Test + void connectionAndConcurrencyStatesAreTransient() { + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("08006"))); // connection failure + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("08001"))); // unable to connect + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("40001"))); // serialization_failure + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("40P01"))); // deadlock_detected + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57014"))); // query_canceled + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("53300"))); // too_many_connections + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("55P03"))); // lock_not_available + } + + @Test + void constraintAndDataStatesArePermanent() { + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("23502"))); // not_null_violation + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("23503"))); // foreign_key_violation + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("23514"))); // check_violation + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("22P02"))); // invalid_text_representation + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(new RuntimeException("no sqlstate at all"))); + } + + @Test + void classifiesThroughTheCauseChain() { + // A real failure is wrapped several layers deep (JdbcTemplate -> DataAccessException -> SQLException). + Throwable wrapped = new RuntimeException("outer", + new DataIntegrityViolationException("mid", sqlState("23505"))); + assertEquals(Kind.BENIGN, DbExceptionClassifier.classify(wrapped)); + } + + @Test + void connectionAcquisitionFailureIsTransientEvenWithoutSqlState() { + // Spring files connection-acquisition failures on its NON-transient branch and they may carry no + // SQLState - the classifier must still recognise them as transient via the cause-chain class name. + Throwable ex = new CannotGetJdbcConnectionException("could not get connection"); + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(ex)); + } +} diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java index b9bc8c12503..34ea3d08728 100644 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java @@ -1,50 +1,140 @@ package org.egov.infra.persist.consumer; -import static org.mockito.Mockito.any; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.egov.infra.persist.service.PersistService; +import org.egov.tracer.kafka.CustomKafkaTemplate; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.sql.SQLException; +import java.util.LinkedHashMap; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.egov.infra.persist.service.PersistService; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -@ContextConfiguration(classes = {PersisterMessageListener.class}) -@ExtendWith(SpringExtension.class) +/** + * Routing tests for the single-record listener: benign duplicates are idempotent successes, transient + * failures retry in place (rethrow, never dead-lettered), permanent failures go to the DLQ and then to + * parking once the retry budget is spent, and a pre-serialised String body is never double-encoded. + */ class PersisterMessageListenerTest { - @MockBean - private ObjectMapper objectMapper; - @MockBean + private static final String DLQ = "dlq"; + private static final String PARK = "park"; + private static final int MAX_RETRIES = 5; + private PersistService persistService; + private ObjectMapper objectMapper; + private CustomKafkaTemplate kafkaTemplate; + private PersisterMessageListener listener; - @Autowired - private PersisterMessageListener persisterMessageListener; + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() { + persistService = mock(PersistService.class); + objectMapper = mock(ObjectMapper.class); + kafkaTemplate = mock(CustomKafkaTemplate.class); - //@Test - void testOnMessage() throws JsonProcessingException { - doNothing().when(this.persistService).persist((String) any(), (String) any()); - when(this.objectMapper.writeValueAsString((Object) any())).thenReturn("42"); - this.persisterMessageListener.onMessage(new ConsumerRecord<>("Topic", 1, 1L, "Key", "Value")); - verify(this.persistService).persist((String) any(), (String) any()); - verify(this.objectMapper).writeValueAsString((Object) any()); + listener = new PersisterMessageListener(); + ReflectionTestUtils.setField(listener, "persistService", persistService); + ReflectionTestUtils.setField(listener, "objectMapper", objectMapper); + ReflectionTestUtils.setField(listener, "kafkaTemplate", kafkaTemplate); + ReflectionTestUtils.setField(listener, "persistAuditKafkaTopic", "audit-create"); + ReflectionTestUtils.setField(listener, "auditGenerateKafkaTopic", "process-audit-records"); + ReflectionTestUtils.setField(listener, "deadLetterTopic", DLQ); + ReflectionTestUtils.setField(listener, "parkingTopic", PARK); + ReflectionTestUtils.setField(listener, "maxRetries", MAX_RETRIES); } - //@Test - void testOnMessage2() throws JsonProcessingException { - doNothing().when(this.persistService).persist((String) any(), (String) any()); - when(this.objectMapper.writeValueAsString((Object) any())).thenThrow(mock(JsonProcessingException.class)); - this.persisterMessageListener.onMessage(new ConsumerRecord<>("Topic", 1, 1L, "Key", "Value")); - verify(this.persistService).persist((String) any(), (String) any()); - verify(this.objectMapper).writeValueAsString((Object) any()); + /** Wrap a SQLState in the kind of unchecked exception JdbcTemplate would surface. */ + private static RuntimeException dbError(String sqlState) { + return new RuntimeException("db failure", new SQLException("db failure", sqlState)); } -} + private static ConsumerRecord record(String topic, Object value) { + return new ConsumerRecord<>(topic, 0, 0L, "key", value); + } + + @Test + void benignDuplicateIsIdempotentSuccessAndNotDeadLettered() { + doThrow(dbError("23505")).when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", "{\"id\":\"1\"}")); + + // No dead-letter, no park, no audit: a duplicate is a silent idempotent success. + verify(kafkaTemplate, never()).send(anyString(), any()); + } + + @Test + void transientFailureIsRethrownForInPlaceRetryAndNeverDeadLettered() { + doThrow(dbError("08006")).when(persistService).persist(anyString(), anyString()); + + assertThrows(TransientPersistException.class, + () -> listener.onMessage(record("orig", "{\"id\":\"1\"}"))); + + // Crucial: a transient (DB-down) failure must NOT be diverted to DLQ or parking. + verify(kafkaTemplate, never()).send(anyString(), any()); + } + + @Test + void permanentFreshRecordIsDeadLettered() { + doThrow(dbError("23502")).when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", "{\"id\":\"1\"}")); + + verify(kafkaTemplate).send(eq(DLQ), any()); + verify(kafkaTemplate, never()).send(eq(PARK), any()); + } + + @Test + @SuppressWarnings("unchecked") + void permanentRecordFromDlqAtRetryCeilingIsParked() { + doThrow(dbError("23502")).when(persistService).persist(anyString(), anyString()); + + LinkedHashMap envelope = new LinkedHashMap<>(); + envelope.put("source", "orig"); + envelope.put("body", "{\"id\":\"1\"}"); + envelope.put("attempts", MAX_RETRIES); // budget already spent + + listener.onMessage(record(DLQ, envelope)); + + // Terminal park, and NOT re-queued to the DLQ -> no infinite loop (R4). + verify(kafkaTemplate).send(eq(PARK), any()); + verify(kafkaTemplate, never()).send(eq(DLQ), any()); + } + + @Test + void preSerialisedStringBodyIsNotDoubleEncoded() throws Exception { + doNothing().when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", "{\"id\":\"1\"}")); + + // A String body is passed through verbatim; re-serialising it would double-encode the JSON and + // PersistService would silently extract no rows. + verify(objectMapper, never()).writeValueAsString(any()); + verify(persistService).persist(eq("orig"), eq("{\"id\":\"1\"}")); + } + + @Test + void structuredObjectBodyIsSerialisedExactlyOnce() throws Exception { + LinkedHashMap body = new LinkedHashMap<>(); + body.put("id", "1"); + when(objectMapper.writeValueAsString(any())).thenReturn("{\"id\":\"1\"}"); + doNothing().when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", body)); + + verify(objectMapper).writeValueAsString(body); + verify(persistService).persist(eq("orig"), eq("{\"id\":\"1\"}")); + } +} diff --git a/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml b/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml index 341954f962e..2b6c7ce6bed 100644 --- a/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml +++ b/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-pg-txns isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, txn_status_msg, gateway, consumer_code, bill_id, product_info, user_uuid, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, receipt, additional_details, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, txn_status_msg, gateway, consumer_code, bill_id, product_info, user_uuid, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, receipt, additional_details, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; basePath: Transaction jsonMaps: @@ -73,7 +73,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; basePath: TransactionDump jsonMaps: diff --git a/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml b/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml index acba6cd82af..9665ebc494c 100644 --- a/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml +++ b/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml @@ -97,7 +97,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_wf_businessservice_v2(businessServiceSla, businessservice, business, tenantid, uuid, geturi, posturi, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_wf_businessservice_v2(businessServiceSla, businessservice, business, tenantid, uuid, geturi, posturi, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, businessservice) DO NOTHING; basePath: BusinessServices.* jsonMaps: - jsonPath: $.BusinessServices.*.businessServiceSla @@ -125,7 +125,7 @@ serviceMaps: - - query: INSERT INTO eg_wf_state_v2(seq, uuid, tenantid, businessserviceid, state,applicationStatus,sla,docuploadrequired, isstartstate, isterminatestate,isStateUpdatable, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (nextval('seq_eg_wf_state_v2'),? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_wf_state_v2(seq, uuid, tenantid, businessserviceid, state,applicationStatus,sla,docuploadrequired, isstartstate, isterminatestate,isStateUpdatable, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (nextval('seq_eg_wf_state_v2'),? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (state, businessserviceid) DO NOTHING; basePath: BusinessServices.*.states.* jsonMaps: - jsonPath: $.BusinessServices.*.states.*.uuid diff --git a/core-services/mdms-v2/src/main/resources/mdms-persister.yml b/core-services/mdms-v2/src/main/resources/mdms-persister.yml index 6771da75718..fda771dcf71 100644 --- a/core-services/mdms-v2/src/main/resources/mdms-persister.yml +++ b/core-services/mdms-v2/src/main/resources/mdms-persister.yml @@ -6,7 +6,7 @@ serviceMaps: fromTopic: save-mdms-schema-definition isTransaction: true queryMaps: - - query: INSERT INTO eg_mdms_schema_definition (id ,tenantid, code, description, definition, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_mdms_schema_definition (id ,tenantid, code, description, definition, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, code) DO NOTHING; basePath: $.SchemaDefinition jsonMaps: - jsonPath: $.SchemaDefinition.id @@ -37,7 +37,7 @@ serviceMaps: fromTopic: save-mdms-data isTransaction: true queryMaps: - - query: INSERT INTO eg_mdms_data (tenantid, uniqueidentifier, schemacode, data, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_mdms_data (tenantid, uniqueidentifier, schemacode, data, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, schemacode, uniqueidentifier) DO NOTHING; basePath: $.Mdms jsonMaps: - jsonPath: $.Mdms.tenantId diff --git a/core-services/service-request/src/main/resources/service-request-persister.yml b/core-services/service-request/src/main/resources/service-request-persister.yml index f3a6c07e3cb..b4bd7ea8068 100644 --- a/core-services/service-request/src/main/resources/service-request-persister.yml +++ b/core-services/service-request/src/main/resources/service-request-persister.yml @@ -7,7 +7,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_service_definition(id, tenantid, code, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails, clientid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_service_definition(id, tenantid, code, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails, clientid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, code) DO NOTHING; basePath: $.ServiceDefinition jsonMaps: - jsonPath: $.ServiceDefinition.id @@ -33,7 +33,7 @@ serviceMaps: - jsonPath: $.ServiceDefinition.clientId - - query: INSERT INTO eg_service_attribute_definition(id, referenceid, tenantid, code, datatype, "values", isactive, required, regex, "order", createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + - query: INSERT INTO eg_service_attribute_definition(id, referenceid, tenantid, code, datatype, "values", isactive, required, regex, "order", createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (code, referenceid) DO NOTHING; basePath: $.ServiceDefinition.attributes.* jsonMaps: - jsonPath: $.ServiceDefinition.attributes.*.id From 49339439d6aa56c08edfbc615e59d4cd09f9233b Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 6 Jul 2026 19:43:58 +0530 Subject: [PATCH 07/24] Revert persister-config edits in other modules (keep only egov-persister + boundary-service) 05f6889a53 also carried ON CONFLICT edits to persister YAMLs in unrelated modules (audit-service, egov-pg-service, egov-workflow-v2, mdms-v2, service-request). These are out of scope for the persister RCA reliability fix and the boundary-relationship enhancement, so they are reverted to their prior state. This branch now touches only egov-persister and boundary-service. --- .../configs/pgr-services-audit-persister.yml | 2 +- .../resources/configs/tradelicense-audit.yml | 18 +++++++++--------- .../main/resources/pg-service-persister.yml | 4 ++-- .../resources/egov-workflow-v2-persister.yml | 4 ++-- .../src/main/resources/mdms-persister.yml | 4 ++-- .../resources/service-request-persister.yml | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml b/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml index 51c31a6de9c..4ca25d9eec3 100644 --- a/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml +++ b/core-services/audit-service/src/main/resources/configs/pgr-services-audit-persister.yml @@ -14,7 +14,7 @@ serviceMaps: queryMaps: - - query: INSERT INTO eg_pgr_service_v2(id, tenantid, servicecode, servicerequestid, description, accountid, additionaldetails, applicationstatus, source, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, servicerequestid) DO NOTHING; + - query: INSERT INTO eg_pgr_service_v2(id, tenantid, servicecode, servicerequestid, description, accountid, additionaldetails, applicationstatus, source, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.service jsonMaps: - jsonPath: $.service.id diff --git a/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml b/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml index a083505b1b6..71ab06e57bb 100644 --- a/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml +++ b/core-services/audit-service/src/main/resources/configs/tradelicense-audit.yml @@ -13,7 +13,7 @@ serviceMaps: auditAttributeBasePath: $.Licenses.* queryMaps: - - query: INSERT INTO eg_tl_tradelicense( id, accountid,tenantid,tradeName, validfrom,validto,licensetype,applicationNumber, licenseNumber, oldlicensenumber, propertyid, oldpropertyid, applicationdate, commencementdate, financialyear, action, status, createdby, lastmodifiedby, createdtime, lastmodifiedtime, businessservice, applicationtype, workflowcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_tradelicense( id, accountid,tenantid,tradeName, validfrom,validto,licensetype,applicationNumber, licenseNumber, oldlicensenumber, propertyid, oldpropertyid, applicationdate, commencementdate, financialyear, action, status, createdby, lastmodifiedby, createdtime, lastmodifiedtime, businessservice, applicationtype, workflowcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.* jsonMaps: - jsonPath: $.Licenses.*.id @@ -65,7 +65,7 @@ serviceMaps: - jsonPath: $.Licenses.*.workflowCode - - query: INSERT INTO eg_tl_tradelicensedetail( id, surveyno, subownershipcategory, channel, additionaldetail, tradelicenseid,structureType,operationalArea,noOfEmployees,adhocExemption,adhocPenalty,adhocExemptionReason,adhocPenaltyReason, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_tradelicensedetail( id, surveyno, subownershipcategory, channel, additionaldetail, tradelicenseid,structureType,operationalArea,noOfEmployees,adhocExemption,adhocPenalty,adhocExemptionReason,adhocPenaltyReason, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.id @@ -106,7 +106,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_tradeunit( id, tenantid,active, tradetype, uom, uomvalue, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_tradeunit( id, tenantid,active, tradetype, uom, uomvalue, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.tradeUnits.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.tradeUnits.*.id @@ -133,7 +133,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_accessory( id, tenantid,active, accessoryCategory, uom, uomvalue, count, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_accessory( id, tenantid,active, accessoryCategory, uom, uomvalue, count, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.accessories.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.accessories.*.id @@ -161,7 +161,7 @@ serviceMaps: - jsonPath: $.Licenses[*][?({id} in @.tradeLicenseDetail.accessories[*].id)].auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_owner( id,tenantid,active,institutionid, tradelicensedetailid, isprimaryowner, ownertype, ownershippercentage, relationship, createdby,lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id, tradelicensedetailid) DO NOTHING; + - query: INSERT INTO eg_tl_owner( id,tenantid,active,institutionid, tradelicensedetailid, isprimaryowner, ownertype, ownershippercentage, relationship, createdby,lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.owners.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.owners.*.uuid @@ -192,7 +192,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_document_owner(id,tenantId,userid,active, tradeLicenseDetailId, documenttype, fileStoreId,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ? ,? ,?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_document_owner(id,tenantId,userid,active, tradeLicenseDetailId, documenttype, fileStoreId,documentuid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ? ,? ,?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.owners.*.documents.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.owners.*.documents.*.id @@ -221,7 +221,7 @@ serviceMaps: - - query: INSERT INTO eg_tl_applicationdocument( id, tenantid, active, documenttype, tradecategorydetail, filestoreid, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_applicationdocument( id, tenantid, active, documenttype, tradecategorydetail, filestoreid, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.applicationDocuments.* jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.applicationDocuments.*.id @@ -247,7 +247,7 @@ serviceMaps: - jsonPath: $.Licenses[*][?({id} in @.tradeLicenseDetail.applicationDocuments[*].id)].auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_address( id, tenantid, doorno,street,buildingName, latitude,longitude, addressid, addressnumber,locality, type, addressline1, addressline2, landmark, city, pincode, detail, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_address( id, tenantid, doorno,street,buildingName, latitude,longitude, addressid, addressnumber,locality, type, addressline1, addressline2, landmark, city, pincode, detail, tradelicensedetailid, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Licenses.*.tradeLicenseDetail.address jsonMaps: - jsonPath: $.Licenses.*.tradeLicenseDetail.address.id @@ -295,7 +295,7 @@ serviceMaps: - jsonPath: $.Licenses.*.auditDetails.lastModifiedTime - - query: INSERT INTO eg_tl_institution(tenantId,active,id,instituionName,contactNo,organisationRegistrationNo,address, tradelicensedetailid, name, type,designation, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING; + - query: INSERT INTO eg_tl_institution(tenantId,active,id,instituionName,contactNo,organisationRegistrationNo,address, tradelicensedetailid, name, type,designation, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ; basePath: $.Licenses.*.tradeLicenseDetail.institution jsonMaps: diff --git a/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml b/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml index 2b6c7ce6bed..341954f962e 100644 --- a/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml +++ b/core-services/egov-pg-service/src/main/resources/pg-service-persister.yml @@ -7,7 +7,7 @@ serviceMaps: fromTopic: save-pg-txns isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, txn_status_msg, gateway, consumer_code, bill_id, product_info, user_uuid, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, receipt, additional_details, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; + - query: INSERT INTO eg_pg_transactions (txn_id, txn_amount, txn_status, txn_status_msg, gateway, consumer_code, bill_id, product_info, user_uuid, user_name, mobile_number, email_id, name, user_tenant_id, tenant_id, gateway_txn_id, gateway_payment_mode, gateway_status_code, gateway_status_msg, receipt, additional_details, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?::numeric, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: Transaction jsonMaps: @@ -73,7 +73,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (txn_id) DO NOTHING; + - query: INSERT INTO eg_pg_transactions_dump (txn_id, txn_request, txn_response, created_by, created_time, last_modified_by, last_modified_time) VALUES (?, ?, ?, ?, ?, ?, ?); basePath: TransactionDump jsonMaps: diff --git a/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml b/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml index 9665ebc494c..acba6cd82af 100644 --- a/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml +++ b/core-services/egov-workflow-v2/src/main/resources/egov-workflow-v2-persister.yml @@ -97,7 +97,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_wf_businessservice_v2(businessServiceSla, businessservice, business, tenantid, uuid, geturi, posturi, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, businessservice) DO NOTHING; + - query: INSERT INTO eg_wf_businessservice_v2(businessServiceSla, businessservice, business, tenantid, uuid, geturi, posturi, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: BusinessServices.* jsonMaps: - jsonPath: $.BusinessServices.*.businessServiceSla @@ -125,7 +125,7 @@ serviceMaps: - - query: INSERT INTO eg_wf_state_v2(seq, uuid, tenantid, businessserviceid, state,applicationStatus,sla,docuploadrequired, isstartstate, isterminatestate,isStateUpdatable, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (nextval('seq_eg_wf_state_v2'),? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (state, businessserviceid) DO NOTHING; + - query: INSERT INTO eg_wf_state_v2(seq, uuid, tenantid, businessserviceid, state,applicationStatus,sla,docuploadrequired, isstartstate, isterminatestate,isStateUpdatable, createdby, createdtime, lastmodifiedby, lastmodifiedtime) VALUES (nextval('seq_eg_wf_state_v2'),? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: BusinessServices.*.states.* jsonMaps: - jsonPath: $.BusinessServices.*.states.*.uuid diff --git a/core-services/mdms-v2/src/main/resources/mdms-persister.yml b/core-services/mdms-v2/src/main/resources/mdms-persister.yml index fda771dcf71..6771da75718 100644 --- a/core-services/mdms-v2/src/main/resources/mdms-persister.yml +++ b/core-services/mdms-v2/src/main/resources/mdms-persister.yml @@ -6,7 +6,7 @@ serviceMaps: fromTopic: save-mdms-schema-definition isTransaction: true queryMaps: - - query: INSERT INTO eg_mdms_schema_definition (id ,tenantid, code, description, definition, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, code) DO NOTHING; + - query: INSERT INTO eg_mdms_schema_definition (id ,tenantid, code, description, definition, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.SchemaDefinition jsonMaps: - jsonPath: $.SchemaDefinition.id @@ -37,7 +37,7 @@ serviceMaps: fromTopic: save-mdms-data isTransaction: true queryMaps: - - query: INSERT INTO eg_mdms_data (tenantid, uniqueidentifier, schemacode, data, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, schemacode, uniqueidentifier) DO NOTHING; + - query: INSERT INTO eg_mdms_data (tenantid, uniqueidentifier, schemacode, data, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.Mdms jsonMaps: - jsonPath: $.Mdms.tenantId diff --git a/core-services/service-request/src/main/resources/service-request-persister.yml b/core-services/service-request/src/main/resources/service-request-persister.yml index b4bd7ea8068..f3a6c07e3cb 100644 --- a/core-services/service-request/src/main/resources/service-request-persister.yml +++ b/core-services/service-request/src/main/resources/service-request-persister.yml @@ -7,7 +7,7 @@ serviceMaps: isTransaction: true queryMaps: - - query: INSERT INTO eg_service_definition(id, tenantid, code, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails, clientid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantid, code) DO NOTHING; + - query: INSERT INTO eg_service_definition(id, tenantid, code, isactive, createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails, clientid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.ServiceDefinition jsonMaps: - jsonPath: $.ServiceDefinition.id @@ -33,7 +33,7 @@ serviceMaps: - jsonPath: $.ServiceDefinition.clientId - - query: INSERT INTO eg_service_attribute_definition(id, referenceid, tenantid, code, datatype, "values", isactive, required, regex, "order", createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (code, referenceid) DO NOTHING; + - query: INSERT INTO eg_service_attribute_definition(id, referenceid, tenantid, code, datatype, "values", isactive, required, regex, "order", createdby, lastmodifiedby, createdtime, lastmodifiedtime, additionaldetails) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); basePath: $.ServiceDefinition.attributes.* jsonMaps: - jsonPath: $.ServiceDefinition.attributes.*.id From a23744a586a9edd1e6574f7ea1e2d3dfd9f588dd Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Tue, 7 Jul 2026 19:43:34 +0530 Subject: [PATCH 08/24] JsonPath fixes --- .../persist/repository/PersistRepository.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java index d35956cb5ec..479c957bd5a 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/repository/PersistRepository.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONArray; import org.apache.commons.lang3.StringUtils; @@ -32,6 +34,13 @@ public class PersistRepository { @Autowired private ObjectMapper objectMapper; + private static final Configuration LENIENT_VALUE_READ = + Configuration.defaultConfiguration().addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL); + + private T readValue(Object jsonObj, String jsonPath) { + return JsonPath.using(LENIENT_VALUE_READ).parse(jsonObj).read(jsonPath); + } + public void persist(String query, List rows) { @@ -109,7 +118,7 @@ public List getRows(List jsonMaps, Object jsonObj, String bas if (jsonPath.contains("{")) { String attribute = jsonPath.substring(jsonPath.indexOf("{") + 1, jsonPath.indexOf("}")); jsonPath = jsonPath.replace("{".concat(attribute).concat("}"), "\"" + rawDataRecord.get(attribute).toString() + "\""); - JSONArray jsonArray = JsonPath.read(jsonObj, jsonPath); + JSONArray jsonArray = readValue(jsonObj, jsonPath); row.add(jsonArray.get(0)); continue; @@ -125,7 +134,7 @@ else if (dbType.equals(TypeEnum.LONG)) } else if ((type.equals(TypeEnum.ARRAY)) && dbType.equals(TypeEnum.STRING)) { - List list1 = JsonPath.read(jsonObj, jsonPath); + List list1 = readValue(jsonObj, jsonPath); if (CollectionUtils.isEmpty(list1)) { value = null; } else { @@ -140,7 +149,7 @@ else if (jsonPath.contains("*.")) { } else if (!(type.equals(TypeEnum.CURRENTDATE) || jsonPath.startsWith("default"))) { - value = JsonPath.read(jsonObj, jsonPath); + value = readValue(jsonObj, jsonPath); } if (jsonPath.startsWith("default")) From c1abc8b760cb92fa9dd08c610325603bfe6386a4 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Thu, 9 Jul 2026 12:37:57 +0530 Subject: [PATCH 09/24] Per record poison data handling --- .../consumer/PersisterBatchListner.java | 31 +++- .../consumer/PersisterMessageListener.java | 62 +++++++- ...sisterBatchListnerRecordIsolationTest.java | 146 ++++++++++++++++++ .../PersisterMessageListenerTest.java | 103 ++++++++++++ 4 files changed, 332 insertions(+), 10 deletions(-) create mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchListnerRecordIsolationTest.java diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java index 28b07056ce0..b11ab76071c 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterBatchListner.java @@ -175,14 +175,37 @@ private int[] processSingleTopic(String topic, List messages) { persisted.add(message); } catch (Exception ex) { DbExceptionClassifier.Kind kind = DbExceptionClassifier.classify(ex); - if (kind == DbExceptionClassifier.Kind.BENIGN) { - persisted.add(message); // already present (duplicate) -> idempotent success - } else if (kind == DbExceptionClassifier.Kind.TRANSIENT) { + if (kind == DbExceptionClassifier.Kind.TRANSIENT) { // DB went down partway through isolation: abort and retry the whole batch rather // than parking the remaining good records as if they were poison. throw new TransientPersistException("Transient DB failure isolating record for topic " + topic, ex); + } + // A bulk producer publishes a whole list as ONE message, so isolate WITHIN the + // message (R1 at record granularity): good sibling rows must not share a poison + // row's dead-letter, and a BENIGN duplicate must not absorb not-yet-persisted + // siblings (the array insert aborts on the duplicate before reaching them). + List records = RecordSplitter.split(message); + if (records != null) { + for (String record : records) { + try { + persistService.persist(topic, Collections.singletonList(record)); + persisted.add(record); + } catch (Exception rex) { + DbExceptionClassifier.Kind recordKind = DbExceptionClassifier.classify(rex); + if (recordKind == DbExceptionClassifier.Kind.BENIGN) { + persisted.add(record); // already present -> idempotent success + } else if (recordKind == DbExceptionClassifier.Kind.TRANSIENT) { + throw new TransientPersistException("Transient DB failure isolating record for topic " + topic, rex); + } else { + sendToDlq(topic, record, 0, rex); // only the offending record + failed++; + } + } + } + } else if (kind == DbExceptionClassifier.Kind.BENIGN) { + persisted.add(message); // single already-present record -> idempotent success } else { - sendToDlq(topic, message, 0, ex); // isolate only the offending (bad-data) record + sendToDlq(topic, message, 0, ex); // unsplittable payload: message-level isolation failed++; } } diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java index 782003dd6f7..ecda60099c4 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/PersisterMessageListener.java @@ -57,6 +57,7 @@ public void onMessage(ConsumerRecord data) { String topic = data.topic(); boolean fromDlq = Objects.equals(topic, deadLetterTopic); Object body = null; + String json = null; int attempts = 0; try { @@ -76,16 +77,11 @@ public void onMessage(ConsumerRecord data) { // Batch-originated envelopes carry body as a pre-serialized JSON String; the single path // carries a structured object. Re-serialize ONLY the latter, otherwise the String would be // double-encoded and PersistService would silently extract no rows. - String json = (body instanceof String) ? (String) body : objectMapper.writeValueAsString(body); + json = (body instanceof String) ? (String) body : objectMapper.writeValueAsString(body); persistService.persist(topic, json); log.info("Message from topic {} persisted in {} ms.", topic, System.currentTimeMillis() - startTime); } catch (Exception e) { DbExceptionClassifier.Kind kind = DbExceptionClassifier.classify(e); - if (kind == DbExceptionClassifier.Kind.BENIGN) { - // Row already present (redelivery / DLQ replay) -> idempotent success; never dead-letter it. - log.info("Record for topic {} already present (duplicate) - idempotent success.", topic); - return; - } if (kind == DbExceptionClassifier.Kind.TRANSIENT) { // DB/infra momentarily unavailable. The record is GOOD, so it must not be dead-lettered or // parked: rethrow so the container error handler retries it in place (offset NOT committed) @@ -95,6 +91,21 @@ public void onMessage(ConsumerRecord data) { log.warn("Transient failure persisting record from topic {} - retrying in place (not dead-lettered)", topic, e); throw new TransientPersistException("Transient DB failure persisting topic " + topic, e); } + // Bulk producers publish a whole list as ONE message, and the whole-message transaction has + // rolled back - so a failure here must be isolated to the offending RECORD(s), never shared + // by the good siblings (R1 at record granularity). This also covers BENIGN on a mixed batch: + // a duplicate row aborts the array insert with unique_violation, and treating that as a + // message-level idempotent success would silently drop the not-yet-persisted siblings. + List records = RecordSplitter.split(json); + if (records != null) { + persistRecordsIndividually(topic, records, fromDlq, attempts); + return; + } + if (kind == DbExceptionClassifier.Kind.BENIGN) { + // Single row already present (redelivery / DLQ replay) -> idempotent success; never dead-letter it. + log.info("Record for topic {} already present (duplicate) - idempotent success.", topic); + return; + } // PERMANENT (bad data): bounded dead-letter reprocessing, then terminal parking (R3 + R4). if (!fromDlq) { // First failure -> dead-letter for bounded reprocessing. @@ -123,6 +134,45 @@ public void onMessage(ConsumerRecord data) { } } + /** + * Record-level isolation for a failed multi-record message: each record is persisted in its own + * transaction; duplicates are idempotent successes; only genuinely bad records continue on the + * bounded DLQ -> park path (with the message's remaining retry budget). A transient failure + * mid-sweep aborts and rethrows so the ORIGINAL message retries in place once the DB recovers - + * records persisted before the abort then come back BENIGN, so the sweep converges. + */ + private void persistRecordsIndividually(String topic, List records, boolean fromDlq, int attempts) { + int persisted = 0; + int duplicates = 0; + int deadLettered = 0; + int parked = 0; + for (String record : records) { + try { + persistService.persist(topic, record); + persisted++; + } catch (Exception e) { + DbExceptionClassifier.Kind kind = DbExceptionClassifier.classify(e); + if (kind == DbExceptionClassifier.Kind.BENIGN) { + duplicates++; + } else if (kind == DbExceptionClassifier.Kind.TRANSIENT) { + log.warn("Transient failure during record-level isolation for topic {} - retrying whole message in place", topic, e); + throw new TransientPersistException("Transient DB failure isolating record for topic " + topic, e); + } else if (!fromDlq) { + sendToDlq(topic, record, 1, e); + deadLettered++; + } else if (attempts < maxRetries) { + sendToDlq(topic, record, attempts + 1, e); + deadLettered++; + } else { + sendToParking(topic, record, attempts, e); + parked++; + } + } + } + log.warn("Record-level isolation for topic {}: {} record(s) -> {} persisted, {} duplicate(s), {} dead-lettered, {} parked", + topic, records.size(), persisted, duplicates, deadLettered, parked); + } + /** Awaited dead-letter publish; rethrows on failure so the record is not silently dropped. */ private void sendToDlq(String topic, Object body, int attempts, Exception cause) { try { diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchListnerRecordIsolationTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchListnerRecordIsolationTest.java new file mode 100644 index 00000000000..1ccc9f94e35 --- /dev/null +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterBatchListnerRecordIsolationTest.java @@ -0,0 +1,146 @@ +package org.egov.infra.persist.consumer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.egov.infra.persist.service.PersistService; +import org.egov.tracer.kafka.CustomKafkaTemplate; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Record-level isolation in the batch listener: a bulk producer publishes a whole list as ONE + * message, so when that message fails permanently the failure must be narrowed to the offending + * record(s) - good siblings persist, duplicates are idempotent successes, and a transient failure + * aborts the sweep for in-place retry. + */ +class PersisterBatchListnerRecordIsolationTest { + + private static final String DLQ = "dlq"; + private static final String TOPIC = "save-entity-topic"; + + private PersistService persistService; + private CustomKafkaTemplate kafkaTemplate; + private PersisterBatchListner listener; + + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() throws Exception { + persistService = mock(PersistService.class); + kafkaTemplate = mock(CustomKafkaTemplate.class); + ObjectMapper objectMapper = mock(ObjectMapper.class); + // Pass raw String values through unchanged (the production mapper re-serialises the + // deserialised payload; our test payloads are already JSON strings). + when(objectMapper.writeValueAsString(any())).thenAnswer(inv -> inv.getArgument(0).toString()); + + listener = new PersisterBatchListner(); + ReflectionTestUtils.setField(listener, "objectMapper", objectMapper); + ReflectionTestUtils.setField(listener, "persistService", persistService); + ReflectionTestUtils.setField(listener, "kafkaTemplate", kafkaTemplate); + ReflectionTestUtils.setField(listener, "persistAuditKafkaTopic", "audit-create"); + ReflectionTestUtils.setField(listener, "auditGenerateKafkaTopic", "process-audit-records"); + ReflectionTestUtils.setField(listener, "deadLetterTopic", DLQ); + ReflectionTestUtils.setField(listener, "topicProcessorThreadPoolSize", 1); + listener.init(); + } + + /** Wrap a SQLState in the kind of unchecked exception JdbcTemplate would surface. */ + private static RuntimeException dbError(String sqlState) { + return new RuntimeException("db failure", new SQLException("db failure", sqlState)); + } + + private static List> poll(String value) { + List> records = new ArrayList<>(); + records.add(new ConsumerRecord<>(TOPIC, 0, 0L, "key", value)); + return records; + } + + /** Route persist failures by payload content: "bad" -> permanent, "slow" -> transient, "dup" -> benign. */ + private void routeByContent() { + doAnswer(inv -> { + List jsons = inv.getArgument(1); + String joined = String.join("|", jsons); + if (joined.contains("bad")) { + throw dbError("23502"); // not_null_violation + } + if (joined.contains("slow")) { + throw dbError("08006"); // connection_failure + } + if (joined.contains("dup")) { + throw dbError("23505"); // unique_violation + } + return null; + }).when(persistService).persist(anyString(), anyList()); + } + + /** TC063: [good, poison, good] in ONE message -> goods persist, ONLY the poison is dead-lettered. */ + @Test + void poisonRecordInsideBulkMessageIsIsolatedAndGoodSiblingsPersist() { + routeByContent(); + + listener.onMessage(poll("[{\"id\":\"g1\"},{\"id\":\"bad\"},{\"id\":\"g2\"}]")); + + // After whole-message failure, each record is persisted individually... + verify(persistService).persist(TOPIC, Collections.singletonList("[{\"id\":\"g1\"}]")); + verify(persistService).persist(TOPIC, Collections.singletonList("[{\"id\":\"bad\"}]")); + verify(persistService).persist(TOPIC, Collections.singletonList("[{\"id\":\"g2\"}]")); + // ...and only the poison record reaches the dead-letter topic. + verify(kafkaTemplate).send(eq(DLQ), argThat(payload -> + ((Map) payload).get("body").toString().contains("bad") + && !((Map) payload).get("body").toString().contains("g1"))); + } + + /** A duplicate inside the array must not absorb its not-yet-persisted siblings (silent-loss hole). */ + @Test + void benignDuplicateInsideBulkMessageDoesNotSilentlyDropSiblings() { + routeByContent(); + + listener.onMessage(poll("[{\"id\":\"new1\"},{\"id\":\"dup\"},{\"id\":\"new2\"}]")); + + verify(persistService).persist(TOPIC, Collections.singletonList("[{\"id\":\"new1\"}]")); + verify(persistService).persist(TOPIC, Collections.singletonList("[{\"id\":\"new2\"}]")); + verify(kafkaTemplate, never()).send(eq(DLQ), any()); + } + + /** Transient mid-sweep -> abort and rethrow for in-place retry; nothing dead-lettered. */ + @Test + void transientFailureDuringIsolationRethrowsForInPlaceRetry() { + doAnswer(inv -> { + List jsons = inv.getArgument(1); + String joined = String.join("|", jsons); + if (joined.contains("bad") && joined.contains("slow")) { + throw dbError("23502"); // full message fails permanent -> triggers isolation + } + if (joined.contains("slow")) { + throw dbError("08006"); // DB dies while isolating this record + } + if (joined.contains("bad")) { + throw dbError("23502"); + } + return null; + }).when(persistService).persist(anyString(), anyList()); + + assertThrows(TransientPersistException.class, + () -> listener.onMessage(poll("[{\"id\":\"slow\"},{\"id\":\"bad\"}]"))); + + verify(kafkaTemplate, never()).send(eq(DLQ), any()); + } +} diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java index 34ea3d08728..838ce048810 100644 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/PersisterMessageListenerTest.java @@ -14,7 +14,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -113,6 +115,107 @@ void permanentRecordFromDlqAtRetryCeilingIsParked() { verify(kafkaTemplate, never()).send(eq(DLQ), any()); } + @Test + void permanentPoisonInBulkArrayIsIsolatedAndGoodSiblingsPersist() { + doAnswer(inv -> { + String json = inv.getArgument(1); + if (json.contains("bad")) { + throw dbError("23502"); // not_null_violation - fails the whole array, then only [bad] + } + return null; + }).when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", "[{\"id\":\"g1\"},{\"id\":\"bad\"},{\"id\":\"g2\"}]")); + + // Whole array first, then each of the 3 records individually. + verify(persistService).persist("orig", "[{\"id\":\"g1\"},{\"id\":\"bad\"},{\"id\":\"g2\"}]"); + verify(persistService).persist("orig", "[{\"id\":\"g1\"}]"); + verify(persistService).persist("orig", "[{\"id\":\"bad\"}]"); + verify(persistService).persist("orig", "[{\"id\":\"g2\"}]"); + // Exactly ONE dead-letter: the poison record alone; nothing parked. + verify(kafkaTemplate).send(eq(DLQ), argThat(payload -> + ((java.util.Map) payload).get("body").toString().contains("bad") + && !((java.util.Map) payload).get("body").toString().contains("g1"))); + verify(kafkaTemplate, never()).send(eq(PARK), any()); + } + + /** + * A duplicate row inside a bulk array aborts the whole-array insert with unique_violation + * (BENIGN). Treating that as a message-level idempotent success would silently drop the + * not-yet-persisted siblings - they must be persisted individually instead. + */ + @Test + void benignDuplicateInBulkArrayDoesNotSilentlyDropSiblings() { + doAnswer(inv -> { + String json = inv.getArgument(1); + if (json.contains("dup")) { + throw dbError("23505"); // unique_violation on the duplicate row + } + return null; + }).when(persistService).persist(anyString(), anyString()); + + listener.onMessage(record("orig", "[{\"id\":\"new1\"},{\"id\":\"dup\"},{\"id\":\"new2\"}]")); + + // The two new records are persisted individually; the duplicate is an idempotent success. + verify(persistService).persist("orig", "[{\"id\":\"new1\"}]"); + verify(persistService).persist("orig", "[{\"id\":\"new2\"}]"); + // Nothing is dead-lettered or parked for a duplicate. + verify(kafkaTemplate, never()).send(eq(DLQ), any()); + verify(kafkaTemplate, never()).send(eq(PARK), any()); + } + + /** A transient failure mid-isolation must abort the sweep and retry the ORIGINAL message in place. */ + @Test + void transientFailureDuringIsolationRethrowsAndNothingIsDeadLettered() { + doAnswer(inv -> { + String json = inv.getArgument(1); + if (json.contains("bad") && json.contains("slow")) { + throw dbError("23502"); // full array fails permanent -> triggers isolation + } + if (json.contains("slow")) { + throw dbError("08006"); // DB dies when this record is retried individually + } + if (json.contains("bad")) { + throw dbError("23502"); + } + return null; + }).when(persistService).persist(anyString(), anyString()); + + assertThrows(TransientPersistException.class, + () -> listener.onMessage(record("orig", "[{\"id\":\"slow\"},{\"id\":\"bad\"}]"))); + + // The outage must not burn the retry budget of any record. + verify(kafkaTemplate, never()).send(anyString(), any()); + } + + /** + * A multi-record body replayed from the DLQ at the retry ceiling: only the still-failing record + * is parked; its sibling persists (no wholesale parking of good data). + */ + @Test + @SuppressWarnings("unchecked") + void multiRecordDlqBodyAtCeilingParksOnlyTheOffendingRecord() { + doAnswer(inv -> { + String json = inv.getArgument(1); + if (json.contains("bad")) { + throw dbError("23502"); + } + return null; + }).when(persistService).persist(anyString(), anyString()); + + LinkedHashMap envelope = new LinkedHashMap<>(); + envelope.put("source", "orig"); + envelope.put("body", "[{\"id\":\"good\"},{\"id\":\"bad\"}]"); + envelope.put("attempts", MAX_RETRIES); // budget already spent + + listener.onMessage(record(DLQ, envelope)); + + verify(persistService).persist("orig", "[{\"id\":\"good\"}]"); + verify(kafkaTemplate).send(eq(PARK), argThat(payload -> + ((java.util.Map) payload).get("body").toString().contains("bad"))); + verify(kafkaTemplate, never()).send(eq(DLQ), any()); + } + @Test void preSerialisedStringBodyIsNotDoubleEncoded() throws Exception { doNothing().when(persistService).persist(anyString(), anyString()); From 58d12257732f484ed249de1533f87e3796762f4b Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Thu, 9 Jul 2026 12:51:26 +0530 Subject: [PATCH 10/24] Per record poison data handling --- .../persist/consumer/RecordSplitter.java | 54 +++++++++++++++++++ .../persist/consumer/RecordSplitterTest.java | 44 +++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java create mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java new file mode 100644 index 00000000000..461998b2e5e --- /dev/null +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java @@ -0,0 +1,54 @@ +package org.egov.infra.persist.consumer; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + +import java.util.ArrayList; +import java.util.List; + +/** + * Splits a multi-record bulk message into standalone single-record messages so a permanent failure + * can be isolated to the offending record(s) instead of failing every record that happened to share + * the same Kafka message (R1 at record granularity, not just message granularity). + * + *

Bulk producers publish a whole validated list as ONE message whose payload is a bare JSON + * array, and the persister maps such topics with array base paths ({@code $.*}), so a + * single-element array is processed identically to the full array. Only bare arrays with more than + * one element are split; object-shaped payloads (whose base paths may not be per-element) keep + * message-level handling.

+ */ +final class RecordSplitter { + + /** Used for structural parse/re-emit only, never for domain (de)serialisation. */ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private RecordSplitter() { + } + + /** + * @return one single-element-array JSON string per record, or null when the payload is not a + * multi-record bare array (not JSON / an object / an array of 0..1 elements) and the + * caller must keep message-level handling. + */ + static List split(String json) { + if (json == null) { + return null; + } + try { + JsonNode root = MAPPER.readTree(json); + if (root == null || !root.isArray() || root.size() <= 1) { + return null; + } + List records = new ArrayList<>(root.size()); + for (JsonNode element : root) { + ArrayNode single = MAPPER.createArrayNode(); + single.add(element); + records.add(MAPPER.writeValueAsString(single)); + } + return records; + } catch (Exception e) { + return null; + } + } +} diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java new file mode 100644 index 00000000000..361570a3cea --- /dev/null +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java @@ -0,0 +1,44 @@ +package org.egov.infra.persist.consumer; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class RecordSplitterTest { + + @Test + void multiRecordArraySplitsIntoSingleElementArraysPreservingContentAndOrder() { + List records = RecordSplitter.split("[{\"id\":\"a\"},{\"id\":\"b\"},{\"id\":\"c\"}]"); + + assertEquals(3, records.size()); + assertEquals("[{\"id\":\"a\"}]", records.get(0)); + assertEquals("[{\"id\":\"b\"}]", records.get(1)); + assertEquals("[{\"id\":\"c\"}]", records.get(2)); + } + + @Test + void singleElementArrayIsNotSplit() { + assertNull(RecordSplitter.split("[{\"id\":\"a\"}]")); + } + + @Test + void emptyArrayIsNotSplit() { + assertNull(RecordSplitter.split("[]")); + } + + @Test + void objectPayloadIsNotSplit() { + // Object-shaped payloads (e.g. {"RequestInfo":...,"Entity":{...}}) may not map per-element - + // they must keep message-level handling. + assertNull(RecordSplitter.split("{\"RequestInfo\":{},\"records\":[{\"id\":\"a\"},{\"id\":\"b\"}]}")); + } + + @Test + void unparseableOrNullPayloadIsNotSplit() { + assertNull(RecordSplitter.split("not-json")); + assertNull(RecordSplitter.split(null)); + } +} From d0f2bc4b126cf89c30e512d41ef7772b60da9928 Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Thu, 9 Jul 2026 15:43:08 +0530 Subject: [PATCH 11/24] propagate correlationId + tenantId across Kafka (tracer) --- core-services/libraries/tracer/pom.xml | 2 +- .../tracer/config/TracerConfiguration.java | 10 +++ .../KafkaTemplateLoggingInterceptors.java | 32 ++------ .../tracer/kafka/MdcRecordInterceptor.java | 25 ++++++ .../egov/tracer/kafka/TracerKafkaMdcUtil.java | 82 +++++++++++++++++++ .../src/main/resources/tracer.properties | 3 + 6 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/MdcRecordInterceptor.java create mode 100644 core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/TracerKafkaMdcUtil.java diff --git a/core-services/libraries/tracer/pom.xml b/core-services/libraries/tracer/pom.xml index aeb99a3f68f..46c958aaafa 100644 --- a/core-services/libraries/tracer/pom.xml +++ b/core-services/libraries/tracer/pom.xml @@ -10,7 +10,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT tracer Assist in tracing http and message queue flows diff --git a/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java b/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java index 889f8a652bf..19797784caf 100644 --- a/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java +++ b/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java @@ -2,6 +2,7 @@ import org.egov.tracer.http.RestTemplateLoggingInterceptor; import org.egov.tracer.http.filters.TracerFilter; +import org.egov.tracer.kafka.MdcRecordInterceptor; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.client.RestTemplateBuilder; @@ -10,6 +11,7 @@ import org.springframework.core.env.Environment; import org.springframework.http.client.BufferingClientHttpRequestFactory; import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.kafka.listener.RecordInterceptor; import org.springframework.web.client.RestTemplate; import java.util.Collections; @@ -38,6 +40,14 @@ public RestTemplate logAwareRestTemplate(RestTemplateBuilder builder, TracerProp .build(); } + // MDC rebuild + cleanup from Kafka headers. Boot auto-applies to the autoconfig record-listener + // factory; custom-factory or batch-listener services must wire it manually. + @Bean + @ConditionalOnProperty(name = "tracer.kafka.mdc.enabled", havingValue = "true", matchIfMissing = true) + public RecordInterceptor mdcRecordInterceptor() { + return new MdcRecordInterceptor<>(); + } + /** * Configure tracer filter with order one * diff --git a/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/KafkaTemplateLoggingInterceptors.java b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/KafkaTemplateLoggingInterceptors.java index cb37f29865a..509fa169518 100644 --- a/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/KafkaTemplateLoggingInterceptors.java +++ b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/KafkaTemplateLoggingInterceptors.java @@ -11,14 +11,12 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.TopicPartition; -import org.slf4j.MDC; import org.springframework.util.ObjectUtils; import java.util.Map; import static java.util.Objects.isNull; import static org.egov.tracer.constants.TracerConstants.*; -import static org.springframework.util.StringUtils.isEmpty; @Slf4j public class KafkaTemplateLoggingInterceptors implements ConsumerInterceptor, ProducerInterceptor { @@ -44,10 +42,9 @@ public KafkaTemplateLoggingInterceptors() { public ConsumerRecords onConsume(ConsumerRecords consumerRecords) { for (ConsumerRecord consumerRecord : consumerRecords) { final String keyAsString = ObjectUtils.nullSafeToString(consumerRecord.key()); - String correlationId = getCorrelationIdFromBody(consumerRecord.value()); - if (!isEmpty(correlationId)) - MDC.put(CORRELATION_ID_MDC, correlationId); + // MDC from headers (body fallback for correlationId) + TracerKafkaMdcUtil.applyMdcFromRecord(consumerRecord); if (log.isDebugEnabled()) { final String bodyAsJsonString = getMessageBodyAsJsonString(consumerRecord.value()); @@ -69,6 +66,9 @@ public void onCommit(Map map) { public ProducerRecord onSend(ProducerRecord producerRecord) { final String keyAsString = ObjectUtils.nullSafeToString(producerRecord.key()); + // stamp correlationId + tenantId headers from MDC + TracerKafkaMdcUtil.stampHeadersFromMdc(producerRecord); + if (log.isDebugEnabled()) { final String bodyAsJsonString = getMessageBodyAsJsonString(producerRecord.value()); log.debug(SEND_SUCCESS_MESSAGE_WITH_BODY, producerRecord.topic(), producerRecord.partition(), bodyAsJsonString, @@ -107,26 +107,4 @@ private String getMessageBodyAsJsonString(Object value) { } } - @SuppressWarnings("unchecked") - private String getCorrelationIdFromBody(Object value) { - String correlationId = null; - try { - Map requestMap = objectMapper.convertValue(value, Map.class); - - Object requestInfo = requestMap.containsKey(REQUEST_INFO_FIELD_NAME_IN_JAVA_CLASS_CASE) ? requestMap.get - (REQUEST_INFO_FIELD_NAME_IN_JAVA_CLASS_CASE) : requestMap.get(REQUEST_INFO_IN_CAMEL_CASE); - - if (isNull(requestInfo)) - return null; - else { - if (requestInfo instanceof Map) { - correlationId = (String) ((Map) requestInfo).get(CORRELATION_ID_FIELD_NAME); - } - } - } catch (Exception ignored) { - } - - return correlationId; - } - } diff --git a/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/MdcRecordInterceptor.java b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/MdcRecordInterceptor.java new file mode 100644 index 00000000000..7008a25ddb6 --- /dev/null +++ b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/MdcRecordInterceptor.java @@ -0,0 +1,25 @@ +package org.egov.tracer.kafka; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.kafka.listener.RecordInterceptor; + +/** Rebuilds MDC from record headers before each record and clears it after (per-record, with cleanup). */ +public class MdcRecordInterceptor implements RecordInterceptor { + + @Override + public ConsumerRecord intercept(ConsumerRecord record, Consumer consumer) { + TracerKafkaMdcUtil.applyMdcFromRecord(record); + return record; + } + + @Override + public void success(ConsumerRecord record, Consumer consumer) { + TracerKafkaMdcUtil.clearMdc(); + } + + @Override + public void failure(ConsumerRecord record, Exception exception, Consumer consumer) { + TracerKafkaMdcUtil.clearMdc(); + } +} diff --git a/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/TracerKafkaMdcUtil.java b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/TracerKafkaMdcUtil.java new file mode 100644 index 00000000000..c3c8a24116f --- /dev/null +++ b/core-services/libraries/tracer/src/main/java/org/egov/tracer/kafka/TracerKafkaMdcUtil.java @@ -0,0 +1,82 @@ +package org.egov.tracer.kafka; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.Header; +import org.slf4j.MDC; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static java.util.Objects.isNull; +import static org.egov.tracer.constants.TracerConstants.*; +import static org.springframework.util.StringUtils.isEmpty; + +/** Carries correlationId + tenantId across Kafka via record headers, and rebuilds MDC on consume. */ +@Slf4j +public final class TracerKafkaMdcUtil { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private TracerKafkaMdcUtil() { + } + + /** Copy correlationId + tenantId from MDC into record headers (only if present and not already set). */ + public static void stampHeadersFromMdc(ProducerRecord producerRecord) { + addHeaderIfAbsent(producerRecord, CORRELATION_ID_HEADER, MDC.get(CORRELATION_ID_MDC)); + addHeaderIfAbsent(producerRecord, TENANT_ID_HEADER, MDC.get(TENANTID_MDC)); + } + + /** Set MDC from record headers (correlationId falls back to body); removes the key when absent. */ + public static void applyMdcFromRecord(ConsumerRecord consumerRecord) { + String correlationId = headerValue(consumerRecord, CORRELATION_ID_HEADER); + if (isEmpty(correlationId)) + correlationId = correlationIdFromBody(consumerRecord.value()); + setOrRemove(CORRELATION_ID_MDC, correlationId); + + setOrRemove(TENANTID_MDC, headerValue(consumerRecord, TENANT_ID_HEADER)); + } + + /** Remove the tracing keys from MDC (call after each record; listener threads are reused). */ + public static void clearMdc() { + MDC.remove(CORRELATION_ID_MDC); + MDC.remove(TENANTID_MDC); + } + + private static void addHeaderIfAbsent(ProducerRecord producerRecord, String headerName, String value) { + if (!isEmpty(value) && producerRecord.headers().lastHeader(headerName) == null) + producerRecord.headers().add(headerName, value.getBytes(StandardCharsets.UTF_8)); + } + + private static String headerValue(ConsumerRecord consumerRecord, String headerName) { + Header header = consumerRecord.headers().lastHeader(headerName); + if (isNull(header) || isNull(header.value())) + return null; + return new String(header.value(), StandardCharsets.UTF_8); + } + + private static void setOrRemove(String key, String value) { + if (isEmpty(value)) + MDC.remove(key); + else + MDC.put(key, value); + } + + @SuppressWarnings("unchecked") + private static String correlationIdFromBody(Object value) { + try { + Map requestMap = objectMapper.convertValue(value, Map.class); + Object requestInfo = requestMap.containsKey(REQUEST_INFO_FIELD_NAME_IN_JAVA_CLASS_CASE) + ? requestMap.get(REQUEST_INFO_FIELD_NAME_IN_JAVA_CLASS_CASE) + : requestMap.get(REQUEST_INFO_IN_CAMEL_CASE); + if (isNull(requestInfo)) + return null; + if (requestInfo instanceof Map) + return (String) ((Map) requestInfo).get(CORRELATION_ID_FIELD_NAME); + } catch (Exception ignored) { + } + return null; + } +} diff --git a/core-services/libraries/tracer/src/main/resources/tracer.properties b/core-services/libraries/tracer/src/main/resources/tracer.properties index 84a88389fea..8aea2169bb6 100644 --- a/core-services/libraries/tracer/src/main/resources/tracer.properties +++ b/core-services/libraries/tracer/src/main/resources/tracer.properties @@ -15,6 +15,9 @@ tracer.filterSkipPattern=/api-docs.*|/autoconfig|/configprops|/dump|/health|/inf #Logging config spring.kafka.properties.interceptor.classes=org.egov.tracer.kafka.KafkaTemplateLoggingInterceptors +# per-record MDC rebuild + cleanup on Kafka consume (default on) +tracer.kafka.mdc.enabled=true + # Actuator Configs endpoints.enabled=false endpoints.health.enabled=true From bdc9ff8f2c36f6a5a950edc9fb2b748aeade6242 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 13 Jul 2026 14:51:00 +0530 Subject: [PATCH 12/24] Bulk kafka handling --- .../src/main/java/digit/kafka/Producer.java | 14 ++ .../BoundaryRelationshipRepository.java | 15 ++- .../BoundaryRelationshipRepositoryImpl.java | 77 ++++++++--- .../service/BoundaryRelationshipService.java | 16 +-- .../BulkBoundaryRelationshipRequestDTO.java | 42 ++++++ .../src/main/resources/boundary-persister.yml | 24 ++-- ...aryRelationshipRepositoryImplBulkTest.java | 125 ++++++++++++++++++ 7 files changed, 269 insertions(+), 44 deletions(-) create mode 100644 core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java create mode 100644 core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java diff --git a/core-services/boundary-service/src/main/java/digit/kafka/Producer.java b/core-services/boundary-service/src/main/java/digit/kafka/Producer.java index 542f4f686c0..90e03619b14 100644 --- a/core-services/boundary-service/src/main/java/digit/kafka/Producer.java +++ b/core-services/boundary-service/src/main/java/digit/kafka/Producer.java @@ -17,4 +17,18 @@ public class Producer { public void push(String topic, Object value) { kafkaTemplate.send(topic, value); } + + /** + * Keyed publish: routes the message to a partition by {@code key} so all messages sharing a key + * are ordered on the same partition. Used by the bulk path to key a batch by its parent code, so + * batches of siblings under the same parent keep a deterministic per-parent order. A null key + * falls back to the keyless (default-partitioner) behaviour of {@link #push(String, Object)}. + */ + public void push(String topic, String key, Object value) { + if (key == null) { + kafkaTemplate.send(topic, value); + } else { + kafkaTemplate.send(topic, key, value); + } + } } diff --git a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java index e44f1639edd..bfd656cdf9d 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java +++ b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java @@ -14,15 +14,16 @@ public interface BoundaryRelationshipRepository { /** * Persists the given validated and enriched boundary relationships through egov-persister (no direct - * DB write): each record is published, one message per record, to the same save-boundary-relationship - * topic the single {@link #create} uses. The publish is blocking (it returns once the broker has - * accepted each record); egov-persister then writes each via an idempotent - * INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING, so redelivery is a safe no-op and - * one un-insertable record never fails the others. Optionally listing that topic in the persister's - * persister.batch.topics lets it aggregate a poll into one multi-row insert for throughput. + * DB write): the WHOLE list is published as ONE message to the same save-boundary-relationship topic + * the single {@link #create} uses, carrying the relationships as an array under the same + * {@code BoundaryRelationship} key. The publish is blocking (it returns once the broker has accepted + * the message); egov-persister reads the array (array base path) and writes it as a single + * batchUpdate through the idempotent INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING, + * so redelivery is a safe no-op and duplicates are skipped without aborting the batch. Batching is + * WITHIN the one message, so it needs no persister.batch.topics / persister.bulk.enabled configuration. * * @param boundaryRelationships validated and enriched relationships to persist - * @param requestInfo request info propagated onto each published message + * @param requestInfo request info propagated onto the published message */ public void createBulk(List boundaryRelationships, RequestInfo requestInfo); diff --git a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java index b8128dfb930..777436f4475 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java +++ b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java @@ -53,37 +53,80 @@ public void create(BoundaryRelationshipRequest boundaryRelationshipRequest) { /** * Persists the given validated and enriched boundary relationships through egov-persister rather than - * a direct JDBC write. Each relationship is published as its OWN message to the SAME topic the single - * create uses ({@code save-boundary-relationship}) via {@link #create}, so both paths write identical - * rows through the identical, idempotent - * {@code INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING} mapping. + * a direct JDBC write. The WHOLE validated list is published as ONE message to the SAME topic the + * single create uses ({@code save-boundary-relationship}), under the same {@code BoundaryRelationship} + * key but carrying an ARRAY (whereas single-create carries one object). The persister's mapping reads + * it with an array base path ({@code $.BoundaryRelationship.*}), so {@code PersistRepository.getRows} + * emits one row per element and the listener performs ONE {@code jdbcTemplate.batchUpdate} for the + * whole message through the same idempotent + * {@code INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING} query. One message per + * batch (instead of one message per record) is what restores batched throughput while keeping the + * write owned by the persister. * *

Reusing that topic (instead of a dedicated {@code -batch} topic) is deliberate for * deployment-safety: {@code save-boundary-relationship} is always consumed by the persister's normal - * listener, so bulk creation works on any persister deployment with no extra configuration. Adding - * {@code save-boundary-relationship} to the persister's {@code persister.batch.topics} (with - * {@code persister.bulk.enabled=true}) is a pure, optional throughput optimization: its batch listener - * then aggregates a poll into one multi-row insert. A dedicated {@code -batch} topic, by contrast, is - * dropped by the normal listener and would be silently orphaned if batch mode were not enabled.

+ * single-record listener. Batching here is WITHIN a single message (the array), so it does not depend + * on {@code persister.bulk.enabled}: the normal listener maps the array to N rows and batch-inserts + * them in one transaction. Because the insert is idempotent, at-least-once redelivery is a safe no-op; + * duplicates within/across messages are silently skipped by ON CONFLICT and never abort the batch.

* - *

One message per record (rather than one message carrying the whole batch) preserves per-record - * isolation on either listener: a single un-insertable record fails/dead-letters on its own without - * affecting the rest. Because the insert is idempotent, at-least-once redelivery is a safe no-op.

+ *

The message is keyed by the batch's parent code (callers batch siblings under one already-persisted + * parent), so batches for the same parent stay ordered on the same partition. A null/mixed parent falls + * back to the keyless behaviour of the single path.

* * @param boundaryRelationships validated and enriched relationships to persist - * @param requestInfo request info propagated onto each published message + * @param requestInfo request info propagated onto the published message */ @Override public void createBulk(List boundaryRelationships, RequestInfo requestInfo) { if (CollectionUtils.isEmpty(boundaryRelationships)) return; + // Convert each validated+enriched contract POJO to the DTO that exposes ancestralMaterializedPath + // on the wire (it is @JsonIgnore on BoundaryRelation but @JsonProperty on the DTO), mirroring the + // single-create serialization so both paths persist identical rows. + List boundaryRelationshipDTOs = new ArrayList<>(boundaryRelationships.size()); for (BoundaryRelation boundaryRelationship : boundaryRelationships) { - create(BoundaryRelationshipRequest.builder() - .requestInfo(requestInfo) - .boundaryRelationship(boundaryRelationship) - .build()); + boundaryRelationshipDTOs.add(convertRelationPOJOToDTO(boundaryRelationship)); } + + BulkBoundaryRelationshipRequestDTO batchMessage = BulkBoundaryRelationshipRequestDTO.builder() + .requestInfo(requestInfo) + .boundaryRelationship(boundaryRelationshipDTOs) + .build(); + + // Publish the whole validated list as ONE message to the unchanged topic. + producer.push(applicationProperties.getCreateBoundaryRelationshipTopic(), resolveBatchKey(boundaryRelationships), batchMessage); + } + + /** + * Kafka key for a batch: the shared parent code when every record in the batch has the same + * (non-null) parent, else null (keyless, i.e. the single-create default-partitioner behaviour). + * Keying by parent keeps sibling batches under one parent ordered on the same partition; a mixed or + * root batch must not be forced onto one partition, so it falls back to keyless. + */ + private String resolveBatchKey(List boundaryRelationships) { + String firstParent = boundaryRelationships.get(0).getParent(); + if (firstParent == null) + return null; + for (BoundaryRelation boundaryRelationship : boundaryRelationships) { + if (!firstParent.equals(boundaryRelationship.getParent())) + return null; + } + return firstParent; + } + + /** + * Copies a validated+enriched {@link BoundaryRelation} into a {@link BoundaryRelationshipDTO}, + * carrying over the enriched {@code ancestralMaterializedPath} explicitly (it is not copied by + * BeanUtils onto the wire because it is {@code @JsonIgnore} on the source). Mirrors the field copy + * that {@link #convertContractPOJOToDTO} performs for the single-create path. + */ + private BoundaryRelationshipDTO convertRelationPOJOToDTO(BoundaryRelation boundaryRelationship) { + BoundaryRelationshipDTO boundaryRelationshipDTO = new BoundaryRelationshipDTO(); + BeanUtils.copyProperties(boundaryRelationship, boundaryRelationshipDTO); + boundaryRelationshipDTO.setAncestralMaterializedPath(boundaryRelationship.getAncestralMaterializedPath()); + return boundaryRelationshipDTO; } /** diff --git a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java index 666ca629647..3b382831eb0 100644 --- a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java +++ b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java @@ -158,14 +158,14 @@ public BulkBoundaryRelationshipResponse createBulkBoundaryRelationship(BulkBound } } - // Persistence is delegated to egov-persister (no direct DB write from this service): each - // validated + enriched record is published, one message per record, to the save-boundary-relationship - // topic, which egov-persister writes via an idempotent INSERT ... ON CONFLICT DO NOTHING. Publishing - // is blocking (CustomKafkaTemplate.send().get()); a publish failure (broker unreachable within - // max.block.ms, serialization) is reported transient so the caller retries the affected records — - // re-publishing is a safe no-op because the insert is idempotent. One message per record preserves - // per-record isolation on the persister side regardless of whether it runs the normal or the - // (optional) batch listener. + // Persistence is delegated to egov-persister (no direct DB write from this service): the whole + // validated + enriched list is published as ONE message (relationships as an array) to the + // save-boundary-relationship topic, which egov-persister maps to N rows and writes in a single + // batchUpdate via the idempotent INSERT ... ON CONFLICT DO NOTHING. Publishing is blocking + // (CustomKafkaTemplate.send().get()); a publish failure (broker unreachable within max.block.ms, + // serialization) is reported transient so the caller retries the batch — re-publishing is a safe + // no-op because the insert is idempotent and duplicates are skipped by ON CONFLICT without + // aborting the batch. List successfulRelationships = validatedRelationships; if (!CollectionUtils.isEmpty(validatedRelationships)) { try { diff --git a/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java new file mode 100644 index 00000000000..d44a56d640b --- /dev/null +++ b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java @@ -0,0 +1,42 @@ +package digit.web.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.egov.common.contract.request.RequestInfo; + +import java.util.List; + +/** + * Kafka payload for persisting a validated batch of boundary relationships in ONE message. + * + *

The whole validated list is published as a single message under the {@code BoundaryRelationship} + * key (an ARRAY here, whereas the single-create {@link BoundaryRelationshipRequestDTO} publishes the + * same key as a single object). The persister's {@code save-boundary-relationship} mapping reads this + * with an array base path ({@code $.BoundaryRelationship.*}) so {@code PersistRepository.getRows} + * emits one row per element and the listener does ONE {@code jdbcTemplate.batchUpdate} for the whole + * message.

+ * + *

{@code RequestInfo} is retained at the top level so the persister's version filter + * ({@code $.RequestInfo.ver}) still selects the mapping exactly as it does for the single message.

+ * + *

The elements are {@link BoundaryRelationshipDTO} (not the contract {@link BoundaryRelation}) + * because {@code BoundaryRelation.ancestralMaterializedPath} is {@code @JsonIgnore}; the DTO exposes it + * as {@code ancestralMaterializedPath}, so the enriched path is carried on the wire and persisted — + * identical to the single-create path, which serializes the same DTO field.

+ */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class BulkBoundaryRelationshipRequestDTO { + + @JsonProperty("RequestInfo") + private RequestInfo requestInfo; + + @JsonProperty("BoundaryRelationship") + private List boundaryRelationship; + +} diff --git a/core-services/boundary-service/src/main/resources/boundary-persister.yml b/core-services/boundary-service/src/main/resources/boundary-persister.yml index 57522b2db73..a070352bb8d 100644 --- a/core-services/boundary-service/src/main/resources/boundary-persister.yml +++ b/core-services/boundary-service/src/main/resources/boundary-persister.yml @@ -80,29 +80,29 @@ serviceMaps: isTransaction: true queryMaps: - query: INSERT INTO boundary_relationship (id, tenantId, code, hierarchyType, boundaryType, parent, ancestralMaterializedPath, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING; - basePath: $.BoundaryRelationship + basePath: $.BoundaryRelationship.* jsonMaps: - - jsonPath: $.BoundaryRelationship.id + - jsonPath: $.BoundaryRelationship.*.id - - jsonPath: $.BoundaryRelationship.tenantId + - jsonPath: $.BoundaryRelationship.*.tenantId - - jsonPath: $.BoundaryRelationship.code + - jsonPath: $.BoundaryRelationship.*.code - - jsonPath: $.BoundaryRelationship.hierarchyType + - jsonPath: $.BoundaryRelationship.*.hierarchyType - - jsonPath: $.BoundaryRelationship.boundaryType + - jsonPath: $.BoundaryRelationship.*.boundaryType - - jsonPath: $.BoundaryRelationship.parent + - jsonPath: $.BoundaryRelationship.*.parent - - jsonPath: $.BoundaryRelationship.ancestralMaterializedPath + - jsonPath: $.BoundaryRelationship.*.ancestralMaterializedPath - - jsonPath: $.BoundaryRelationship.auditDetails.createdTime + - jsonPath: $.BoundaryRelationship.*.auditDetails.createdTime - - jsonPath: $.BoundaryRelationship.auditDetails.createdBy + - jsonPath: $.BoundaryRelationship.*.auditDetails.createdBy - - jsonPath: $.BoundaryRelationship.auditDetails.lastModifiedTime + - jsonPath: $.BoundaryRelationship.*.auditDetails.lastModifiedTime - - jsonPath: $.BoundaryRelationship.auditDetails.lastModifiedBy + - jsonPath: $.BoundaryRelationship.*.auditDetails.lastModifiedBy - version: 1.0 description: Updates boundary relationship data fromTopic: update-boundary-relationship diff --git a/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java b/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java new file mode 100644 index 00000000000..db5cccd9765 --- /dev/null +++ b/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java @@ -0,0 +1,125 @@ +package digit.repository.impl; + +import digit.config.ApplicationProperties; +import digit.kafka.Producer; +import digit.web.models.BoundaryRelation; +import digit.web.models.BulkBoundaryRelationshipRequestDTO; +import org.egov.common.contract.models.AuditDetails; +import org.egov.common.contract.request.RequestInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Unit tests for {@link BoundaryRelationshipRepositoryImpl#createBulk}. + * + *

The batch change must publish the WHOLE validated list as EXACTLY ONE Kafka message (previously it + * looped and published one message per record). These tests pin that contract: one push per bulk call, + * carrying an array whose size equals the input size, on the unchanged topic, with the enriched + * ancestralMaterializedPath preserved and the batch keyed by the shared parent.

+ */ +@ExtendWith(MockitoExtension.class) +class BoundaryRelationshipRepositoryImplBulkTest { + + private static final String TOPIC = "save-boundary-relationship"; + + @Mock + private Producer producer; + + @Mock + private ApplicationProperties applicationProperties; + + @InjectMocks + private BoundaryRelationshipRepositoryImpl repository; + + private RequestInfo requestInfo() { + return RequestInfo.builder().apiId("boundary").ver("1.0").build(); + } + + private BoundaryRelation relation(String code, String parent, String amp) { + return BoundaryRelation.builder() + .id("id-" + code) + .code(code) + .tenantId("mz") + .hierarchyType("ADMIN") + .boundaryType("Village") + .parent(parent) + .ancestralMaterializedPath(amp) + .auditDetails(AuditDetails.builder() + .createdBy("u1").createdTime(1L).lastModifiedBy("u1").lastModifiedTime(1L).build()) + .build(); + } + + @Test + void createBulk_publishesExactlyOneMessageCarryingTheWholeList() { + lenient().when(applicationProperties.getCreateBoundaryRelationshipTopic()).thenReturn(TOPIC); + + List input = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + input.add(relation("V" + i, "P1", "R1|P1")); + } + + repository.createBulk(input, requestInfo()); + + ArgumentCaptor topicCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(Object.class); + + // EXACTLY ONE push (was N=5 with the old per-record loop). + verify(producer, times(1)).push(topicCaptor.capture(), keyCaptor.capture(), valueCaptor.capture()); + verify(producer, never()).push(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + assertEquals(TOPIC, topicCaptor.getValue(), "topic must be unchanged save-boundary-relationship"); + assertEquals("P1", keyCaptor.getValue(), "batch keyed by the shared parent code"); + + Object value = valueCaptor.getValue(); + assertNotNull(value); + assertEquals(BulkBoundaryRelationshipRequestDTO.class, value.getClass(), "payload must be the batch DTO"); + BulkBoundaryRelationshipRequestDTO payload = (BulkBoundaryRelationshipRequestDTO) value; + + assertEquals(5, payload.getBoundaryRelationship().size(), "array size must equal input size"); + assertNotNull(payload.getRequestInfo(), "RequestInfo retained for persister version filter"); + + // ancestralMaterializedPath is @JsonIgnore on BoundaryRelation but must survive on the DTO. + assertEquals("R1|P1", payload.getBoundaryRelationship().get(0).getAncestralMaterializedPath()); + assertEquals("V0", payload.getBoundaryRelationship().get(0).getCode()); + assertEquals("id-V0", payload.getBoundaryRelationship().get(0).getId()); + assertNotNull(payload.getBoundaryRelationship().get(0).getAuditDetails()); + } + + @Test + void createBulk_mixedParents_fallsBackToKeylessPush() { + lenient().when(applicationProperties.getCreateBoundaryRelationshipTopic()).thenReturn(TOPIC); + + List input = new ArrayList<>(); + input.add(relation("V0", "P1", "R1|P1")); + input.add(relation("V1", "P2", "R1|P2")); + + repository.createBulk(input, requestInfo()); + + ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(String.class); + verify(producer, times(1)).push(org.mockito.ArgumentMatchers.eq(TOPIC), keyCaptor.capture(), org.mockito.ArgumentMatchers.any()); + assertEquals(null, keyCaptor.getValue(), "mixed parents must fall back to keyless (null key)"); + } + + @Test + void createBulk_emptyList_publishesNothing() { + repository.createBulk(new ArrayList<>(), requestInfo()); + verify(producer, never()).push(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + verify(producer, never()).push(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + } +} From 0aeb68a10d9e2604c55e96ed9f3f4521afd39651 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 13 Jul 2026 16:10:49 +0530 Subject: [PATCH 13/24] Bulk kafka handling --- .../digit/config/ApplicationProperties.java | 6 + .../BoundaryRelationshipRepository.java | 18 ++- .../BoundaryRelationshipRepositoryImpl.java | 37 ++--- .../service/BoundaryRelationshipService.java | 6 +- .../BulkBoundaryRelationshipRequestDTO.java | 10 +- .../src/main/resources/application.properties | 4 + .../src/main/resources/boundary-persister.yml | 31 +++- ...aryRelationshipRepositoryImplBulkTest.java | 11 +- .../BoundaryRelationshipMappingShapeTest.java | 142 ++++++++++++++++++ 9 files changed, 229 insertions(+), 36 deletions(-) create mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java diff --git a/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java b/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java index fbc569aeb7e..c22ed759cd6 100644 --- a/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java +++ b/core-services/boundary-service/src/main/java/digit/config/ApplicationProperties.java @@ -99,6 +99,12 @@ public class ApplicationProperties { @Value("${kafka.topics.update.boundary.relationship}") private String updateBoundaryRelationshipTopic; + // Dedicated topic for the batched bulk-create path: createBulk publishes ONE array message here, mapped by + // the persister with an array base path ($.BoundaryRelationship.*). Kept separate from the single-create + // topic because a single object and an array cannot share one persister queryMap. + @Value("${kafka.topics.bulk.create.boundary.relationship.job}") + private String bulkCreateBoundaryRelationshipJobTopic; + // Upper bound on records accepted by POST /boundary-relationships/bulk/_create. Enforced in the // service (bean validation is not active in this deployment). Keep in sync with the caller's chunk size. @Value("${boundary.bulk.max.size:100}") diff --git a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java index bfd656cdf9d..21017f86ede 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java +++ b/core-services/boundary-service/src/main/java/digit/repository/BoundaryRelationshipRepository.java @@ -14,13 +14,17 @@ public interface BoundaryRelationshipRepository { /** * Persists the given validated and enriched boundary relationships through egov-persister (no direct - * DB write): the WHOLE list is published as ONE message to the same save-boundary-relationship topic - * the single {@link #create} uses, carrying the relationships as an array under the same - * {@code BoundaryRelationship} key. The publish is blocking (it returns once the broker has accepted - * the message); egov-persister reads the array (array base path) and writes it as a single - * batchUpdate through the idempotent INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING, - * so redelivery is a safe no-op and duplicates are skipped without aborting the batch. Batching is - * WITHIN the one message, so it needs no persister.batch.topics / persister.bulk.enabled configuration. + * DB write): the WHOLE list is published as ONE message to the DEDICATED bulk topic + * {@code boundary-relationship-bulk-create-job} (NOT the single {@link #create} topic + * {@code save-boundary-relationship}), carrying the relationships as an array under the + * {@code BoundaryRelationship} key. A dedicated topic is required because the persister maps this one + * with an array base path ({@code $.BoundaryRelationship.*}) while single-create stays single-object + * ({@code $.BoundaryRelationship}) — one queryMap has one base path, so the two shapes cannot share a + * topic. The publish is blocking (it returns once the broker has accepted the message); egov-persister + * reads the array and writes it as a single batchUpdate through the idempotent + * INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING, so redelivery is a safe no-op and + * duplicates are skipped without aborting the batch. Batching is WITHIN the one message, so it needs no + * persister.batch.topics / persister.bulk.enabled configuration. * * @param boundaryRelationships validated and enriched relationships to persist * @param requestInfo request info propagated onto the published message diff --git a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java index 777436f4475..ee846cb7db3 100644 --- a/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java +++ b/core-services/boundary-service/src/main/java/digit/repository/impl/BoundaryRelationshipRepositoryImpl.java @@ -53,22 +53,25 @@ public void create(BoundaryRelationshipRequest boundaryRelationshipRequest) { /** * Persists the given validated and enriched boundary relationships through egov-persister rather than - * a direct JDBC write. The WHOLE validated list is published as ONE message to the SAME topic the - * single create uses ({@code save-boundary-relationship}), under the same {@code BoundaryRelationship} - * key but carrying an ARRAY (whereas single-create carries one object). The persister's mapping reads - * it with an array base path ({@code $.BoundaryRelationship.*}), so {@code PersistRepository.getRows} - * emits one row per element and the listener performs ONE {@code jdbcTemplate.batchUpdate} for the - * whole message through the same idempotent - * {@code INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING} query. One message per - * batch (instead of one message per record) is what restores batched throughput while keeping the - * write owned by the persister. + * a direct JDBC write. The WHOLE validated list is published as ONE message — an ARRAY under the + * {@code BoundaryRelationship} key — to the DEDICATED bulk topic + * ({@code boundary-relationship-bulk-create-job}). The persister maps that topic with an array base + * path ({@code $.BoundaryRelationship.*}), so {@code PersistRepository.getRows} emits one row per + * element and the listener performs ONE {@code jdbcTemplate.batchUpdate} for the whole message through + * the idempotent {@code INSERT ... ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING} query. One + * message per batch (instead of one message per record) is what restores batched throughput while + * keeping the write owned by the persister. * - *

Reusing that topic (instead of a dedicated {@code -batch} topic) is deliberate for - * deployment-safety: {@code save-boundary-relationship} is always consumed by the persister's normal - * single-record listener. Batching here is WITHIN a single message (the array), so it does not depend - * on {@code persister.bulk.enabled}: the normal listener maps the array to N rows and batch-inserts - * them in one transaction. Because the insert is idempotent, at-least-once redelivery is a safe no-op; - * duplicates within/across messages are silently skipped by ON CONFLICT and never abort the batch.

+ *

A DEDICATED topic (NOT the single-create {@code save-boundary-relationship}) is required for + * correctness: single-create publishes {@code BoundaryRelationship} as a single OBJECT, bulk publishes + * it as an ARRAY. A persister queryMap has exactly one base path, so the two shapes cannot share a + * topic — an array mapping ({@code .*}) mis-reads a single object (JsonPath {@code .*} over an object + * yields its property values, not one row) and a single mapping mis-reads an array. Each shape + * therefore gets its own topic + queryMap. Batching is WITHIN a single message (the array), so it does + * not depend on {@code persister.bulk.enabled}: the normal listener maps the array to N rows and + * batch-inserts them in one transaction. Because the insert is idempotent, at-least-once redelivery is + * a safe no-op; duplicates within/across messages are silently skipped by ON CONFLICT and never abort + * the batch.

* *

The message is keyed by the batch's parent code (callers batch siblings under one already-persisted * parent), so batches for the same parent stay ordered on the same partition. A null/mixed parent falls @@ -95,8 +98,8 @@ public void createBulk(List boundaryRelationships, RequestInfo .boundaryRelationship(boundaryRelationshipDTOs) .build(); - // Publish the whole validated list as ONE message to the unchanged topic. - producer.push(applicationProperties.getCreateBoundaryRelationshipTopic(), resolveBatchKey(boundaryRelationships), batchMessage); + // Publish the whole validated list as ONE message to the dedicated bulk topic. + producer.push(applicationProperties.getBulkCreateBoundaryRelationshipJobTopic(), resolveBatchKey(boundaryRelationships), batchMessage); } /** diff --git a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java index 3b382831eb0..88e1847bb70 100644 --- a/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java +++ b/core-services/boundary-service/src/main/java/digit/service/BoundaryRelationshipService.java @@ -160,8 +160,10 @@ public BulkBoundaryRelationshipResponse createBulkBoundaryRelationship(BulkBound // Persistence is delegated to egov-persister (no direct DB write from this service): the whole // validated + enriched list is published as ONE message (relationships as an array) to the - // save-boundary-relationship topic, which egov-persister maps to N rows and writes in a single - // batchUpdate via the idempotent INSERT ... ON CONFLICT DO NOTHING. Publishing is blocking + // dedicated boundary-relationship-bulk-create-job topic, which egov-persister maps ($.BoundaryRelationship.*) + // to N rows and writes in a single batchUpdate via the idempotent INSERT ... ON CONFLICT DO NOTHING. + // (Single-create keeps its own save-boundary-relationship topic + single-object mapping; the two + // message shapes cannot share a topic.) Publishing is blocking // (CustomKafkaTemplate.send().get()); a publish failure (broker unreachable within max.block.ms, // serialization) is reported transient so the caller retries the batch — re-publishing is a safe // no-op because the insert is idempotent and duplicates are skipped by ON CONFLICT without diff --git a/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java index d44a56d640b..8dba1c79b8d 100644 --- a/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java +++ b/core-services/boundary-service/src/main/java/digit/web/models/BulkBoundaryRelationshipRequestDTO.java @@ -14,10 +14,12 @@ * *

The whole validated list is published as a single message under the {@code BoundaryRelationship} * key (an ARRAY here, whereas the single-create {@link BoundaryRelationshipRequestDTO} publishes the - * same key as a single object). The persister's {@code save-boundary-relationship} mapping reads this - * with an array base path ({@code $.BoundaryRelationship.*}) so {@code PersistRepository.getRows} - * emits one row per element and the listener does ONE {@code jdbcTemplate.batchUpdate} for the whole - * message.

+ * same key as a single object). It goes to the DEDICATED bulk topic + * ({@code boundary-relationship-bulk-create-job}), whose persister mapping reads it with an array base + * path ({@code $.BoundaryRelationship.*}) so {@code PersistRepository.getRows} emits one row per element + * and the listener does ONE {@code jdbcTemplate.batchUpdate} for the whole message. It must NOT share the + * single-create topic {@code save-boundary-relationship}: a persister queryMap has one base path, and an + * array shape and a single-object shape cannot both be read by the same mapping.

* *

{@code RequestInfo} is retained at the top level so the persister's version filter * ({@code $.RequestInfo.ver}) still selects the mapping exactly as it does for the single message.

diff --git a/core-services/boundary-service/src/main/resources/application.properties b/core-services/boundary-service/src/main/resources/application.properties index 010a62b7291..d9deeedf80e 100644 --- a/core-services/boundary-service/src/main/resources/application.properties +++ b/core-services/boundary-service/src/main/resources/application.properties @@ -95,6 +95,10 @@ kafka.topics.update.boundary.hierarchy = update-boundary-hierarchy-definition # optional throughput optimization that lets the persister aggregate a poll into one multi-row insert. kafka.topics.create.boundary.relationship = save-boundary-relationship kafka.topics.update.boundary.relationship = update-boundary-relationship +# Dedicated topic for the batched bulk-create path (one ARRAY message per batch). Kept separate from the +# single-create topic above because the persister maps this one with an array base path ($.BoundaryRelationship.*) +# while save-boundary-relationship stays single-object ($.BoundaryRelationship) — the two shapes cannot share a topic. +kafka.topics.bulk.create.boundary.relationship.job = boundary-relationship-bulk-create-job boundary.default.offset=0 boundary.default.limit=50 boundary.max.default.limit=300 diff --git a/core-services/boundary-service/src/main/resources/boundary-persister.yml b/core-services/boundary-service/src/main/resources/boundary-persister.yml index a070352bb8d..7674cc101b5 100644 --- a/core-services/boundary-service/src/main/resources/boundary-persister.yml +++ b/core-services/boundary-service/src/main/resources/boundary-persister.yml @@ -75,9 +75,38 @@ serviceMaps: - jsonPath: $.BoundaryHierarchy.auditDetails.lastModifiedBy - version: 1.0 - description: Persists the boundary relationship data + description: Persists a SINGLE boundary relationship (POST /boundary-relationships/_create -> object payload) fromTopic: save-boundary-relationship isTransaction: true + queryMaps: + - query: INSERT INTO boundary_relationship (id, tenantId, code, hierarchyType, boundaryType, parent, ancestralMaterializedPath, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING; + basePath: $.BoundaryRelationship + jsonMaps: + - jsonPath: $.BoundaryRelationship.id + + - jsonPath: $.BoundaryRelationship.tenantId + + - jsonPath: $.BoundaryRelationship.code + + - jsonPath: $.BoundaryRelationship.hierarchyType + + - jsonPath: $.BoundaryRelationship.boundaryType + + - jsonPath: $.BoundaryRelationship.parent + + - jsonPath: $.BoundaryRelationship.ancestralMaterializedPath + + - jsonPath: $.BoundaryRelationship.auditDetails.createdTime + + - jsonPath: $.BoundaryRelationship.auditDetails.createdBy + + - jsonPath: $.BoundaryRelationship.auditDetails.lastModifiedTime + + - jsonPath: $.BoundaryRelationship.auditDetails.lastModifiedBy + - version: 1.0 + description: Persists a BATCH of boundary relationships published as ONE array message (POST /boundary-relationships/bulk/_create -> array payload) + fromTopic: boundary-relationship-bulk-create-job + isTransaction: true queryMaps: - query: INSERT INTO boundary_relationship (id, tenantId, code, hierarchyType, boundaryType, parent, ancestralMaterializedPath, createdTime, createdBy, lastModifiedTime, lastModifiedBy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING; basePath: $.BoundaryRelationship.* diff --git a/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java b/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java index db5cccd9765..308dd4cfa1a 100644 --- a/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java +++ b/core-services/boundary-service/src/test/java/digit/repository/impl/BoundaryRelationshipRepositoryImplBulkTest.java @@ -29,13 +29,14 @@ * *

The batch change must publish the WHOLE validated list as EXACTLY ONE Kafka message (previously it * looped and published one message per record). These tests pin that contract: one push per bulk call, - * carrying an array whose size equals the input size, on the unchanged topic, with the enriched + * carrying an array whose size equals the input size, on the dedicated bulk topic + * (boundary-relationship-bulk-create-job, separate from the single-create topic), with the enriched * ancestralMaterializedPath preserved and the batch keyed by the shared parent.

*/ @ExtendWith(MockitoExtension.class) class BoundaryRelationshipRepositoryImplBulkTest { - private static final String TOPIC = "save-boundary-relationship"; + private static final String TOPIC = "boundary-relationship-bulk-create-job"; @Mock private Producer producer; @@ -66,7 +67,7 @@ private BoundaryRelation relation(String code, String parent, String amp) { @Test void createBulk_publishesExactlyOneMessageCarryingTheWholeList() { - lenient().when(applicationProperties.getCreateBoundaryRelationshipTopic()).thenReturn(TOPIC); + lenient().when(applicationProperties.getBulkCreateBoundaryRelationshipJobTopic()).thenReturn(TOPIC); List input = new ArrayList<>(); for (int i = 0; i < 5; i++) { @@ -83,7 +84,7 @@ void createBulk_publishesExactlyOneMessageCarryingTheWholeList() { verify(producer, times(1)).push(topicCaptor.capture(), keyCaptor.capture(), valueCaptor.capture()); verify(producer, never()).push(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); - assertEquals(TOPIC, topicCaptor.getValue(), "topic must be unchanged save-boundary-relationship"); + assertEquals(TOPIC, topicCaptor.getValue(), "bulk batch must go to the dedicated boundary-relationship-bulk-create-job topic, NOT save-boundary-relationship"); assertEquals("P1", keyCaptor.getValue(), "batch keyed by the shared parent code"); Object value = valueCaptor.getValue(); @@ -103,7 +104,7 @@ void createBulk_publishesExactlyOneMessageCarryingTheWholeList() { @Test void createBulk_mixedParents_fallsBackToKeylessPush() { - lenient().when(applicationProperties.getCreateBoundaryRelationshipTopic()).thenReturn(TOPIC); + lenient().when(applicationProperties.getBulkCreateBoundaryRelationshipJobTopic()).thenReturn(TOPIC); List input = new ArrayList<>(); input.add(relation("V0", "P1", "R1|P1")); diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java new file mode 100644 index 00000000000..aa4cb994011 --- /dev/null +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java @@ -0,0 +1,142 @@ +package org.egov.infra.persist.repository; + +import com.jayway.jsonpath.Configuration; +import org.egov.infra.persist.web.contract.JsonMap; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Proves the boundary-relationship "dedicated bulk topic" fix at the persister layer, using the REAL + * {@link PersistRepository#getRows} extraction (documents parsed exactly as + * {@code PersistService} does: {@code Configuration.defaultConfiguration().jsonProvider().parse(json)}). + * + *

Two message shapes exist on two different topics: + *

    + *
  • single-create publishes {@code {"BoundaryRelationship": {..one object..}}} to + * {@code save-boundary-relationship} — mapped with base path {@code $.BoundaryRelationship} (no wildcard).
    • + *
  • bulk-create publishes {@code {"BoundaryRelationship": [..array..]}} to + * {@code boundary-relationship-bulk-create-job} — mapped with base path {@code $.BoundaryRelationship.*}.
    • + *
+ * The positive tests prove each shape+mapping pairing extracts the correct number of rows with correct + * field values. The negative tests prove WHY the topics must stay separate: crossing a shape with the + * wrong base path fails (the original single-topic blocker).

+ */ +class BoundaryRelationshipMappingShapeTest { + + // Column order of the INSERT in boundary-persister.yml + private static final String[] FIELDS = { + "id", "tenantId", "code", "hierarchyType", "boundaryType", "parent", + "ancestralMaterializedPath", + "auditDetails.createdTime", "auditDetails.createdBy", + "auditDetails.lastModifiedTime", "auditDetails.lastModifiedBy" + }; + + private List jsonMaps(String prefix) { + List maps = new ArrayList<>(); + for (String f : FIELDS) { + JsonMap m = new JsonMap(); + m.setJsonPath(prefix + f); // e.g. "$.BoundaryRelationship." + "code" or "$.BoundaryRelationship.*." + "code" + maps.add(m); + } + return maps; + } + + private String relJson(String id, String code, String parent, String amp) { + return "{" + + "\"id\":\"" + id + "\"," + + "\"tenantId\":\"mz\"," + + "\"code\":\"" + code + "\"," + + "\"hierarchyType\":\"ADMIN\"," + + "\"boundaryType\":\"Village\"," + + "\"parent\":" + (parent == null ? "null" : "\"" + parent + "\"") + "," + + "\"ancestralMaterializedPath\":\"" + amp + "\"," + + "\"auditDetails\":{\"createdTime\":100,\"createdBy\":\"u1\",\"lastModifiedTime\":200,\"lastModifiedBy\":\"u2\"}" + + "}"; + } + + private Object parse(String json) { + // identical to PersistService.persist(...) + return Configuration.defaultConfiguration().jsonProvider().parse(json); + } + + // -------------------- POSITIVE: correct shape + correct mapping -------------------- + + @Test + void singleObject_onSingleMapping_extractsExactlyOneRowWithCorrectValues() { + String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":" + + relJson("id-C1", "C1", null, "C1") + "}"; + + List rows = new PersistRepository() + .getRows(jsonMaps("$.BoundaryRelationship."), parse(json), "$.BoundaryRelationship"); + + assertEquals(1, rows.size(), "single-object message must yield exactly ONE row"); + Object[] r = rows.get(0); + assertEquals(11, r.length, "row must have all 11 columns"); + assertEquals("id-C1", r[0]); + assertEquals("mz", r[1]); + assertEquals("C1", r[2]); + assertEquals("ADMIN", r[3]); + assertEquals("Village", r[4]); + assertEquals(null, r[5], "root parent is null"); + assertEquals("C1", r[6]); + assertEquals("u1", r[8]); + assertEquals("u2", r[10]); + } + + @Test + void array_onBulkMapping_extractsOneRowPerElementWithCorrectValues() { + StringBuilder arr = new StringBuilder(); + int n = 5; + for (int i = 0; i < n; i++) { + if (i > 0) arr.append(","); + arr.append(relJson("id-V" + i, "V" + i, "P1", "R1|P1|V" + i)); + } + String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":[" + arr + "]}"; + + List rows = new PersistRepository() + .getRows(jsonMaps("$.BoundaryRelationship.*."), parse(json), "$.BoundaryRelationship.*"); + + assertEquals(n, rows.size(), "array message must yield ONE row per element"); + // first element + assertEquals("id-V0", rows.get(0)[0]); + assertEquals("V0", rows.get(0)[2]); + assertEquals("P1", rows.get(0)[5]); + assertEquals("R1|P1|V0", rows.get(0)[6]); + // last element + assertEquals("id-V4", rows.get(4)[0]); + assertEquals("V4", rows.get(4)[2]); + assertEquals("R1|P1|V4", rows.get(4)[6]); + // every row fully populated + for (Object[] r : rows) { + assertEquals(11, r.length); + assertEquals("mz", r[1]); + assertEquals("ADMIN", r[3]); + } + } + + // -------------------- NEGATIVE: why the topics must stay separate (the original blocker) -------------------- + + @Test + void singleObject_onBulkMapping_breaks() { + // This is exactly what happened when both shapes shared save-boundary-relationship with a .* mapping. + String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":" + + relJson("id-C1", "C1", null, "C1") + "}"; + assertThrows(Exception.class, () -> new PersistRepository() + .getRows(jsonMaps("$.BoundaryRelationship.*."), parse(json), "$.BoundaryRelationship.*"), + "a single OBJECT under a .* (array) base path must fail — this is why bulk needs its own topic"); + } + + @Test + void array_onSingleMapping_breaks() { + String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":[" + + relJson("id-V0", "V0", "P1", "R1|P1|V0") + "]}"; + assertThrows(Exception.class, () -> new PersistRepository() + .getRows(jsonMaps("$.BoundaryRelationship."), parse(json), "$.BoundaryRelationship"), + "an ARRAY under a single (non-wildcard) base path must fail — confirms single-create keeps its object topic"); + } +} From 7c8d3ea104e93e601d950d0e20320d1fc09a42f1 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 13 Jul 2026 16:18:37 +0530 Subject: [PATCH 14/24] Remove boundary-relationship persister shape test BoundaryRelationshipMappingShapeTest was a local verification harness used to prove the dedicated-bulk-topic fix (single-object vs array extraction). Its purpose is served, and a boundary-relationship-specific test does not belong in the generic egov-persister module. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../BoundaryRelationshipMappingShapeTest.java | 142 ------------------ 1 file changed, 142 deletions(-) delete mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java deleted file mode 100644 index aa4cb994011..00000000000 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/repository/BoundaryRelationshipMappingShapeTest.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.egov.infra.persist.repository; - -import com.jayway.jsonpath.Configuration; -import org.egov.infra.persist.web.contract.JsonMap; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * Proves the boundary-relationship "dedicated bulk topic" fix at the persister layer, using the REAL - * {@link PersistRepository#getRows} extraction (documents parsed exactly as - * {@code PersistService} does: {@code Configuration.defaultConfiguration().jsonProvider().parse(json)}). - * - *

Two message shapes exist on two different topics: - *

    - *
  • single-create publishes {@code {"BoundaryRelationship": {..one object..}}} to - * {@code save-boundary-relationship} — mapped with base path {@code $.BoundaryRelationship} (no wildcard).
    • - *
  • bulk-create publishes {@code {"BoundaryRelationship": [..array..]}} to - * {@code boundary-relationship-bulk-create-job} — mapped with base path {@code $.BoundaryRelationship.*}.
    • - *
- * The positive tests prove each shape+mapping pairing extracts the correct number of rows with correct - * field values. The negative tests prove WHY the topics must stay separate: crossing a shape with the - * wrong base path fails (the original single-topic blocker).

- */ -class BoundaryRelationshipMappingShapeTest { - - // Column order of the INSERT in boundary-persister.yml - private static final String[] FIELDS = { - "id", "tenantId", "code", "hierarchyType", "boundaryType", "parent", - "ancestralMaterializedPath", - "auditDetails.createdTime", "auditDetails.createdBy", - "auditDetails.lastModifiedTime", "auditDetails.lastModifiedBy" - }; - - private List jsonMaps(String prefix) { - List maps = new ArrayList<>(); - for (String f : FIELDS) { - JsonMap m = new JsonMap(); - m.setJsonPath(prefix + f); // e.g. "$.BoundaryRelationship." + "code" or "$.BoundaryRelationship.*." + "code" - maps.add(m); - } - return maps; - } - - private String relJson(String id, String code, String parent, String amp) { - return "{" - + "\"id\":\"" + id + "\"," - + "\"tenantId\":\"mz\"," - + "\"code\":\"" + code + "\"," - + "\"hierarchyType\":\"ADMIN\"," - + "\"boundaryType\":\"Village\"," - + "\"parent\":" + (parent == null ? "null" : "\"" + parent + "\"") + "," - + "\"ancestralMaterializedPath\":\"" + amp + "\"," - + "\"auditDetails\":{\"createdTime\":100,\"createdBy\":\"u1\",\"lastModifiedTime\":200,\"lastModifiedBy\":\"u2\"}" - + "}"; - } - - private Object parse(String json) { - // identical to PersistService.persist(...) - return Configuration.defaultConfiguration().jsonProvider().parse(json); - } - - // -------------------- POSITIVE: correct shape + correct mapping -------------------- - - @Test - void singleObject_onSingleMapping_extractsExactlyOneRowWithCorrectValues() { - String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":" - + relJson("id-C1", "C1", null, "C1") + "}"; - - List rows = new PersistRepository() - .getRows(jsonMaps("$.BoundaryRelationship."), parse(json), "$.BoundaryRelationship"); - - assertEquals(1, rows.size(), "single-object message must yield exactly ONE row"); - Object[] r = rows.get(0); - assertEquals(11, r.length, "row must have all 11 columns"); - assertEquals("id-C1", r[0]); - assertEquals("mz", r[1]); - assertEquals("C1", r[2]); - assertEquals("ADMIN", r[3]); - assertEquals("Village", r[4]); - assertEquals(null, r[5], "root parent is null"); - assertEquals("C1", r[6]); - assertEquals("u1", r[8]); - assertEquals("u2", r[10]); - } - - @Test - void array_onBulkMapping_extractsOneRowPerElementWithCorrectValues() { - StringBuilder arr = new StringBuilder(); - int n = 5; - for (int i = 0; i < n; i++) { - if (i > 0) arr.append(","); - arr.append(relJson("id-V" + i, "V" + i, "P1", "R1|P1|V" + i)); - } - String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":[" + arr + "]}"; - - List rows = new PersistRepository() - .getRows(jsonMaps("$.BoundaryRelationship.*."), parse(json), "$.BoundaryRelationship.*"); - - assertEquals(n, rows.size(), "array message must yield ONE row per element"); - // first element - assertEquals("id-V0", rows.get(0)[0]); - assertEquals("V0", rows.get(0)[2]); - assertEquals("P1", rows.get(0)[5]); - assertEquals("R1|P1|V0", rows.get(0)[6]); - // last element - assertEquals("id-V4", rows.get(4)[0]); - assertEquals("V4", rows.get(4)[2]); - assertEquals("R1|P1|V4", rows.get(4)[6]); - // every row fully populated - for (Object[] r : rows) { - assertEquals(11, r.length); - assertEquals("mz", r[1]); - assertEquals("ADMIN", r[3]); - } - } - - // -------------------- NEGATIVE: why the topics must stay separate (the original blocker) -------------------- - - @Test - void singleObject_onBulkMapping_breaks() { - // This is exactly what happened when both shapes shared save-boundary-relationship with a .* mapping. - String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":" - + relJson("id-C1", "C1", null, "C1") + "}"; - assertThrows(Exception.class, () -> new PersistRepository() - .getRows(jsonMaps("$.BoundaryRelationship.*."), parse(json), "$.BoundaryRelationship.*"), - "a single OBJECT under a .* (array) base path must fail — this is why bulk needs its own topic"); - } - - @Test - void array_onSingleMapping_breaks() { - String json = "{\"RequestInfo\":{\"ver\":\"1.0\"},\"BoundaryRelationship\":[" - + relJson("id-V0", "V0", "P1", "R1|P1|V0") + "]}"; - assertThrows(Exception.class, () -> new PersistRepository() - .getRows(jsonMaps("$.BoundaryRelationship."), parse(json), "$.BoundaryRelationship"), - "an ARRAY under a single (non-wildcard) base path must fail — confirms single-create keeps its object topic"); - } -} From 80308c38f3bd66c407fc00f8e676f61f909c57e6 Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Fri, 17 Jul 2026 12:17:47 +0530 Subject: [PATCH 15/24] tracer changes 2.9.3 version CHANGELOG added --- core-services/libraries/tracer/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core-services/libraries/tracer/CHANGELOG.md b/core-services/libraries/tracer/CHANGELOG.md index 94b19eb2aba..956e8bf578f 100644 --- a/core-services/libraries/tracer/CHANGELOG.md +++ b/core-services/libraries/tracer/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog All notable changes to this library will be documented in this file. +## 2.9.3 - 2026-07-17 +- Propagated correlationId and tenantId across Kafka — stamp both onto message headers on produce and rebuild the MDC from them on consume, so consumer/async logs carry the same IDs +- Added MdcRecordInterceptor (auto-attached to the default Kafka listener container factory), KafkaTemplateLoggingInterceptors, and TracerKafkaMdcUtil helper +- Added kill-switch `tracer.kafka.mdc.enabled` (default true) + +## 2.9.2 - 2026-03-10 +- Addition of Data Access Exception Handling in ExceptionAdvice. + ## 2.9.0 - 2024-02-29 - Upgraded spring boot version from 2.2.6.RELEASE to 3.2.2 - Upgraded java version from 1.8 to 17 From b242c9b6c1af46d47d0d4ed9a185686b1acac387 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Mon, 20 Jul 2026 17:19:44 +0530 Subject: [PATCH 16/24] Version bumps for 2.1 release --- core-services/boundary-service/CHANGELOG.md | 11 +++++ core-services/boundary-service/README.md | 25 ++++++----- core-services/boundary-service/pom.xml | 2 +- core-services/egov-persister/CHANGELOG.md | 10 +++++ core-services/egov-persister/README.md | 47 ++++++++++++++++++++- core-services/egov-persister/pom.xml | 2 +- core-services/libraries/tracer/readme.md | 42 +++++++++++++++--- 7 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 core-services/boundary-service/CHANGELOG.md diff --git a/core-services/boundary-service/CHANGELOG.md b/core-services/boundary-service/CHANGELOG.md new file mode 100644 index 00000000000..6cd144c381f --- /dev/null +++ b/core-services/boundary-service/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +All notable changes to this module will be documented in this file. + +## 1.0.2 - 2026-07-20 +- Bulk relationship create API (`POST /boundary-relationships/bulk/_create`): relationships are validated + enriched synchronously and published together as **one array message keyed by the shared parent code** to a dedicated bulk topic, instead of one message per record — new models `BulkBoundaryRelationshipRequest` / `BulkBoundaryRelationshipRequestDTO` / `BulkBoundaryRelationshipResponse` / `FailedBoundaryRelationship` +- Persister config (`boundary-persister.yml`) now carries **two separate relationship queryMaps** — a single-object map for `save-boundary-relationship` (single create) and an array map for the dedicated bulk topic +- Boundary relationship creation performance enhancement: query-builder and repository bulk paths reworked; new search-index migration `V20260616120000__boundary_relationship_search_indexes.sql` +- `correlationId` + `tenantId` now propagated across Kafka on publish (via tracer `2.9.3-SNAPSHOT`) +- New error codes and configurable properties added (`ApplicationProperties`, `ErrorCodes`, `application.properties`) +- Code-review (CodeRabbit) fixes applied diff --git a/core-services/boundary-service/README.md b/core-services/boundary-service/README.md index 614a4952835..07aa5dc64da 100644 --- a/core-services/boundary-service/README.md +++ b/core-services/boundary-service/README.md @@ -28,12 +28,12 @@ All endpoints are `POST`. Base URL: `http://:8081/boundary-service`. ### Single vs. bulk relationship create -Both paths validate + enrich a relationship (assign `id`, audit details, and the ancestral materialized path) and then **publish it to the `save-boundary-relationship` topic**, which `egov-persister` writes with an idempotent `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`. Neither path writes to the database directly. +Both paths validate + enrich a relationship (assign `id`, audit details, and the ancestral materialized path) before anything is placed on Kafka, and `egov-persister` writes every record with an idempotent `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`. Neither path writes to the database directly. The two paths use **different topics** because the message shapes differ: -- The **single** `_create` validates one record and publishes it, returning `202 Accepted` (acceptance, not a committed write). -- The **bulk** `_create` validates + enriches every record in the request **synchronously**, publishes the valid ones (one message per record), and returns `200 OK` with a **per-record outcome** so the caller learns immediately which records were accepted and which failed validation, and with what reason. +- The **single** `_create` validates one record and publishes it as a single object to the **`save-boundary-relationship`** topic, returning `202 Accepted` (acceptance, not a committed write). +- The **bulk** `_create` validates + enriches every record in the request **synchronously**, then publishes the *whole validated batch as ONE array message* (the `BoundaryRelationship` field is a JSON array) to the dedicated **`boundary-relationship-bulk-create-job`** topic — keyed by the batch's shared parent code (keyless when parents differ or the batch is root) — and returns `200 OK` with a **per-record outcome** so the caller learns immediately which records were accepted and which failed validation, and with what reason. -See [`docs/Bulk-Boundary-Relationship-Creation-Design.docx`](docs/Bulk-Boundary-Relationship-Creation-Design.docx) for the design rationale and [`docs/Bulk-Boundary-Relationship-Flow.docx`](docs/Bulk-Boundary-Relationship-Flow.docx) for the end-to-end flow. +The design rationale and end-to-end flow for bulk creation are documented in the sections below. --- @@ -41,13 +41,13 @@ See [`docs/Bulk-Boundary-Relationship-Creation-Design.docx`](docs/Bulk-Boundary- `POST /boundary-relationships/bulk/_create` -Creates many relationships in one request with a **per-record outcome**. Validation and enrichment happen synchronously on the request thread **before anything is placed on Kafka**; each valid record is then published to `save-boundary-relationship` for `egov-persister` to write. Individual record failures do **not** fail the whole request. +Creates many relationships in one request with a **per-record outcome**. Validation and enrichment happen synchronously on the request thread **before anything is placed on Kafka**; the valid records are then published together as one array message to the dedicated `boundary-relationship-bulk-create-job` topic for `egov-persister` to write. Individual record failures do **not** fail the whole request. ### Semantics - **Request guards.** `RequestInfo.userInfo` must be present, and the request must carry `1 … boundary.bulk.max.size` (default `100`) relationships. These are enforced in-service (bean validation is not active in this deployment) and return a structured `400` (`BULK_REQUEST_INFO_MISSING` / `BULK_REQUEST_EMPTY` / `BULK_REQUEST_SIZE_EXCEEDED`). - **Per-record validation.** Each record runs the same business rules as the single create (boundary entity exists, no duplicate, parent exists, correct hierarchy level; `code`/`tenantId`/`hierarchyType` must not contain the reserved `|` path delimiter). A record that fails validation is reported in `failedBoundaryRelationships`; the rest continue. -- **Persistence via egov-persister.** Validated + enriched records are published, **one message per record**, to `save-boundary-relationship`. The publish is blocking (it returns once the broker has accepted each record). `egov-persister` writes each with `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`, so redelivery / re-submission is a safe no-op and one un-insertable record never fails the others. A record accepted here is reported in `successfulBoundaryRelationships` ("accepted for persistence", not "already committed"). +- **Persistence via egov-persister.** The validated + enriched records are published as **ONE array message** (`BoundaryRelationship` is a JSON array, keyed by the batch's shared parent code) to the dedicated `boundary-relationship-bulk-create-job` topic. The publish is blocking (it returns once the broker has accepted the batch message). `egov-persister` maps that array (`$.BoundaryRelationship.*`) to N rows and writes them in a single `jdbcTemplate.batchUpdate` via `INSERT … ON CONFLICT (tenantId, code, hierarchyType) DO NOTHING`, so redelivery / re-submission is a safe no-op and an already-present (conflicting) row is skipped without failing the others. A record whose batch message is accepted here is reported in `successfulBoundaryRelationships` ("accepted for persistence", not "already committed"). - **Transient failures are retryable, not fatal.** A parent/entity not yet persisted, a transient DB error, or a publish failure (broker unreachable within `max.block.ms`) is reported with a retryable code (`PARENT_NOT_FOUND`, `BOUNDARY_ENTITY_DOES_NOT_EXIST`, `BULK_RELATIONSHIP_PERSIST_TRANSIENT`) rather than aborting; the caller retries only those records. - **Intra-request de-duplication.** Two records with the same `(tenantId, hierarchyType, code)` in one request are rejected (`DUPLICATE_RECORD_IN_REQUEST`). @@ -63,7 +63,7 @@ The endpoint expects each batch to be a set of **siblings whose parent is alread - **Horizontal scale** is by running more boundary-service replicas behind the load balancer (a stateless HTTP path) and by the caller submitting chunks concurrently — there is no Kafka consumer group to size and no per-partition head-of-line blocking. - **Idempotency** comes from `INSERT … ON CONFLICT (tenantid, code, hierarchytype) DO NOTHING`, so a parent that is briefly late simply causes its children to be retried by the caller until it lands; ordering across levels is *not* required. -- **DB-write reliability** (per-record isolation of an un-insertable row, transient-DB retry, dead-lettering) is owned by `egov-persister`. Because it is one message per record, a bad record is isolated on its own regardless of whether the persister runs its normal listener or the optional batch listener. +- **DB-write reliability** (transient-DB retry, dead-lettering) is owned by `egov-persister`. The batch is one array message written in a single `batchUpdate`; a row that already exists is isolated via `ON CONFLICT … DO NOTHING` — it is skipped without aborting the rest of the batch, and no per-record Kafka message or optional batch listener is involved. ### Request @@ -142,7 +142,7 @@ mvn clean install # Run (after configuring datasource & kafka in application.properties) mvn spring-boot:run # or -java -jar target/boundary-service-1.0.1.jar +java -jar target/boundary-service-1.0.2.jar ``` ### Key configuration (`src/main/resources/application.properties`) @@ -154,17 +154,20 @@ java -jar target/boundary-service-1.0.1.jar | `spring.flyway.enabled` | `false` | Enable to run DB migrations on startup | | `spring.kafka.bootstrap-servers` | `localhost:9092` | Kafka brokers used by the producer. In DIGIT deployments the common Helm chart injects `SPRING_KAFKA_BOOTSTRAP_SERVERS`. | | `spring.kafka.producer.properties.max.block.ms` | `15000` | Bounds how long a synchronous publish blocks if the broker is unreachable, so an outage fails fast (as a transient error the caller retries) instead of tying up request threads. | -| `kafka.topics.create.boundary.relationship` | `save-boundary-relationship` | Topic for **both** single and bulk relationship create, consumed by `egov-persister`. | +| `kafka.topics.create.boundary.relationship` | `save-boundary-relationship` | Topic for **single** relationship create only (single-object payload), consumed by `egov-persister`. | +| `kafka.topics.bulk.create.boundary.relationship.job` | `boundary-relationship-bulk-create-job` | **Dedicated** topic for `/bulk/_create`; carries one **array** message per batch (`$.BoundaryRelationship.*`), kept separate from the single-create topic because the array and single-object shapes cannot share a topic. | | `boundary.bulk.max.size` | `100` | Max records accepted by `/bulk/_create` (enforced in-service; keep ≥ the caller's chunk size). | | `boundary.default.limit` / `boundary.max.default.limit` | `50` / `300` | Search paging defaults | -> Bulk creation writes through `egov-persister`, not directly to PostgreSQL, so the request path is not gated by the DB connection pool for writes. To make the persister aggregate high-volume creates into batched multi-row inserts, add `save-boundary-relationship` to the persister's `persister.batch.topics` (with `persister.bulk.enabled=true`); this is an **optional throughput optimization** — the topic is otherwise consumed one record at a time by the persister's normal listener, so bulk creation works on any persister deployment with no extra configuration. +> Bulk creation writes through `egov-persister`, not directly to PostgreSQL, so the request path is not gated by the DB connection pool for writes. Batching is **within the one array message** — the persister maps `$.BoundaryRelationship.*` to N rows and writes them in a single `jdbcTemplate.batchUpdate` on its **normal** listener. It therefore needs **no** `persister.batch.topics` / `persister.bulk.enabled` configuration; the dedicated `boundary-relationship-bulk-create-job` topic plus the array queryMap does the batched multi-row insert on any persister deployment with no extra tuning. --- ## Database -Relationships are stored in `boundary_relationship` (`tenantId, code, hierarchyType` primary key; `ancestralMaterializedPath` holds the `|`-delimited ancestor chain used for subtree search). All writes go through the `egov-persister` mapping in `src/main/resources/boundary-persister.yml`, which uses `INSERT … ON CONFLICT (tenantid, code, hierarchytype) DO NOTHING` so at-least-once redelivery and caller re-submission are idempotent. +Relationships are stored in `boundary_relationship` (`tenantId, code, hierarchyType` primary key; `ancestralMaterializedPath` holds the `|`-delimited ancestor chain used for subtree search). All writes go through the `egov-persister` mapping in `src/main/resources/boundary-persister.yml`. + +That mapping has **two separate relationship queryMaps**: one for the single-create topic `save-boundary-relationship` (single-object base path `$.BoundaryRelationship`) and a dedicated one for the bulk topic `boundary-relationship-bulk-create-job` (array base path `$.BoundaryRelationship.*`, written as a single `batchUpdate`). Both, together with the boundary-entity and boundary-hierarchy inserts, are now idempotent — every insert uses `ON CONFLICT … DO NOTHING` (`(code, tenantId)` for `boundary`, `(tenantId, hierarchyType)` for `boundary_hierarchy`, `(tenantId, code, hierarchyType)` for `boundary_relationship`), so at-least-once redelivery and caller re-submission are safe no-ops. Bulk messages are keyed by the batch's shared parent code, so sibling batches under one parent stay ordered on the same partition. ### Search indexes diff --git a/core-services/boundary-service/pom.xml b/core-services/boundary-service/pom.xml index 33276f04667..c0818c0220b 100644 --- a/core-services/boundary-service/pom.xml +++ b/core-services/boundary-service/pom.xml @@ -4,7 +4,7 @@ boundary-service jar boundary-service - 1.0.1 + 1.0.2 17 ${java.version} diff --git a/core-services/egov-persister/CHANGELOG.md b/core-services/egov-persister/CHANGELOG.md index 9cc3eaf0a5f..5042c180203 100644 --- a/core-services/egov-persister/CHANGELOG.md +++ b/core-services/egov-persister/CHANGELOG.md @@ -2,6 +2,16 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-20 +- At-least-once delivery: manual offset commit (`spring.kafka.consumer.enable-auto-commit=false`), per-record (RECORD) ack on the single container and per-batch (BATCH) ack on the batch container — offsets commit only after durable handling +- SQLSTATE-based failure classification: benign duplicate (`unique_violation` 23505) treated as idempotent success, transient failures (connection/deadlock/serialization) retried in place, permanent/bad-data records routed to the dead-letter topic +- Dead-letter topic (`tracer.errorsTopic`, default `egov-persister-deadletter`) with a bounded reprocessor and a terminal parking topic (`egov-persister-deadletter-processed`); durable DLQ/parking publishes (`acks=all`, idempotent producer) +- DB-health pause/resume monitor: the single container is paused while the datasource is unreachable and resumed on recovery +- Per-record poison isolation: a failing bulk (bare JSON array) message is split so only the offending record is dead-lettered while its siblings commit +- Batch persist optimization: rows aggregated per QueryMap across all messages into a single order-preserving `batchUpdate` +- Idempotent service configs: added `ON CONFLICT (uuid) DO NOTHING` to inserts to support safe redelivery / DLQ replay +- New config keys: `persister.batch.topics`, `persister.dead-letter.*`, `persister.db-health.check-interval-ms`, `persister.custom.executor.*`, `persister.batch.parallel-topic-processing.thread-pool-size`, and the live-read `persister.kafka.*` consumer tuning knobs + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-persister/README.md b/core-services/egov-persister/README.md index 381a3316323..5efc73b65f9 100644 --- a/core-services/egov-persister/README.md +++ b/core-services/egov-persister/README.md @@ -75,8 +75,39 @@ The bulk persister have the following two settings: |-------------------------|---------------|-------------------------------------------------| | `persister.bulk.enabled`| false | Switch to turn on or off the bulk kafka consumer| | `persister.batch.size` | 100 | The batch size for bulk update | +| `persister.batch.topics`| (empty) | Comma-separated topics to force through the batch consumer, in addition to any topic whose name contains '-batch' | -Any kafka topic containing data which has to be bulk persisted should have '-batch' appended at the end of topic name example: save-pt-assessment-batch +Any kafka topic containing data which has to be bulk persisted should have '-batch' appended at the end of topic name example: save-pt-assessment-batch. Alternatively, topics can be routed through the batch consumer without renaming them by listing them (comma-separated) in `persister.batch.topics`. Batch topics (both '-batch' named and those in `persister.batch.topics`) are only excluded from the single (record) consumer when `persister.bulk.enabled=true`. + +### Reliability: at-least-once, DB-health pause, dead-letter & parking + +The persister is an at-least-once, poison-tolerant writer. Key operational behaviour: + +1. **At-least-once delivery.** Auto-commit is turned off (`spring.kafka.consumer.enable-auto-commit=false`). Both listeners commit manually — the single (record) consumer per record (`RECORD` ack), the batch consumer per batch (`BATCH` ack). An offset is committed only after the record has been persisted or durably dead-lettered, so a crash mid-processing results in redelivery, never silent loss. +2. **Failure classification by PostgreSQL SQLSTATE.** Database failures are classified into three buckets: + - **BENIGN** — `unique_violation` (23505) is treated as an idempotent success (the row is already there from a prior at-least-once delivery). + - **TRANSIENT** — connection/deadlock/serialization failures (e.g. `08*`, `57*`, `40001`, `40P01`, `53300`, `55P03`, connection-acquisition) are retried in place with back-off and are never dead-lettered. + - **PERMANENT** — bad-data failures are routed to the dead-letter topic. +3. **DB-health pause/resume.** While the datasource is unreachable the single container is paused, and resumed on recovery, so transient outages retry in place instead of hammering a dead DB. The poll interval is `persister.db-health.check-interval-ms` (default `5000` ms). +4. **Per-record poison isolation.** When a bulk (bare JSON array) message fails, it is split into single-record inserts so only the offending record is dead-lettered while its good siblings still commit. +5. **Idempotent writes.** Service persister configs use `ON CONFLICT (uuid) DO NOTHING` on inserts so that redelivery / dead-letter replay is safe. + +**Reliability configuration** + +| variable name | Default value | Description | +|------------------------------------------------------------|--------------------------------------|--------------------------------------------------------------------------------------| +| `spring.kafka.consumer.enable-auto-commit` | false | Manual offset commit; offsets commit only after durable handling (at-least-once) | +| `persister.db-health.check-interval-ms` | 5000 | Interval for the DB-health pause/resume monitor | +| `persister.dead-letter.reprocess.enabled` | true | Re-consume the dead-letter topic on the single listener and retry failed records | +| `persister.dead-letter.reprocess.error-topic` | egov-persister-deadletter-processed | Terminal parking topic for records that exhaust retries | +| `persister.dead-letter.max-retries` | 5 | Bounded DLQ retries before a record is parked | +| `persister.custom.executor.enabled` | false | Optional listener task executor for the single container (off by default) | +| `persister.custom.executor.max-pool-size` | 10 | Max pool size for the optional listener task executor | +| `persister.batch.parallel-topic-processing.thread-pool-size` | 1 | Thread pool for parallel per-topic processing inside a batch | +| `spring.kafka.producer.acks` | all | Durable, no-loss dead-letter / parking publishes | +| `spring.kafka.producer.properties.enable.idempotence` | true | Idempotent producer for dead-letter / parking publishes | + +The following consumer tuning keys are read live and are not written in `application.properties`: `persister.kafka.partition.assignment.strategy` (default `CooperativeStickyAssignor,RangeAssignor`), `persister.kafka.group.instance.id` (Kafka static membership), and `persister.kafka.session.timeout.ms`. ### Persister Config Versioning @@ -89,7 +120,19 @@ Any kafka topic containing data which has to be bulk persisted should have '-bat ### Kafka Consumers - From the Kafka topic which are mentioned in the persister config, persister service get message/data and push the data into the particular tables of the database. +- When `persister.dead-letter.reprocess.enabled=true`, the single listener additionally re-consumes the dead-letter topic (`tracer.errorsTopic`, default `egov-persister-deadletter`) to retry previously failed records. ### Kafka Producers -- NA +- The persister produces to a **dead-letter topic** (`tracer.errorsTopic`, default `egov-persister-deadletter`) when a record cannot be persisted (permanent/bad-data failure, or a split poison record from a bulk message). +- When `persister.dead-letter.reprocess.enabled=true`, the single listener re-consumes the dead-letter topic and retries each record up to `persister.dead-letter.max-retries` (default `5`). Records that exhaust their retries are produced to a **terminal parking topic** (`persister.dead-letter.reprocess.error-topic`, default `egov-persister-deadletter-processed`). +- Dead-letter and parking publishes are durable (`spring.kafka.producer.acks=all`, `spring.kafka.producer.properties.enable.idempotence=true`). + +### Dead-letter & parking topics + +| topic | property | Purpose | +|----------------------------------------|-------------------------------------------------|----------------------------------------------------------------| +| `egov-persister-deadletter` | `tracer.errorsTopic` | Dead-letter topic for failed records; re-consumed on retry | +| `egov-persister-deadletter-processed` | `persister.dead-letter.reprocess.error-topic` | Terminal parking topic for records that exhaust their retries | + +> Parking-topic growth is the terminal-poison signal — monitor it (and dead-letter lag) in ops. diff --git a/core-services/egov-persister/pom.xml b/core-services/egov-persister/pom.xml index cd599e83aae..002f2fc2dcd 100644 --- a/core-services/egov-persister/pom.xml +++ b/core-services/egov-persister/pom.xml @@ -9,7 +9,7 @@ org.egov egov-persister - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-persister egov persister framework diff --git a/core-services/libraries/tracer/readme.md b/core-services/libraries/tracer/readme.md index d03d20bf86e..0f763b4fc7b 100644 --- a/core-services/libraries/tracer/readme.md +++ b/core-services/libraries/tracer/readme.md @@ -41,20 +41,48 @@ Map map = new HashMap<>(); The logging of the http request/response body and Kakfa message body can be toggled on/off using "tracer.detailed.tracing.enabled" application property. +The per-record Kafka MDC rebuild + cleanup (see "Setting the correlation id in the MDC") can be +toggled on/off using the "tracer.kafka.mdc.enabled" application property. It defaults to true; set it +to false to disable registration of the record interceptor. + ###### Correlation id retrieval and forwarding - The library takes care of retrieving the correlation id from - - Incoming http request body or header -- Kafka message payload +- Kafka message headers (with a fallback to the message payload) For an outgoing http request the correlation id is sent as a custom request header "x-correlation-id". +As of version 2.9.3 both the correlation id and the tenant id are also propagated across Kafka via +message headers - + +- On produce, the correlation id and tenant id are stamped from the MDC onto the outgoing message + headers "x-correlation-id" (CORRELATION_ID_HEADER) and "tenantId" (TENANT_ID_HEADER), only when + present in the MDC and not already set on the record. +- On consume, the MDC is rebuilt from these headers. The correlation id falls back to the message + payload (RequestInfo.correlationId / requestInfo.correlationId) when the header is absent. + +This means consumer and downstream async logs carry the same correlation id and tenant id as the +originating request. + ###### Setting the correlation id in the MDC - Given the library takes care of placing the correlation id into the MDC, any custom logging done in the application would seamlessly include the correlation id in the log message. +For Kafka consumers, as of version 2.9.3 the MDC is managed per record by a RecordInterceptor +(MdcRecordInterceptor). Before each record is processed the correlation id and tenant id are rebuilt +into the MDC from the record headers, and after the record completes (on both success and failure) +both MDC keys are cleared. This per-record cleanup prevents the ids from leaking across records when +listener threads are reused. + +Note - The mdcRecordInterceptor bean is registered by TracerConfiguration, but Spring Boot only +auto-attaches it to its autoconfigured record-listener container factory. Services that define a +custom ConcurrentKafkaListenerContainerFactory or use batch listeners must wire it manually by calling +setRecordInterceptor(mdcRecordInterceptor()) on their factory; otherwise their consumer logs will not +carry the propagated correlation id and tenant id. + Note - See the "logging.pattern" mentioned in the "Tracer integration" section. #### Steps to integrate Tracer to your Spring application - @@ -114,10 +142,14 @@ Note - See the "logging.pattern" mentioned in the "Tracer integration" section. logging. - The LogAwareKafkaTemplate is a wrapper Spring bean class for Spring Kafka's KafkaTemplate that performs the logging of messages sent to Kafka. -- For a Kafka consumer implemented using Spring Kafka's KafkaListener annotation an AspectJ's aspect is used to log and - retrieve the correlation id from the received payload. -- The correlation id retrieved via the filter or aspect is then stored in a thread local variable to - forward as necessary. +- For Kafka, a producer/consumer interceptor (KafkaTemplateLoggingInterceptors) performs the logging of messages + sent and received. As of version 2.9.3, on produce it stamps the correlation id and tenant id from the MDC onto the + message headers ("x-correlation-id" and "tenantId"), and on consume it rebuilds the MDC from those headers (with a + fallback to the payload for the correlation id) via the TracerKafkaMdcUtil helper. +- For Kafka consumers, a RecordInterceptor (MdcRecordInterceptor) rebuilds the MDC (correlation id + tenant id) before + each record and clears both keys afterwards, so ids do not leak across records on reused listener threads. +- The correlation id retrieved via the filter (for http) or the Kafka headers (for consumers) is placed into the MDC + to forward as necessary. #### Change log - From fe334d2c304d494cb427f2dbdd43aa6f4102ca21 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Wed, 22 Jul 2026 18:57:35 +0530 Subject: [PATCH 17/24] Record Splitting updated for non bare array --- .../persist/consumer/RecordSplitter.java | 96 ++++++++++++++++--- .../persist/consumer/RecordSplitterTest.java | 44 ++++++++- 2 files changed, 121 insertions(+), 19 deletions(-) diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java index 461998b2e5e..afdece6d274 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/RecordSplitter.java @@ -3,20 +3,36 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Splits a multi-record bulk message into standalone single-record messages so a permanent failure * can be isolated to the offending record(s) instead of failing every record that happened to share * the same Kafka message (R1 at record granularity, not just message granularity). * - *

Bulk producers publish a whole validated list as ONE message whose payload is a bare JSON - * array, and the persister maps such topics with array base paths ({@code $.*}), so a - * single-element array is processed identically to the full array. Only bare arrays with more than - * one element are split; object-shaped payloads (whose base paths may not be per-element) keep - * message-level handling.

+ *

Two payload shapes are split:

+ *
    + *
  • bare array — {@code [ {..}, {..} ]} (base path {@code $.*}): re-emitted as one + * single-element array per record.
    • + *
  • object with one record array — {@code { "Boundary": [ {..}, {..} ] }} (base path + * {@code $.Boundary.*}): the message is deep-copied and only that array is replaced with a + * single element, so sibling keys (e.g. {@code RequestInfo}) are preserved on the re-persist.
    • + *
+ * + *

The object case only fires when EXACTLY ONE top-level field is a multi-element array AND every + * element of that array is a JSON object. The single-array rule makes the record list unambiguous; + * the objects-only rule keeps the splitter off list-valued COLUMNS of a single whole-message record + * (e.g. {@code {"entityIds":["e1","e2"],...}} persisted with base path {@code $}), whose truncation + * would corrupt the record instead of isolating it. Anything else — not JSON, a single element, zero + * or several top-level array fields, non-object elements, or the array nested deeper — is left + * unsplit and the caller keeps message-level handling. Residual blind spot: an object-element array + * that is itself a column of a whole-message-mapped record is structurally indistinguishable from a + * record list; no current persister mapping has that shape (a mapping-driven splitter is required if + * one ever does).

*/ final class RecordSplitter { @@ -27,9 +43,9 @@ private RecordSplitter() { } /** - * @return one single-element-array JSON string per record, or null when the payload is not a - * multi-record bare array (not JSON / an object / an array of 0..1 elements) and the - * caller must keep message-level handling. + * @return one single-record JSON string per element, or null when the payload is not a + * multi-record array (bare or under a single object key) and the caller must keep + * message-level handling. */ static List split(String json) { if (json == null) { @@ -37,18 +53,68 @@ static List split(String json) { } try { JsonNode root = MAPPER.readTree(json); - if (root == null || !root.isArray() || root.size() <= 1) { + if (root == null) { return null; } - List records = new ArrayList<>(root.size()); - for (JsonNode element : root) { - ArrayNode single = MAPPER.createArrayNode(); - single.add(element); - records.add(MAPPER.writeValueAsString(single)); + if (root.isArray()) { + return root.size() <= 1 ? null : splitArray(root, null, null); + } + if (root.isObject()) { + String field = singleMultiElementArrayField((ObjectNode) root); + if (field != null) { + return splitArray((ArrayNode) root.get(field), (ObjectNode) root, field); + } } - return records; + return null; } catch (Exception e) { return null; } } + + /** + * Emit one message per element. When {@code wrapper} is null the record is a bare single-element + * array; otherwise the wrapper object is deep-copied and its {@code field} set to a single-element + * array, preserving every sibling key. + */ + private static List splitArray(JsonNode array, ObjectNode wrapper, String field) throws Exception { + List records = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ArrayNode single = MAPPER.createArrayNode().add(element); + if (wrapper == null) { + records.add(MAPPER.writeValueAsString(single)); + } else { + ObjectNode clone = wrapper.deepCopy(); + clone.set(field, single); + records.add(MAPPER.writeValueAsString(clone)); + } + } + return records; + } + + /** + * The single top-level field holding a multi-element array whose elements are all JSON objects, + * or null when no field is unambiguously the record list (zero or several top-level array fields, + * an array of 0..1 elements, or any non-object element — a scalar list is a column of one record, + * not a record list). + */ + private static String singleMultiElementArrayField(ObjectNode root) { + String found = null; + for (Map.Entry field : root.properties()) { + if (field.getValue().isArray()) { + if (found != null) { + return null; // more than one array field -> ambiguous, do not split + } + found = field.getKey(); + } + } + if (found == null || root.get(found).size() <= 1) { + return null; + } + for (JsonNode element : root.get(found)) { + if (!element.isObject()) { + return null; // scalar/mixed elements -> a list-valued column, not a record list + } + } + return found; + } } diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java index 361570a3cea..17b5dc46ba4 100644 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/RecordSplitterTest.java @@ -30,10 +30,46 @@ void emptyArrayIsNotSplit() { } @Test - void objectPayloadIsNotSplit() { - // Object-shaped payloads (e.g. {"RequestInfo":...,"Entity":{...}}) may not map per-element - - // they must keep message-level handling. - assertNull(RecordSplitter.split("{\"RequestInfo\":{},\"records\":[{\"id\":\"a\"},{\"id\":\"b\"}]}")); + void objectWithSingleRecordArraySplitsPreservingSiblingKeys() { + // {"RequestInfo":{...},"Boundary":[a,b]} -> one message per element, RequestInfo preserved. + List records = + RecordSplitter.split("{\"RequestInfo\":{\"ver\":\"1.0\"},\"Boundary\":[{\"id\":\"a\"},{\"id\":\"b\"}]}"); + + assertEquals(2, records.size()); + assertEquals("{\"RequestInfo\":{\"ver\":\"1.0\"},\"Boundary\":[{\"id\":\"a\"}]}", records.get(0)); + assertEquals("{\"RequestInfo\":{\"ver\":\"1.0\"},\"Boundary\":[{\"id\":\"b\"}]}", records.get(1)); + } + + @Test + void objectWithSingleElementArrayIsNotSplit() { + assertNull(RecordSplitter.split("{\"Boundary\":[{\"id\":\"a\"}]}")); + } + + @Test + void objectWithMultipleArrayFieldsIsNotSplit() { + // Ambiguous which array is the record list -> keep message-level handling. + assertNull(RecordSplitter.split("{\"Boundary\":[{\"id\":\"a\"},{\"id\":\"b\"}],\"extra\":[{\"x\":1}]}")); + } + + @Test + void objectWithScalarArrayIsNotSplit() { + // A scalar list is a list-valued COLUMN of one whole-message record (e.g. privacy-audit + // enc-user-audit-info entityIds, base path $), not a record list - splitting would truncate + // the record's column and duplicate its key. Must keep message-level handling. + assertNull(RecordSplitter.split( + "{\"id\":\"u1\",\"userId\":\"9\",\"entityIds\":[\"e1\",\"e2\",\"e3\"],\"purpose\":{\"code\":\"kyc\"}}")); + } + + @Test + void objectWithMixedElementArrayIsNotSplit() { + // Every element must be a JSON object for the array to count as a record list. + assertNull(RecordSplitter.split("{\"records\":[{\"id\":\"a\"},\"stray\",{\"id\":\"b\"}]}")); + } + + @Test + void objectWithNoTopLevelArrayIsNotSplit() { + // Array nested deeper (e.g. $.bill.billDetails.*) is not reached -> message-level handling. + assertNull(RecordSplitter.split("{\"bill\":{\"billDetails\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}")); } @Test From 0a4c80ac7e3baa164aa0eb075ea16dbd4d2ff9b9 Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Thu, 23 Jul 2026 15:26:05 +0530 Subject: [PATCH 18/24] tracer changes 2.9.3 version updated for boundary-service and persister --- core-services/egov-persister/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-services/egov-persister/pom.xml b/core-services/egov-persister/pom.xml index 002f2fc2dcd..dd142e84bad 100644 --- a/core-services/egov-persister/pom.xml +++ b/core-services/egov-persister/pom.xml @@ -89,7 +89,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.flywaydb From ad568fb50a46800db02143453fea3ffd1891b9f6 Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Thu, 23 Jul 2026 15:27:13 +0530 Subject: [PATCH 19/24] tracer changes 2.9.3 version updated for boundary-service --- core-services/boundary-service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-services/boundary-service/pom.xml b/core-services/boundary-service/pom.xml index c0818c0220b..5c6a65a318d 100644 --- a/core-services/boundary-service/pom.xml +++ b/core-services/boundary-service/pom.xml @@ -90,7 +90,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.egov.services From c287a5d15b930c20da2db7354f5f39d68e104848 Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Mon, 27 Jul 2026 12:07:59 +0530 Subject: [PATCH 20/24] tracer version update from 2.9.2 to 2.9.3 in core services. --- core-services/egov-accesscontrol/CHANGELOG.md | 3 +++ core-services/egov-accesscontrol/pom.xml | 4 ++-- core-services/egov-enc-service/CHANGELOG.md | 3 +++ core-services/egov-enc-service/pom.xml | 4 ++-- core-services/egov-filestore/CHANGELOG.md | 3 +++ core-services/egov-filestore/pom.xml | 4 ++-- core-services/egov-indexer/CHANGELOG.md | 3 +++ core-services/egov-indexer/pom.xml | 4 ++-- core-services/egov-localization/CHANGELOG.md | 3 +++ core-services/egov-localization/pom.xml | 4 ++-- core-services/egov-mdms-service/CHANGELOG.md | 3 +++ core-services/egov-mdms-service/pom.xml | 4 ++-- core-services/egov-persister/CHANGELOG.md | 3 +++ core-services/egov-persister/pom.xml | 4 ++-- core-services/egov-url-shortening/CHANGELOG.md | 3 +++ core-services/egov-url-shortening/pom.xml | 4 ++-- core-services/egov-workflow-v2/CHANGELOG.md | 3 +++ core-services/egov-workflow-v2/pom.xml | 4 ++-- core-services/mdms-v2/CHANGELOG.md | 3 +++ core-services/mdms-v2/pom.xml | 4 ++-- 20 files changed, 50 insertions(+), 20 deletions(-) diff --git a/core-services/egov-accesscontrol/CHANGELOG.md b/core-services/egov-accesscontrol/CHANGELOG.md index 0c0e46d240b..4f936754f59 100644 --- a/core-services/egov-accesscontrol/CHANGELOG.md +++ b/core-services/egov-accesscontrol/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-accesscontrol/pom.xml b/core-services/egov-accesscontrol/pom.xml index 7d5bbaad3f2..f98dfae9cb4 100644 --- a/core-services/egov-accesscontrol/pom.xml +++ b/core-services/egov-accesscontrol/pom.xml @@ -5,7 +5,7 @@ org.egov egov-accesscontrol - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT jar egov-accesscontrol @@ -57,7 +57,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT diff --git a/core-services/egov-enc-service/CHANGELOG.md b/core-services/egov-enc-service/CHANGELOG.md index 5bc6b351043..1e35728c2ba 100644 --- a/core-services/egov-enc-service/CHANGELOG.md +++ b/core-services/egov-enc-service/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-enc-service/pom.xml b/core-services/egov-enc-service/pom.xml index 4d094a46ab4..8f7629e894b 100644 --- a/core-services/egov-enc-service/pom.xml +++ b/core-services/egov-enc-service/pom.xml @@ -9,7 +9,7 @@ org.egov egov-enc-service - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-enc-service 17 @@ -71,7 +71,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.projectlombok diff --git a/core-services/egov-filestore/CHANGELOG.md b/core-services/egov-filestore/CHANGELOG.md index d725d85a0db..3d4f8c8517f 100644 --- a/core-services/egov-filestore/CHANGELOG.md +++ b/core-services/egov-filestore/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-filestore/pom.xml b/core-services/egov-filestore/pom.xml index c938b5c9c6f..93f6ddb2ebf 100644 --- a/core-services/egov-filestore/pom.xml +++ b/core-services/egov-filestore/pom.xml @@ -12,7 +12,7 @@ org.egov egov-filestore - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-filestore eGov File store project for eGov services @@ -53,7 +53,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.springframework.boot diff --git a/core-services/egov-indexer/CHANGELOG.md b/core-services/egov-indexer/CHANGELOG.md index 94968009bea..c11f55e746b 100644 --- a/core-services/egov-indexer/CHANGELOG.md +++ b/core-services/egov-indexer/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-indexer/pom.xml b/core-services/egov-indexer/pom.xml index f19aeb18d57..70ca035bad1 100644 --- a/core-services/egov-indexer/pom.xml +++ b/core-services/egov-indexer/pom.xml @@ -10,7 +10,7 @@ org.egov egov-indexer - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-indexer egov indexer framework http://maven.apache.org @@ -54,7 +54,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.springframework.boot diff --git a/core-services/egov-localization/CHANGELOG.md b/core-services/egov-localization/CHANGELOG.md index cef8652ac11..110e26339ef 100644 --- a/core-services/egov-localization/CHANGELOG.md +++ b/core-services/egov-localization/CHANGELOG.md @@ -2,6 +2,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-localization/pom.xml b/core-services/egov-localization/pom.xml index 5e26260ba0b..ed292989fec 100644 --- a/core-services/egov-localization/pom.xml +++ b/core-services/egov-localization/pom.xml @@ -9,7 +9,7 @@ org.egov egov-localization - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-localization Localization for messages @@ -91,7 +91,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.springframework.boot diff --git a/core-services/egov-mdms-service/CHANGELOG.md b/core-services/egov-mdms-service/CHANGELOG.md index 3d6e96fec74..33b43e20526 100644 --- a/core-services/egov-mdms-service/CHANGELOG.md +++ b/core-services/egov-mdms-service/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.2 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.1 - 2025-05-21 - Upgraded tracer version from 2.9.0 to 2.9.1 - added variables in application.properties required for opentelemetry diff --git a/core-services/egov-mdms-service/pom.xml b/core-services/egov-mdms-service/pom.xml index 1eee10a5383..0981ed0e01d 100644 --- a/core-services/egov-mdms-service/pom.xml +++ b/core-services/egov-mdms-service/pom.xml @@ -9,7 +9,7 @@ org.egov.mdms egov-mdms-service-test - 2.9.1-SNAPSHOT + 2.9.2-SNAPSHOT egov-infra-mdms-service http://maven.apache.org @@ -74,7 +74,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.egov diff --git a/core-services/egov-persister/CHANGELOG.md b/core-services/egov-persister/CHANGELOG.md index 9cc3eaf0a5f..d0e332c7984 100644 --- a/core-services/egov-persister/CHANGELOG.md +++ b/core-services/egov-persister/CHANGELOG.md @@ -2,6 +2,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-persister/pom.xml b/core-services/egov-persister/pom.xml index cd599e83aae..dd142e84bad 100644 --- a/core-services/egov-persister/pom.xml +++ b/core-services/egov-persister/pom.xml @@ -9,7 +9,7 @@ org.egov egov-persister - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT egov-persister egov persister framework @@ -89,7 +89,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT org.flywaydb diff --git a/core-services/egov-url-shortening/CHANGELOG.md b/core-services/egov-url-shortening/CHANGELOG.md index 85ec23be0f1..d46914c88a0 100644 --- a/core-services/egov-url-shortening/CHANGELOG.md +++ b/core-services/egov-url-shortening/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-url-shortening/pom.xml b/core-services/egov-url-shortening/pom.xml index 018d0aaa6af..ef575798263 100644 --- a/core-services/egov-url-shortening/pom.xml +++ b/core-services/egov-url-shortening/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.egov egov-url-shortening - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT org.springframework.boot spring-boot-starter-parent @@ -72,7 +72,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT diff --git a/core-services/egov-workflow-v2/CHANGELOG.md b/core-services/egov-workflow-v2/CHANGELOG.md index 88d3362ecfe..38099089bae 100644 --- a/core-services/egov-workflow-v2/CHANGELOG.md +++ b/core-services/egov-workflow-v2/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 2.9.4 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 2.9.3 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common, mdms-client, enc-client library versions diff --git a/core-services/egov-workflow-v2/pom.xml b/core-services/egov-workflow-v2/pom.xml index f3ccbd1f7d0..6a9b971c5b1 100644 --- a/core-services/egov-workflow-v2/pom.xml +++ b/core-services/egov-workflow-v2/pom.xml @@ -4,7 +4,7 @@ egov-workflow-v2 jar egov-workflow-v2 - 2.9.3-SNAPSHOT + 2.9.4-SNAPSHOT 3.1.1 17 @@ -69,7 +69,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT diff --git a/core-services/mdms-v2/CHANGELOG.md b/core-services/mdms-v2/CHANGELOG.md index 9a9cc2d77ea..daac3bad473 100644 --- a/core-services/mdms-v2/CHANGELOG.md +++ b/core-services/mdms-v2/CHANGELOG.md @@ -3,6 +3,9 @@ # Changelog All notable changes to this module will be documented in this file. +## 1.3.6 - 2026-07-27 +- Bumped tracer to 2.9.3-SNAPSHOT for end-to-end correlationId + tenantId propagation across HTTP↔Kafka (HCM flow tracing) + ## 1.3.5 - 2026-03-16 - Upgraded Spring Boot version from 3.2.2 to 3.4.5 to fix HIGH/CRITICAL CVEs - Upgraded tracer, services-common library versions diff --git a/core-services/mdms-v2/pom.xml b/core-services/mdms-v2/pom.xml index 5e33c67ee43..7b881579560 100644 --- a/core-services/mdms-v2/pom.xml +++ b/core-services/mdms-v2/pom.xml @@ -11,7 +11,7 @@ org.egov.mdms egov-mdms-service-test - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT egov-infra-mdms-service http://maven.apache.org @@ -62,7 +62,7 @@ org.egov.services tracer - 2.9.2-SNAPSHOT + 2.9.3-SNAPSHOT From 879ac5397b7835e4c21443346553d26712bac737 Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Wed, 29 Jul 2026 17:57:08 +0530 Subject: [PATCH 21/24] Restricting characters for hierarchy creation --- .../main/java/digit/errors/ErrorCodes.java | 4 + .../validator/BoundaryHierarchyValidator.java | 90 +++++++++++++ .../BoundaryHierarchyTypeSeparatorTest.java | 123 ++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 core-services/boundary-service/src/test/java/digit/service/validator/BoundaryHierarchyTypeSeparatorTest.java diff --git a/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java b/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java index f9efbfa76f0..67377d4cc41 100644 --- a/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java +++ b/core-services/boundary-service/src/main/java/digit/errors/ErrorCodes.java @@ -44,5 +44,9 @@ public class ErrorCodes { public static final String BULK_REQUEST_INFO_MISSING_MSG = "Bulk boundary relationship request is missing RequestInfo.userInfo."; public static final String INVALID_BOUNDARY_CODE_CODE = "INVALID_BOUNDARY_CODE"; public static final String INVALID_BOUNDARY_CODE_MSG = "code, tenantId and hierarchyType must not contain the '|' character, which is reserved as the ancestral materialized-path delimiter."; + // Distinct code (not reused from INVALID_HIERARCHY_DEFINITION) so the UI can render a specific, + // actionable message for this case rather than a generic hierarchy-definition failure. + public static final String INVALID_HIERARCHY_TYPE_SEPARATOR_CODE = "INVALID_HIERARCHY_TYPE_SEPARATOR"; + public static final String INVALID_HIERARCHY_TYPE_SEPARATOR_MSG = "hierarchyType must not contain separator characters. The characters '.', ':', '-', '/', '_' and any whitespace are all normalized to '_' when deriving the localisation module name and the boundary code prefix, so two hierarchy types differing only by these characters would silently collide. Offending character(s): "; } diff --git a/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryHierarchyValidator.java b/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryHierarchyValidator.java index 460799ddfa0..02985436092 100644 --- a/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryHierarchyValidator.java +++ b/core-services/boundary-service/src/main/java/digit/service/validator/BoundaryHierarchyValidator.java @@ -12,12 +12,38 @@ import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Component public class BoundaryHierarchyValidator { + /** + * Characters that must not appear in a hierarchyType. + * + * Downstream (boundary-management getTransformedLocale, project-factory) every one of these is + * rewritten to '_' when deriving the localisation module name (hcm-boundary-<type>) and the + * boundary code prefix. Two hierarchy types differing only by these characters therefore collapse + * onto the SAME module and the SAME code prefix - e.g. "MODFIX-A", "MODFIX_A" and "MODFIX A" all + * become "modfix_a" - and silently overwrite each other's data. '_' itself is included because it + * is the normalization target, so it collides with all the others. + * + * The set is written out explicitly rather than using \s: Java's \s is ASCII-only, and + * Character.isWhitespace() returns false for NBSP (U+00A0), figure space (U+2007) and narrow NBSP + * (U+202F). This class mirrors JavaScript's \s exactly, so server and client agree character for + * character. + */ + private static final Pattern FORBIDDEN_HIERARCHY_TYPE_CHARS = Pattern.compile( + "[.:\\-/_" + + "\\u0009\\u000A\\u000B\\u000C\\u000D\\u0020" + // tab, LF, VT, FF, CR, space + "\\u00A0\\u1680\\u2000-\\u200A" + // NBSP, ogham, en/em/thin/hair spaces + "\\u2028\\u2029\\u202F\\u205F\\u3000\\uFEFF" + // line/para sep, narrow NBSP, math, ideographic, BOM + "]"); + private BoundaryHierarchyRepository boundaryHierarchyRepository; @Autowired @@ -31,6 +57,10 @@ public BoundaryHierarchyValidator(BoundaryHierarchyRepository boundaryHierarchyR */ public void validateBoundaryTypeHierarchy(BoundaryTypeHierarchyRequest body) { + // Validate that the hierarchy type carries no separator character that would collide + // downstream. Runs first so a bad name is rejected before any DB lookup. + validateHierarchyTypeSeparators(body); + // Validate if only single root node exists validateIfSingleRootNodeExists(body); @@ -73,6 +103,66 @@ private void validateIfBoundaryHierarchyFormsDAG(BoundaryTypeHierarchyRequest bo }); } + /** + * Rejects a hierarchyType containing any character that is normalized to '_' downstream, plus '_' + * itself. Without this, "MODFIX-A" and "MODFIX_A" are accepted as two distinct hierarchies but + * share one localisation module and one boundary code prefix, so the second silently overwrites + * the first. Verified live: three such hierarchies produced 30/30 identical boundary codes. + * + * Thrown with its own error code so the UI can show a specific message; the offending characters + * are named in the message (with code points, since several are invisible - NBSP pasted from + * Word/Excel being the common one). + * @param body + */ + private void validateHierarchyTypeSeparators(BoundaryTypeHierarchyRequest body) { + + String hierarchyType = body.getBoundaryHierarchy().getHierarchyType(); + + if (ObjectUtils.isEmpty(hierarchyType)) + return; + + // Collect every distinct offending character so the caller can fix them all in one go + // rather than resubmitting once per bad character. + Set offenders = new LinkedHashSet<>(); + Matcher matcher = FORBIDDEN_HIERARCHY_TYPE_CHARS.matcher(hierarchyType); + while (matcher.find()) { + char offender = matcher.group().charAt(0); + offenders.add(String.format("'%s' (U+%04X)", describe(offender), (int) offender)); + } + + if (!CollectionUtils.isEmpty(offenders)) { + throw new CustomException(ErrorCodes.INVALID_HIERARCHY_TYPE_SEPARATOR_CODE, + ErrorCodes.INVALID_HIERARCHY_TYPE_SEPARATOR_MSG + String.join(", ", offenders)); + } + } + + /** + * Renders a character for an error message. Invisible characters have no useful glyph, so they + * are named instead - an operator seeing "'-' (U+002D)" can act, but a bare NBSP looks like a + * space and reads as a service bug. + */ + private String describe(char c) { + switch (c) { + case '\t': return "tab"; + case '\n': return "line feed"; + case '\u000B': return "vertical tab"; + case '\f': return "form feed"; + case '\r': return "carriage return"; + case '\u0020': return "space"; + case '\u00A0': return "non-breaking space"; + case '\u2007': return "figure space"; + case '\u202F': return "narrow non-breaking space"; + case '\u2028': return "line separator"; + case '\u2029': return "paragraph separator"; + case '\u3000': return "ideographic space"; + case '\uFEFF': return "zero-width no-break space"; + default: + // Remaining matches are either visible punctuation or one of the Unicode space + // separators, which have no distinct name worth spelling out individually. + return Character.isSpaceChar(c) ? "space separator" : String.valueOf(c); + } + } + /** * This method validates if only a single root node has been defined in hierarchy definition. * @param body diff --git a/core-services/boundary-service/src/test/java/digit/service/validator/BoundaryHierarchyTypeSeparatorTest.java b/core-services/boundary-service/src/test/java/digit/service/validator/BoundaryHierarchyTypeSeparatorTest.java new file mode 100644 index 00000000000..d7617934208 --- /dev/null +++ b/core-services/boundary-service/src/test/java/digit/service/validator/BoundaryHierarchyTypeSeparatorTest.java @@ -0,0 +1,123 @@ +package digit.service.validator; + +import digit.errors.ErrorCodes; +import digit.repository.BoundaryHierarchyRepository; +import digit.web.models.BoundaryTypeHierarchy; +import digit.web.models.BoundaryTypeHierarchyDefinition; +import digit.web.models.BoundaryTypeHierarchyRequest; +import org.egov.tracer.model.CustomException; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Covers the hierarchyType separator guard: every character that getTransformedLocale rewrites to + * '_' (plus '_' itself) must be rejected, and nothing else may be. + */ +class BoundaryHierarchyTypeSeparatorTest { + + private BoundaryHierarchyValidator validator() { + BoundaryHierarchyRepository repo = mock(BoundaryHierarchyRepository.class); + // Not reached for the rejection cases; empty so accepted names fall through cleanly. + when(repo.search(org.mockito.ArgumentMatchers.any())).thenReturn(Collections.emptyList()); + return new BoundaryHierarchyValidator(repo); + } + + private BoundaryTypeHierarchyRequest requestFor(String hierarchyType) { + BoundaryTypeHierarchy node = BoundaryTypeHierarchy.builder() + .boundaryType("COUNTRY") + .parentBoundaryType(null) + .active(Boolean.TRUE) + .build(); + BoundaryTypeHierarchyDefinition definition = BoundaryTypeHierarchyDefinition.builder() + .tenantId("dev") + .hierarchyType(hierarchyType) + .boundaryHierarchy(List.of(node)) + .build(); + return BoundaryTypeHierarchyRequest.builder() + .boundaryHierarchy(definition) + .build(); + } + + private static final char[] MUST_REJECT = { + '.', ':', '-', '/', '_', '\t', + '\n', '\u000B', '\f', '\r', '\u0020', '\u00A0', + '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', + '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', + '\u2028', '\u2029', '\u202F', '\u205F', '\u3000', '\uFEFF' + }; + + @Test + void rejectsEveryCharacterThatNormalizesToUnderscore() { + BoundaryHierarchyValidator v = validator(); + assertEquals(30, MUST_REJECT.length, "expected 29 transform chars + '_' itself"); + + for (char c : MUST_REJECT) { + String hierarchyType = "MODFIX" + c + "A"; + CustomException ex = assertThrows(CustomException.class, + () -> v.validateBoundaryTypeHierarchy(requestFor(hierarchyType)), + () -> String.format("U+%04X should have been rejected", (int) c)); + assertEquals(ErrorCodes.INVALID_HIERARCHY_TYPE_SEPARATOR_CODE, ex.getCode(), + () -> String.format("U+%04X rejected with the wrong error code", (int) c)); + assertTrue(ex.getMessage().contains(String.format("U+%04X", (int) c)), + () -> String.format("message should name the offending code point U+%04X", (int) c)); + } + } + + @Test + void acceptsCleanHierarchyTypes() { + BoundaryHierarchyValidator v = validator(); + for (String ok : List.of("MODFIXA", "LNPERF115K", "MICROPLAN", "SIERRALEON", "Nigeria2026")) { + assertDoesNotThrow(() -> v.validateBoundaryTypeHierarchy(requestFor(ok)), + () -> ok + " should be accepted"); + } + } + + @Test + void doesNotRejectSemicolonOrOtherPunctuation() { + // ';' looks like it falls in a ':'-to-'s' range but the '-' in the source regex is literal. + // Guards against someone "simplifying" the class into an actual range. + BoundaryHierarchyValidator v = validator(); + for (String ok : List.of("MODFIX;A", "MODFIX@A", "MODFIX+A", "MODFIX=A", "MODFIX(A)")) { + assertDoesNotThrow(() -> v.validateBoundaryTypeHierarchy(requestFor(ok)), + () -> ok + " must not be rejected by the separator guard"); + } + } + + @Test + void reportsAllDistinctOffendersAtOnce() { + BoundaryHierarchyValidator v = validator(); + CustomException ex = assertThrows(CustomException.class, + () -> v.validateBoundaryTypeHierarchy(requestFor("A-B_C D.E"))); + assertTrue(ex.getMessage().contains("U+002D"), "should name hyphen"); + assertTrue(ex.getMessage().contains("U+005F"), "should name underscore"); + assertTrue(ex.getMessage().contains("U+0020"), "should name space"); + assertTrue(ex.getMessage().contains("U+002E"), "should name period"); + } + + @Test + void namesInvisibleCharactersReadably() { + BoundaryHierarchyValidator v = validator(); + CustomException nbsp = assertThrows(CustomException.class, + () -> v.validateBoundaryTypeHierarchy(requestFor("CHAD" + "\u00A0" + "SMC"))); + assertTrue(nbsp.getMessage().contains("non-breaking space"), + "NBSP must be named, not rendered as an invisible glyph: " + nbsp.getMessage()); + assertTrue(nbsp.getMessage().contains("U+00A0")); + } + + @Test + void nullHierarchyTypeIsLeftToOtherValidators() { + BoundaryHierarchyValidator v = validator(); + // Null/blank naming is not this validator's concern; it must not throw the separator error. + try { + v.validateBoundaryTypeHierarchy(requestFor(null)); + } catch (CustomException ex) { + assertNotEquals(ErrorCodes.INVALID_HIERARCHY_TYPE_SEPARATOR_CODE, ex.getCode()); + } + } +} From e70d3290177e2d1ce9bf6ba7c018cc61643bca92 Mon Sep 17 00:00:00 2001 From: nikhilmulinti Date: Tue, 11 Aug 2026 09:57:23 +0530 Subject: [PATCH 22/24] chore: dummy commit to trigger egov-localization build for ArgoCD/Kargo pipeline test --- core-services/egov-localization/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core-services/egov-localization/README.md b/core-services/egov-localization/README.md index fc6bcef0a19..ed6ad4f7458 100644 --- a/core-services/egov-localization/README.md +++ b/core-services/egov-localization/README.md @@ -31,3 +31,5 @@ Localisation can be search using combination of code , module, tenantid and loca ### Kafka Consumers ### Kafka Producers + + From cd6015865e0fd0f1db4b4136c9ecd4d40c7641df Mon Sep 17 00:00:00 2001 From: hruthvikl-egov Date: Tue, 11 Aug 2026 14:42:46 +0530 Subject: [PATCH 23/24] remove tracer.kafka.mdc.enabled kill-switch (tracer) --- core-services/libraries/tracer/CHANGELOG.md | 1 - core-services/libraries/tracer/readme.md | 8 +++++--- .../java/org/egov/tracer/config/TracerConfiguration.java | 3 ++- .../libraries/tracer/src/main/resources/tracer.properties | 3 --- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core-services/libraries/tracer/CHANGELOG.md b/core-services/libraries/tracer/CHANGELOG.md index 956e8bf578f..2d20a29716b 100644 --- a/core-services/libraries/tracer/CHANGELOG.md +++ b/core-services/libraries/tracer/CHANGELOG.md @@ -4,7 +4,6 @@ All notable changes to this library will be documented in this file. ## 2.9.3 - 2026-07-17 - Propagated correlationId and tenantId across Kafka — stamp both onto message headers on produce and rebuild the MDC from them on consume, so consumer/async logs carry the same IDs - Added MdcRecordInterceptor (auto-attached to the default Kafka listener container factory), KafkaTemplateLoggingInterceptors, and TracerKafkaMdcUtil helper -- Added kill-switch `tracer.kafka.mdc.enabled` (default true) ## 2.9.2 - 2026-03-10 - Addition of Data Access Exception Handling in ExceptionAdvice. diff --git a/core-services/libraries/tracer/readme.md b/core-services/libraries/tracer/readme.md index 0f763b4fc7b..71e89ced5e1 100644 --- a/core-services/libraries/tracer/readme.md +++ b/core-services/libraries/tracer/readme.md @@ -41,9 +41,11 @@ Map map = new HashMap<>(); The logging of the http request/response body and Kakfa message body can be toggled on/off using "tracer.detailed.tracing.enabled" application property. -The per-record Kafka MDC rebuild + cleanup (see "Setting the correlation id in the MDC") can be -toggled on/off using the "tracer.kafka.mdc.enabled" application property. It defaults to true; set it -to false to disable registration of the record interceptor. +The per-record Kafka MDC rebuild + cleanup (see "Setting the correlation id in the MDC") is not +toggleable. Correlation/tenant propagation over Kafka lives in the client interceptor registered by +"spring.kafka.properties.interceptor.classes", so disabling the record interceptor would have left +propagation running while dropping per-record cleanup — mis-attributing every record in a poll to the +last one. Remove the client interceptor property if the Kafka hop must be switched off entirely. ###### Correlation id retrieval and forwarding - diff --git a/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java b/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java index 19797784caf..93e39de38e8 100644 --- a/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java +++ b/core-services/libraries/tracer/src/main/java/org/egov/tracer/config/TracerConfiguration.java @@ -42,8 +42,9 @@ public RestTemplate logAwareRestTemplate(RestTemplateBuilder builder, TracerProp // MDC rebuild + cleanup from Kafka headers. Boot auto-applies to the autoconfig record-listener // factory; custom-factory or batch-listener services must wire it manually. + // Not conditional: propagation itself lives in the Kafka client interceptor, so gating this bean + // only dropped per-record cleanup and left MDC values leaking across records in a poll. @Bean - @ConditionalOnProperty(name = "tracer.kafka.mdc.enabled", havingValue = "true", matchIfMissing = true) public RecordInterceptor mdcRecordInterceptor() { return new MdcRecordInterceptor<>(); } diff --git a/core-services/libraries/tracer/src/main/resources/tracer.properties b/core-services/libraries/tracer/src/main/resources/tracer.properties index 8aea2169bb6..84a88389fea 100644 --- a/core-services/libraries/tracer/src/main/resources/tracer.properties +++ b/core-services/libraries/tracer/src/main/resources/tracer.properties @@ -15,9 +15,6 @@ tracer.filterSkipPattern=/api-docs.*|/autoconfig|/configprops|/dump|/health|/inf #Logging config spring.kafka.properties.interceptor.classes=org.egov.tracer.kafka.KafkaTemplateLoggingInterceptors -# per-record MDC rebuild + cleanup on Kafka consume (default on) -tracer.kafka.mdc.enabled=true - # Actuator Configs endpoints.enabled=false endpoints.health.enabled=true From 71b89966335aa525ba0cd5f9467e4327f3c4b9fa Mon Sep 17 00:00:00 2001 From: bhaswatmandal-egov Date: Tue, 25 Aug 2026 17:26:11 +0530 Subject: [PATCH 24/24] Readonly db case added to the db exception classifier --- .../consumer/DbExceptionClassifier.java | 23 ++++- .../persist/consumer/DbHealthMonitor.java | 24 ++++- .../consumer/DbExceptionClassifierTest.java | 39 +++++++- .../persist/consumer/DbHealthMonitorTest.java | 88 +++++++++++++++++++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbHealthMonitorTest.java diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java index a9fe95fdc9b..2a203555393 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbExceptionClassifier.java @@ -15,6 +15,20 @@ *
  • TRANSIENT — connection / serialization / deadlock / resource failures: retrying may succeed.
    • *
  • PERMANENT — constraint / data / grammar errors: retrying will always fail the same way.
    • * + * + *

    Note on 25006 (read_only_sql_transaction). SQLSTATE class 25 is "Invalid Transaction + * State" and is normally a programming error, so only this one member of the class is treated as + * transient — not the class as a whole. A managed Postgres instance can turn the whole server + * read-only for a bounded window (failover, maintenance, a storage-full condition), during which + * every write fails with 25006 and every one of them succeeds again once the window closes. + * Observed on ng-central-dev / mhbase on 2026-08-25: the server flipped read-only repeatedly and + * a single ~35s window dead-lettered 4,521 records, logged as 46 consecutive batches of + * "100 record(s) -> 0 persisted, 0 duplicate(s), 100 dead-lettered, 0 parked". Whole batches + * failing identically is the signature of an environmental fault, not of per-record corruption. + * Note this path is reached only when the driver exception carries the SQLSTATE: the same outage's + * connection kills arrived as DataAccessResourceFailureException and were already classified + * TRANSIENT by the cause-chain scan below, but the read-only writes arrived as + * UncategorizedSQLException, which that scan does not match.

    */ public final class DbExceptionClassifier { @@ -29,7 +43,14 @@ public static Kind classify(Throwable t) { } if (sqlState != null && ( sqlState.startsWith("08") // connection exception - || sqlState.startsWith("57") // operator intervention (e.g. admin shutdown, query cancel) + // Operator intervention: admin/crash shutdown, cannot_connect_now, database_dropped. + // The server is going away, so the in-place retry path (paired with the pause-on-DB-health + // backstop) is right. EXCLUDING 57014 query_canceled, which is a per-statement + // cancellation or timeout, not an outage: the next attempt is likely to be cancelled the + // same way, so retrying it in place can loop indefinitely. It must fall through to the + // bounded DLQ / parking flow instead. + || (sqlState.startsWith("57") && !"57014".equals(sqlState)) + || "25006".equals(sqlState) // read_only_sql_transaction — see note below || "40001".equals(sqlState) // serialization_failure || "40P01".equals(sqlState) // deadlock_detected || "53300".equals(sqlState) // too_many_connections diff --git a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java index a6f4692e8f2..4af1867580c 100644 --- a/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java +++ b/core-services/egov-persister/src/main/java/org/egov/infra/persist/consumer/DbHealthMonitor.java @@ -46,11 +46,33 @@ public void checkDatasource() { } } + /** + * Healthy means "a write would be accepted", not merely "the server answers". + * + *

    A read-only Postgres serves SELECTs perfectly, so a bare {@code SELECT 1} reports healthy + * throughout a read-only window while every INSERT/UPDATE fails. Observed on ng-central-dev / + * mhbase on 2026-08-25: the server flipped read-only repeatedly, this monitor never paused, and + * the consumer kept pulling work it could not persist. Both a hot standby and a server carrying + * {@code default_transaction_read_only=on} (how a managed instance signals failover, maintenance + * or a storage-full condition) report {@code transaction_read_only = on}, so that setting is the + * probe. {@code pg_is_in_recovery()} is checked too as a belt-and-braces signal for standby. + * Both are cheap, side-effect free, and need no write.

    + */ private boolean isHealthy() { try { - jdbcTemplate.queryForObject("SELECT 1", Integer.class); + String readOnly = jdbcTemplate.queryForObject("SHOW transaction_read_only", String.class); + if ("on".equalsIgnoreCase(readOnly)) { + log.debug("Datasource probe: server is read-only (transaction_read_only=on)"); + return false; + } + Boolean inRecovery = jdbcTemplate.queryForObject("SELECT pg_is_in_recovery()", Boolean.class); + if (Boolean.TRUE.equals(inRecovery)) { + log.debug("Datasource probe: server is in recovery (standby)"); + return false; + } return true; } catch (Exception e) { + // Covers unreachable/down as well - the probe itself throws. log.debug("Datasource probe failed: {}", e.getMessage()); return false; } diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java index 057dfe898f5..232185fefce 100644 --- a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbExceptionClassifierTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.CannotGetJdbcConnectionException; +import org.springframework.jdbc.UncategorizedSQLException; import java.sql.SQLException; @@ -28,7 +29,7 @@ void connectionAndConcurrencyStatesAreTransient() { assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("08001"))); // unable to connect assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("40001"))); // serialization_failure assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("40P01"))); // deadlock_detected - assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57014"))); // query_canceled + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57P01"))); // admin_shutdown assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("53300"))); // too_many_connections assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("55P03"))); // lock_not_available } @@ -42,6 +43,18 @@ void constraintAndDataStatesArePermanent() { assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(new RuntimeException("no sqlstate at all"))); } + @Test + void queryCanceledIsSplitOutOfTheOperatorInterventionBucket() { + // 57014 query_canceled is a per-statement cancellation/timeout, not the server going away. + // In-place retry can loop indefinitely on it, so it must reach the bounded DLQ/parking flow. + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("57014"))); + + // The rest of class 57 is a genuine outage and stays transient. + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57P01"))); // admin_shutdown + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57P02"))); // crash_shutdown + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("57P03"))); // cannot_connect_now + } + @Test void classifiesThroughTheCauseChain() { // A real failure is wrapped several layers deep (JdbcTemplate -> DataAccessException -> SQLException). @@ -57,4 +70,28 @@ void connectionAcquisitionFailureIsTransientEvenWithoutSqlState() { Throwable ex = new CannotGetJdbcConnectionException("could not get connection"); assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(ex)); } + + @Test + void readOnlyTransactionIsTransientButTheRestOfClass25IsNot() { + // 25006: a managed Postgres can turn the whole server read-only for a bounded window + // (failover, maintenance, storage full), after which the identical write succeeds. + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(sqlState("25006"))); // read_only_sql_transaction + + // Only that one member of class 25 is transient. The rest is genuine invalid-transaction-state + // and must stay permanent - retrying it would loop forever. + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("25001"))); // active_sql_transaction + assertEquals(Kind.PERMANENT, DbExceptionClassifier.classify(sqlState("25P02"))); // in_failed_sql_transaction + } + + @Test + void readOnlyTransactionIsTransientInItsProductionWrapping() { + // The shape seen on ng-central-dev / mhbase 2026-08-25: Spring surfaces it as + // UncategorizedSQLException, whose class name the cause-chain scan does NOT match. This case + // therefore passes only via the SQLSTATE branch - it is the regression guard for that path. + Throwable ex = new UncategorizedSQLException( + "PreparedStatementCallback", + "UPDATE mhbase.eg_cm_campaign_data SET data = ?", + sqlState("25006")); + assertEquals(Kind.TRANSIENT, DbExceptionClassifier.classify(ex)); + } } diff --git a/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbHealthMonitorTest.java b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbHealthMonitorTest.java new file mode 100644 index 00000000000..704c2b0afe2 --- /dev/null +++ b/core-services/egov-persister/src/test/java/org/egov/infra/persist/consumer/DbHealthMonitorTest.java @@ -0,0 +1,88 @@ +package org.egov.infra.persist.consumer; + +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The monitor must gate on WRITABILITY, not reachability. + * + *

    A read-only Postgres answers SELECTs normally, so the previous {@code SELECT 1} probe reported + * healthy for the whole of a read-only window while every write failed. On ng-central-dev / mhbase + * on 2026-08-25 that left the consumer running against a server that could not accept a single + * INSERT or UPDATE. These tests pin the corrected behaviour.

    + */ +class DbHealthMonitorTest { + + private static final String READ_ONLY_PROBE = "SHOW transaction_read_only"; + private static final String RECOVERY_PROBE = "SELECT pg_is_in_recovery()"; + + private static JdbcTemplate jdbc(String transactionReadOnly, Boolean inRecovery) { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + when(jdbcTemplate.queryForObject(eq(READ_ONLY_PROBE), eq(String.class))).thenReturn(transactionReadOnly); + when(jdbcTemplate.queryForObject(eq(RECOVERY_PROBE), eq(Boolean.class))).thenReturn(inRecovery); + return jdbcTemplate; + } + + @Test + void pausesWhenServerIsReadOnly() { + PersisterConsumerConfig consumerConfig = mock(PersisterConsumerConfig.class); + new DbHealthMonitor(jdbc("on", Boolean.FALSE), consumerConfig).checkDatasource(); + + verify(consumerConfig).pauseContainer(); + verify(consumerConfig, never()).resumeContainer(); + } + + @Test + void pausesWhenServerIsInRecovery() { + PersisterConsumerConfig consumerConfig = mock(PersisterConsumerConfig.class); + new DbHealthMonitor(jdbc("off", Boolean.TRUE), consumerConfig).checkDatasource(); + + verify(consumerConfig).pauseContainer(); + } + + @Test + void doesNotPauseWhenServerIsWritable() { + // Control: without this the read-only assertions above would pass even if the monitor + // paused unconditionally. + PersisterConsumerConfig consumerConfig = mock(PersisterConsumerConfig.class); + new DbHealthMonitor(jdbc("off", Boolean.FALSE), consumerConfig).checkDatasource(); + + verify(consumerConfig, never()).pauseContainer(); + verify(consumerConfig, never()).resumeContainer(); + } + + @Test + void resumesOnceTheReadOnlyWindowCloses() { + PersisterConsumerConfig consumerConfig = mock(PersisterConsumerConfig.class); + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + when(jdbcTemplate.queryForObject(eq(RECOVERY_PROBE), eq(Boolean.class))).thenReturn(Boolean.FALSE); + // Read-only on the first poll, writable on the second - the real flapping shape. + when(jdbcTemplate.queryForObject(eq(READ_ONLY_PROBE), eq(String.class))).thenReturn("on", "off"); + + DbHealthMonitor monitor = new DbHealthMonitor(jdbcTemplate, consumerConfig); + monitor.checkDatasource(); + monitor.checkDatasource(); + + verify(consumerConfig).pauseContainer(); + verify(consumerConfig).resumeContainer(); + } + + @Test + void pausesWhenTheProbeItselfFails() { + // Server unreachable: the probe throws rather than returning a value. + PersisterConsumerConfig consumerConfig = mock(PersisterConsumerConfig.class); + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + when(jdbcTemplate.queryForObject(eq(READ_ONLY_PROBE), eq(String.class))) + .thenThrow(new RuntimeException("connection refused")); + + new DbHealthMonitor(jdbcTemplate, consumerConfig).checkDatasource(); + + verify(consumerConfig).pauseContainer(); + } +}