Optimized billing service search flow to improve performance and implemented pagination

business-services/billing-service/src/main/java/org/egov/demand/config/ApplicationProperties.java

@@ -47,10 +47,12 @@
 
 import lombok.Getter;
 import lombok.ToString;
+import org.springframework.stereotype.Component;
 
 @Configuration
 @Getter
 @ToString
+@Component
 public class ApplicationProperties {
 
 
@@ -201,5 +203,16 @@ public class ApplicationProperties {
     @Value("${is.environment.central.instance}")
     private Boolean isEnvironmentCentralInstance;
 
+    // Pagination
+
+	@Value("${search.default.offset}")
+	private Integer defaultOffset;
+
+	@Value("${search.default.limit}")
+	private Integer defaultLimit;
+
+	@Value("${search.max.limit}")
+	private Integer maxLimit;
+
 
 }

business-services/billing-service/src/main/java/org/egov/demand/model/DemandCriteria.java

@@ -86,6 +86,10 @@ public class DemandCriteria {
 	private String status;
 
 	private Boolean isPaymentCompleted;
+
+	private Integer offset;
+
+	private Integer limit;
 
 	@Default
 	private Boolean receiptRequired=false;

business-services/billing-service/src/main/java/org/egov/demand/repository/DemandRepository.java

@@ -41,16 +41,12 @@
 
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import org.egov.common.exception.InvalidTenantIdException;
 import org.egov.common.utils.MultiStateInstanceUtil;
+import org.egov.demand.config.ApplicationProperties;
 import org.egov.demand.model.AuditDetails;
 import org.egov.demand.model.Demand;
 import org.egov.demand.model.DemandCriteria;
@@ -66,10 +62,12 @@
 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.core.PreparedStatementSetter;
+import org.springframework.jdbc.core.SingleColumnRowMapper;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
 
 @Repository
 @Slf4j
@@ -93,17 +91,44 @@ public class DemandRepository {
 	public List<Demand> getDemands(DemandCriteria demandCriteria) {
 
 		List<Object> preparedStatementValues = new ArrayList<>();
+
+		// Fetch demand ids first with pagination.
+		List<String> demandIds = getDemandIdsBasedOnTheCriteriaReceived(demandCriteria);
+
+		if(CollectionUtils.isEmpty(demandIds))
+			return new ArrayList<>();
+
+		// Prepare query with join with demand details to fetch the whole demand response
+		demandCriteria.setDemandId(new HashSet<>(demandIds));
 		String searchDemandQuery = demandQueryBuilder.getDemandQuery(demandCriteria, preparedStatementValues);
 		try {
-			searchDemandQuery=centralInstanceUtil.replaceSchemaPlaceholder(searchDemandQuery, demandCriteria.getTenantId());
+			searchDemandQuery = centralInstanceUtil.replaceSchemaPlaceholder(searchDemandQuery, demandCriteria.getTenantId());
 		} catch (InvalidTenantIdException e) {
 			throw new CustomException("EG_BS_TENANTID_ERROR",
 					"TenantId length is not sufficient to replace query schema in a multi state instance");
 		}
 
 		return jdbcTemplate.query(searchDemandQuery, preparedStatementValues.toArray(), demandRowMapper);
 	}
-
+
+	private List<String> getDemandIdsBasedOnTheCriteriaReceived(DemandCriteria demandCriteria) {
+		List<Object> preparedStatementValues = new ArrayList<>();
+
+		String demandIdsQuery = demandQueryBuilder.getDemandIdsQuery(demandCriteria, preparedStatementValues);
+
+		try {
+			demandIdsQuery = centralInstanceUtil.replaceSchemaPlaceholder(demandIdsQuery, demandCriteria.getTenantId());
+		} catch (InvalidTenantIdException e) {
+			throw new CustomException("EG_BS_TENANTID_ERROR",
+					"TenantId length is not sufficient to replace query schema in a multi state instance");
+		}
+
+		List<String> demandIds = jdbcTemplate.query(demandIdsQuery, preparedStatementValues.toArray(), new SingleColumnRowMapper<>(String.class));
+
+		return demandIds;
+
+	}
+
 	/**
 	 * Fetches demand from DB based on a map of business code and set of consumer codes
 	 * 

business-services/billing-service/src/main/java/org/egov/demand/repository/querybuilder/DemandQueryBuilder.java

@@ -46,19 +46,24 @@
 import java.util.Set;
 
 import org.egov.common.utils.MultiStateInstanceUtil;
+import org.egov.demand.config.ApplicationProperties;
 import org.egov.demand.model.DemandCriteria;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.ObjectUtils;
 
 @Slf4j
 @Component
 public class DemandQueryBuilder {
 
 	@Autowired
 	private MultiStateInstanceUtil centralInstaceUtil;
+
+	@Autowired
+	private ApplicationProperties applicationProperties;
 
 
 	public static final String PAYMENT_BACKUPDATE_AUDIT_INSERT_QUERY = "INSERT INTO {schema}.egbs_payment_backupdate_audit (paymentid, isbackupdatesuccess, isreceiptcancellation, errorMessage)"
@@ -82,6 +87,16 @@ public class DemandQueryBuilder {
 			+ "dmdl.tenantid AS dltenantid,dmdl.additionaldetails as detailadditionaldetails " + "FROM {schema}.egbs_demand_v1 dmd "
 			+ "INNER JOIN {schema}.egbs_demanddetail_v1 dmdl ON dmd.id=dmdl.demandid " + "AND dmd.tenantid=dmdl.tenantid WHERE ";
 
+	public static final String DEMAND_IDS_QUERY = "SELECT dmd.id AS did,dmd.consumercode AS dconsumercode,"
+			+ "dmd.consumertype AS dconsumertype,dmd.businessservice AS dbusinessservice,dmd.payer,"
+			+ "dmd.billexpirytime AS dbillexpirytime, dmd.fixedBillExpiryDate as dfixedBillExpiryDate, "
+			+ "dmd.taxperiodfrom AS dtaxperiodfrom,dmd.taxperiodto AS dtaxperiodto,"
+			+ "dmd.minimumamountpayable AS dminimumamountpayable,dmd.createdby AS dcreatedby,"
+			+ "dmd.lastmodifiedby AS dlastmodifiedby,dmd.createdtime AS dcreatedtime,"
+			+ "dmd.lastmodifiedtime AS dlastmodifiedtime,dmd.tenantid AS dtenantid,dmd.status,"
+			+ "dmd.additionaldetails as demandadditionaldetails,dmd.ispaymentcompleted as ispaymentcompleted "
+			+ "FROM {schema}.egbs_demand_v1 dmd WHERE ";
+
 	public static final String BASE_DEMAND_DETAIL_QUERY = "SELECT "
 			+ "demanddetail.id AS dlid,demanddetail.demandid AS dldemandid,demanddetail.taxheadcode AS dltaxheadcode,"
 			+ "demanddetail.taxamount AS dltaxamount,demanddetail.collectionamount AS dlcollectionamount,"
@@ -162,63 +177,10 @@ public String getDemandQuery(DemandCriteria demandCriteria, List<Object> prepare
 
 		StringBuilder demandQuery = new StringBuilder(BASE_DEMAND_QUERY);
 
-		String tenantId = demandCriteria.getTenantId();
-
-		if (centralInstaceUtil.isTenantIdStateLevel(tenantId)) {
-			demandQuery.append(" dmd.tenantid LIKE ? ");
-			preparedStatementValues.add(demandCriteria.getTenantId() + '%');
-		} else {
-			demandQuery.append(" dmd.tenantid = ? ");
-			preparedStatementValues.add(demandCriteria.getTenantId());
-		}
-
-		if (demandCriteria.getStatus() != null) {
-
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.status=?");
-			preparedStatementValues.add(demandCriteria.getStatus());
-		}
-
-		if (demandCriteria.getDemandId() != null && !demandCriteria.getDemandId().isEmpty()) {
-			addAndClause(demandQuery);
+		if (!CollectionUtils.isEmpty(demandCriteria.getDemandId())) {
 			demandQuery.append("dmd.id IN (" + getIdQueryForStrings(demandCriteria.getDemandId()) + ")");
 			addToPreparedStatement(preparedStatementValues, demandCriteria.getDemandId());
 		}
-		if (!CollectionUtils.isEmpty(demandCriteria.getPayer())) {
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.payer IN (" + getIdQueryForStrings(demandCriteria.getPayer()) + ")");
-			addToPreparedStatement(preparedStatementValues, demandCriteria.getPayer());
-		}
-		if (demandCriteria.getBusinessService() != null) {
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.businessservice=?");
-			preparedStatementValues.add(demandCriteria.getBusinessService());
-		}
-
-		if(demandCriteria.getIsPaymentCompleted() != null){
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.ispaymentcompleted = ?");
-			preparedStatementValues.add(demandCriteria.getIsPaymentCompleted());
-		}
-
-		if (demandCriteria.getPeriodFrom() != null) {
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.taxPeriodFrom >= ?");
-			preparedStatementValues.add(demandCriteria.getPeriodFrom());
-		}
-
-		if(demandCriteria.getPeriodTo() != null){
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.taxPeriodTo <= ?");
-			preparedStatementValues.add(demandCriteria.getPeriodTo());
-		}
-
-		if (demandCriteria.getConsumerCode() != null && !demandCriteria.getConsumerCode().isEmpty()) {
-			addAndClause(demandQuery);
-			demandQuery.append("dmd.consumercode IN ("
-			+ getIdQueryForStrings(demandCriteria.getConsumerCode()) + ")");
-			addToPreparedStatement(preparedStatementValues, demandCriteria.getConsumerCode());
-		}
 
 		addOrderByClause(demandQuery, DEMAND_QUERY_ORDER_BY_CLAUSE);
 		addPagingClause(demandQuery, preparedStatementValues);
@@ -238,6 +200,29 @@ private static void addPagingClause(StringBuilder demandQueryBuilder, List<Objec
 //		preparedStatementValues.add(0);
 	}
 
+	private void addPagingClause(StringBuilder demandQueryBuilder, DemandCriteria criteria, List<Object> preparedStatementValues) {
+
+		// Add clause for offset
+		demandQueryBuilder.append(" OFFSET ?");
+		if(ObjectUtils.isEmpty(criteria.getOffset())) {
+			preparedStatementValues.add(applicationProperties.getDefaultOffset());
+		} else{
+			preparedStatementValues.add(criteria.getOffset());
+		}
+
+		// Add clause for limit
+		demandQueryBuilder.append(" LIMIT ?");
+		if(ObjectUtils.isEmpty(criteria.getLimit())){
+			preparedStatementValues.add(applicationProperties.getDefaultLimit());
+		}else {
+			if (criteria.getLimit() > applicationProperties.getMaxLimit()) {
+				preparedStatementValues.add(applicationProperties.getMaxLimit());
+			} else {
+				preparedStatementValues.add(criteria.getLimit());
+			}
+		}
+	}
+
 	private static boolean addAndClause(StringBuilder queryString) {
 		queryString.append(" AND ");
 		return true;
@@ -258,4 +243,73 @@ private void addToPreparedStatement(List<Object> preparedStmtList, Collection<St
 	{
 		ids.forEach(id ->{ preparedStmtList.add(id);});
 	}
+
+	public String getDemandIdsQuery(DemandCriteria demandCriteria, List<Object> preparedStatementValues) {
+		String finalIdsQuery = "SELECT did FROM ( {INTERNAL_QUERY} ) AS did";
+		StringBuilder demandIdsQuery = new StringBuilder(DEMAND_IDS_QUERY);
+
+		String tenantId = demandCriteria.getTenantId();
+
+		if (centralInstaceUtil.isTenantIdStateLevel(tenantId)) {
+			demandIdsQuery.append(" dmd.tenantid LIKE ? ");
+			preparedStatementValues.add(demandCriteria.getTenantId() + '%');
+		} else {
+			demandIdsQuery.append(" dmd.tenantid = ? ");
+			preparedStatementValues.add(demandCriteria.getTenantId());
+		}
+
+		if (demandCriteria.getStatus() != null) {
+
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.status=?");
+			preparedStatementValues.add(demandCriteria.getStatus());
+		}
+
+		if (demandCriteria.getDemandId() != null && !demandCriteria.getDemandId().isEmpty()) {
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.id IN (" + getIdQueryForStrings(demandCriteria.getDemandId()) + ")");
+			addToPreparedStatement(preparedStatementValues, demandCriteria.getDemandId());
+		}
+		if (!CollectionUtils.isEmpty(demandCriteria.getPayer())) {
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.payer IN (" + getIdQueryForStrings(demandCriteria.getPayer()) + ")");
+			addToPreparedStatement(preparedStatementValues, demandCriteria.getPayer());
+		}
+		if (demandCriteria.getBusinessService() != null) {
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.businessservice=?");
+			preparedStatementValues.add(demandCriteria.getBusinessService());
+		}
+
+		if(demandCriteria.getIsPaymentCompleted() != null){
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.ispaymentcompleted = ?");
+			preparedStatementValues.add(demandCriteria.getIsPaymentCompleted());
+		}
+
+		if (demandCriteria.getPeriodFrom() != null) {
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.taxPeriodFrom >= ?");
+			preparedStatementValues.add(demandCriteria.getPeriodFrom());
+		}
+
+		if(demandCriteria.getPeriodTo() != null){
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.taxPeriodTo <= ?");
+			preparedStatementValues.add(demandCriteria.getPeriodTo());
+		}
+
+		if (demandCriteria.getConsumerCode() != null && !demandCriteria.getConsumerCode().isEmpty()) {
+			addAndClause(demandIdsQuery);
+			demandIdsQuery.append("dmd.consumercode IN ("
+					+ getIdQueryForStrings(demandCriteria.getConsumerCode()) + ")");
+			addToPreparedStatement(preparedStatementValues, demandCriteria.getConsumerCode());
+		}
+
+		addOrderByClause(demandIdsQuery, DEMAND_QUERY_ORDER_BY_CLAUSE);
+		addPagingClause(demandIdsQuery, demandCriteria, preparedStatementValues);
+
+		return finalIdsQuery.replace("{INTERNAL_QUERY}", demandIdsQuery);
+
+	}
 }

business-services/billing-service/src/main/resources/application.properties

@@ -2,7 +2,7 @@
 migration.batch.value=100
 
 spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://localhost:5432/devbackup
+spring.datasource.url=jdbc:postgresql://localhost:5432/demanddb
 spring.datasource.username=postgres
 spring.datasource.password=postgres
 
@@ -131,7 +131,7 @@ egov.mdms.host=https://dev.digit.org
 egov.mdms.search.endpoint=/egov-mdms-service/v1/_search
 
 #----------------------------- FLYWAY CONFIGURATIONS ------------------------------#
-spring.flyway.url=jdbc:postgresql://localhost:5432/billingnew
+spring.flyway.url=jdbc:postgresql://localhost:5432/demanddb
 spring.flyway.user=postgres
 spring.flyway.password=postgres
 spring.flyway.table=public
@@ -166,3 +166,8 @@ egov.state.list=statea,stateb
 
 kafka.topics.receipt.topic.pattern=((^[a-zA-Z]+-)?egov.collection.payment-create)
 kafka.topics.receipt.cancel.topic.pattern=((^[a-zA-Z]+-)?egov.collection.payment-cancel)
+
+# Pagination
+search.default.offset=0
+search.default.limit=10
+search.max.limit=300