From 8c85e0f05dc98e52bcd6eedf3051421fc8dda765 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 13:07:32 +0300 Subject: [PATCH 01/20] feat(dlr): add PostgreSQL schema migration --- pom.xml | 10 + sendium-core/pom.xml | 21 ++ .../V1__create_sendium_dlr_schema.sql | 59 ++++ .../core/dlr/PostgresqlMigrationTest.java | 296 ++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java diff --git a/pom.xml b/pom.xml index 713426e..5d5244a 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,7 @@ false ${skipTests} ${skipTests} + false 7.2.2 3.27.7 @@ -98,4 +99,13 @@ + + + + postgresql-tests + + true + + + diff --git a/sendium-core/pom.xml b/sendium-core/pom.xml index a104efc..d6bb466 100644 --- a/sendium-core/pom.xml +++ b/sendium-core/pom.xml @@ -80,6 +80,26 @@ assertj-core test + + org.flywaydb + flyway-core + test + + + org.flywaydb + flyway-database-postgresql + test + + + org.postgresql + postgresql + test + + + org.testcontainers + testcontainers-postgresql + test + @@ -136,6 +156,7 @@ org.jboss.logmanager.LogManager ${maven.home} + ${sendium.postgresql.tests} diff --git a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql new file mode 100644 index 0000000..a9c052f --- /dev/null +++ b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql @@ -0,0 +1,59 @@ +CREATE SCHEMA IF NOT EXISTS sendium_dlr; + +CREATE TABLE sendium_dlr.tracked_message ( + gateway_message_id UUID PRIMARY KEY, + account_id TEXT, + system_id TEXT, + source_address TEXT, + destination_address TEXT, + forward_dlr_url TEXT, + reassembled_parts TEXT[], + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT tracked_message_status_check + CHECK (status IN ('ACCEPTED', 'SENT', 'DELIVERED', 'FAILED')) +); + +CREATE INDEX tracked_message_created_at_idx + ON sendium_dlr.tracked_message (created_at); + +CREATE TABLE sendium_dlr.operator_correlation ( + operator_message_id TEXT PRIMARY KEY, + gateway_message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT operator_correlation_message_fk + FOREIGN KEY (gateway_message_id) + REFERENCES sendium_dlr.tracked_message (gateway_message_id) + ON DELETE CASCADE +); + +CREATE INDEX operator_correlation_created_at_idx + ON sendium_dlr.operator_correlation (created_at); + +CREATE INDEX operator_correlation_gateway_message_idx + ON sendium_dlr.operator_correlation (gateway_message_id); + +CREATE TABLE sendium_dlr.unpushed_dlr ( + dlr_key TEXT PRIMARY KEY, + system_id TEXT NOT NULL, + account_id TEXT, + source_address TEXT, + destination_address TEXT, + serial TEXT, + message_id INTEGER NOT NULL, + dlr_state INTEGER NOT NULL, + error_code TEXT, + acked BOOLEAN NOT NULL, + priority INTEGER NOT NULL, + reassembled_parts TEXT[], + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT unpushed_dlr_system_id_not_blank + CHECK (system_id !~ '^[[:space:]]*$') +); + +CREATE INDEX unpushed_dlr_system_created_at_idx + ON sendium_dlr.unpushed_dlr (system_id, created_at); + +CREATE INDEX unpushed_dlr_created_at_idx + ON sendium_dlr.unpushed_dlr (created_at); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java new file mode 100644 index 0000000..aa06b87 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java @@ -0,0 +1,296 @@ +package gr.cytech.sendium.core.dlr; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.output.MigrateResult; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.testcontainers.postgresql.PostgreSQLContainer; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") +class PostgresqlMigrationTest { + private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; + private static final UUID INVALID_STATUS_GATEWAY_ID = + UUID.fromString("00000000-0000-0000-0000-000000000001"); + private static final UUID CORRELATION_GATEWAY_ID = + UUID.fromString("00000000-0000-0000-0000-000000000002"); + private static final UUID COMPLETE_GATEWAY_ID = + UUID.fromString("00000000-0000-0000-0000-000000000003"); + private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") + .withDatabaseName("sendium") + .withUsername("sendium") + .withPassword("sendium-test"); + + private static Flyway flyway; + private static MigrateResult initialMigration; + + @BeforeAll + static void migrateSchema() { + POSTGRESQL.start(); + flyway = Flyway.configure() + .dataSource(POSTGRESQL.getJdbcUrl(), POSTGRESQL.getUsername(), POSTGRESQL.getPassword()) + .locations(MIGRATION_LOCATION) + .load(); + initialMigration = flyway.migrate(); + } + + @AfterAll + static void stopPostgresql() { + POSTGRESQL.stop(); + } + + @Test + void migrationCreatesExpectedTablesAndIndexes() throws SQLException { + assertThat(initialMigration.success).isTrue(); + assertThat(initialMigration.migrationsExecuted).isOne(); + + try (Connection connection = connection()) { + assertThat(loadNames(connection, + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'sendium_dlr'")) + .containsExactlyInAnyOrder("tracked_message", "operator_correlation", "unpushed_dlr"); + assertThat(loadNames(connection, + "SELECT indexname FROM pg_indexes WHERE schemaname = 'sendium_dlr'")) + .contains("tracked_message_created_at_idx", + "operator_correlation_created_at_idx", + "operator_correlation_gateway_message_idx", + "unpushed_dlr_system_created_at_idx", + "unpushed_dlr_created_at_idx"); + assertThat(loadColumnType(connection, "tracked_message", "gateway_message_id")) + .isEqualTo("uuid"); + assertThat(loadColumnType(connection, "operator_correlation", "gateway_message_id")) + .isEqualTo("uuid"); + } + } + + @Test + void migrationIsIdempotent() { + MigrateResult repeatedMigration = flyway.migrate(); + + assertThat(repeatedMigration.success).isTrue(); + assertThat(repeatedMigration.migrationsExecuted).isZero(); + } + + @Test + void trackedMessageRejectsUnknownStatus() throws SQLException { + try (Connection connection = connection(); + PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.tracked_message + (gateway_message_id, account_id, system_id, status) + VALUES (?, 'account', 'system', 'UNKNOWN') + """)) { + statement.setObject(1, INVALID_STATUS_GATEWAY_ID); + assertThatThrownBy(statement::executeUpdate).isInstanceOf(SQLException.class); + } + } + + @Test + void correlationIsDeletedWithTrackedMessage() throws SQLException { + try (Connection connection = connection()) { + insertTrackedMessage(connection, CORRELATION_GATEWAY_ID); + insertCorrelation(connection, "operator-1", CORRELATION_GATEWAY_ID); + insertCorrelation(connection, "operator-2", CORRELATION_GATEWAY_ID); + + try (PreparedStatement statement = connection.prepareStatement(""" + DELETE FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? + """)) { + statement.setObject(1, CORRELATION_GATEWAY_ID); + statement.executeUpdate(); + } + + assertThat(countCorrelations(connection, CORRELATION_GATEWAY_ID)).isZero(); + } + } + + @Test + void unpushedDlrRequiresNonBlankSystemId() throws SQLException { + try (Connection connection = connection()) { + assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "empty-system", "")) + .isInstanceOf(SQLException.class); + assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "whitespace-system", " ")) + .isInstanceOf(SQLException.class); + assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "control-whitespace-system", "\t\n")) + .isInstanceOf(SQLException.class); + } + } + + @Test + void typedColumnsStoreCurrentDlrState() throws SQLException { + try (Connection connection = connection()) { + insertCompleteTrackedMessage(connection); + insertCompleteUnpushedDlr(connection); + + try (PreparedStatement statement = connection.prepareStatement(""" + SELECT gateway_message_id, reassembled_parts, created_at, updated_at + FROM sendium_dlr.tracked_message + WHERE gateway_message_id = ? + """)) { + statement.setObject(1, COMPLETE_GATEWAY_ID); + try (ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + assertThat(resultSet.getObject("gateway_message_id", UUID.class)) + .isEqualTo(COMPLETE_GATEWAY_ID); + assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) + .containsExactly("part-1", "part-2"); + assertThat(resultSet.getObject("created_at")).isNotNull(); + assertThat(resultSet.getObject("updated_at")).isNotNull(); + } + } + + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(""" + SELECT account_id, message_id, dlr_state, error_code, acked, priority, reassembled_parts + FROM sendium_dlr.unpushed_dlr + WHERE dlr_key = 'dlr-complete' + """)) { + assertThat(resultSet.next()).isTrue(); + assertThat(resultSet.getString("account_id")).isEqualTo("account"); + assertThat(resultSet.getInt("message_id")).isEqualTo(123); + assertThat(resultSet.getInt("dlr_state")).isEqualTo(1); + assertThat(resultSet.getString("error_code")).isEqualTo("0"); + assertThat(resultSet.getBoolean("acked")).isTrue(); + assertThat(resultSet.getInt("priority")).isEqualTo(2); + assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) + .containsExactly("part-1", "part-2"); + } + } + } + + private static Connection connection() throws SQLException { + return POSTGRESQL.createConnection(""); + } + + private static Set loadNames(Connection connection, String sql) throws SQLException { + Set names = new HashSet<>(); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + names.add(resultSet.getString(1)); + } + } + return names; + } + + private static void insertTrackedMessage(Connection connection, UUID gatewayMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.tracked_message + (gateway_message_id, account_id, system_id, status) + VALUES (?, 'account', 'system', 'ACCEPTED') + """)) { + statement.setObject(1, gatewayMessageId); + statement.executeUpdate(); + } + } + + private static void insertCorrelation(Connection connection, String operatorMessageId, + UUID gatewayMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.operator_correlation + (operator_message_id, gateway_message_id) + VALUES (?, ?) + """)) { + statement.setString(1, operatorMessageId); + statement.setObject(2, gatewayMessageId); + statement.executeUpdate(); + } + } + + private static void insertMinimalUnpushedDlr(Connection connection, String key, + String systemId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.unpushed_dlr + (dlr_key, system_id, message_id, dlr_state, acked, priority) + VALUES (?, ?, 1, 1, FALSE, 0) + """)) { + statement.setString(1, key); + statement.setString(2, systemId); + statement.executeUpdate(); + } + } + + private static void insertCompleteTrackedMessage(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.tracked_message + (gateway_message_id, account_id, system_id, source_address, destination_address, + forward_dlr_url, reassembled_parts, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """)) { + statement.setObject(1, COMPLETE_GATEWAY_ID); + statement.setString(2, "account"); + statement.setString(3, "system"); + statement.setString(4, "source"); + statement.setString(5, "destination"); + statement.setString(6, "https://example.test/dlr"); + statement.setArray(7, connection.createArrayOf("text", new String[]{"part-1", "part-2"})); + statement.setString(8, "ACCEPTED"); + statement.executeUpdate(); + } + } + + private static void insertCompleteUnpushedDlr(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.unpushed_dlr + (dlr_key, system_id, account_id, source_address, destination_address, serial, + message_id, dlr_state, error_code, acked, priority, reassembled_parts) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """)) { + statement.setString(1, "dlr-complete"); + statement.setString(2, "system"); + statement.setString(3, "account"); + statement.setString(4, "source"); + statement.setString(5, "destination"); + statement.setString(6, "serial"); + statement.setInt(7, 123); + statement.setInt(8, 1); + statement.setString(9, "0"); + statement.setBoolean(10, true); + statement.setInt(11, 2); + statement.setArray(12, connection.createArrayOf("text", new String[]{"part-1", "part-2"})); + statement.executeUpdate(); + } + } + + private static int countCorrelations(Connection connection, UUID gatewayMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + SELECT COUNT(*) + FROM sendium_dlr.operator_correlation + WHERE gateway_message_id = ? + """)) { + statement.setObject(1, gatewayMessageId); + try (ResultSet resultSet = statement.executeQuery()) { + resultSet.next(); + return resultSet.getInt(1); + } + } + } + + private static String loadColumnType(Connection connection, String tableName, + String columnName) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + SELECT data_type + FROM information_schema.columns + WHERE table_schema = 'sendium_dlr' + AND table_name = ? + AND column_name = ? + """)) { + statement.setString(1, tableName); + statement.setString(2, columnName); + try (ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + return resultSet.getString(1); + } + } + } +} From 8cc9ddf495c798ab8530e4dee995d0b8ee38692a Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 13:25:59 +0300 Subject: [PATCH 02/20] refactor(dlr): extract storage port --- .../sendium/core/http/KannelResource.java | 4 +- .../InMemorySmppServerMessageStore.java | 6 +- .../sendium/core/worker/DlrService.java | 69 +++++++++ .../sendium/core/worker/DlrStorage.java | 40 ++++++ ...DlrService.java => MvStoreDlrStorage.java} | 41 +++--- .../external/WorkerResourceProvider.java | 8 +- .../sendium/core/http/KannelResourceIT.java | 6 +- .../InMemorySmppServerMessageStoreTest.java | 4 +- .../sendium/core/worker/DlrServiceTest.java | 81 +++++++++++ .../worker/InMemoryMessageTrackerTest.java | 2 +- ...ceTest.java => MvStoreDlrStorageTest.java} | 132 +++++++++--------- 11 files changed, 291 insertions(+), 102 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java rename sendium-core/src/main/java/gr/cytech/sendium/core/worker/{InMemoryDlrService.java => MvStoreDlrStorage.java} (95%) create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java rename sendium-core/src/test/java/gr/cytech/sendium/core/worker/{InMemoryDlrServiceTest.java => MvStoreDlrStorageTest.java} (60%) diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index 8d05962..6f4a02f 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -5,7 +5,7 @@ import gr.cytech.sendium.conf.SendiumConfigurationHandler; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; -import gr.cytech.sendium.core.worker.InMemoryDlrService; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.util.MessageTrace; import jakarta.annotation.security.PermitAll; @@ -44,7 +44,7 @@ public class KannelResource { CredentialFileWatcher credentialFileWatcher; @Inject - InMemoryDlrService dlrService; + DlrService dlrService; @Inject SendiumConfigurationHandler configurationHandler; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java index 3d768e7..05c57ce 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java @@ -1,7 +1,7 @@ package gr.cytech.sendium.core.smpp.server; import gr.cytech.sendium.core.message.StandardMessage; -import gr.cytech.sendium.core.worker.InMemoryDlrService; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.util.MessageTrace; import gr.cytech.sendium.util.SensitiveLogSanitizer; @@ -76,7 +76,7 @@ public boolean markAsUnpushed(StandardMessage msg) { @Override public void onClientConnected(String systemId) { - InMemoryDlrService dlrService = getDlrService(); + DlrService dlrService = getDlrService(); List unpushedDlrs = dlrService.claimUnpushedDlrs(systemId); if (unpushedDlrs.isEmpty()) { logger.info("Unpushed DLR(s) not found for systemId:{}", systemId); @@ -98,7 +98,7 @@ public void onClientConnected(String systemId) { } } - private InMemoryDlrService getDlrService() { + private DlrService getDlrService() { return worker.getWorkerResources().getDlrService(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java new file mode 100644 index 0000000..7810b86 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java @@ -0,0 +1,69 @@ +package gr.cytech.sendium.core.worker; + +import gr.cytech.sendium.core.message.StandardMessage; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.util.List; +import java.util.Optional; + +@ApplicationScoped +public class DlrService { + @Inject + DlrStorage storage; + + @Inject + ForwardDlrService forwardDlrService; + + public void saveInitialState(MessageState state) { + storage.saveInitialState(state); + } + + public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { + storage.linkOperatorId(gatewayMsgId, operatorMsgId); + } + + public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrState) { + Optional state = storage.resolveAndRemoveDlr(operatorMsgId, mapDlrState(dlrState)); + state.filter(messageState -> messageState.getForwardDlrUrl() != null) + .filter(messageState -> !messageState.getForwardDlrUrl().isEmpty()) + .ifPresent(forwardDlrService::forwardDlr); + return state; + } + + public Optional getState(String gatewayMsgId) { + return storage.getState(gatewayMsgId); + } + + public boolean markAsFailed(String gatewayMsgId) { + return storage.markAsFailed(gatewayMsgId); + } + + public boolean saveUnpushedDlr(StandardMessage message) { + return storage.saveUnpushedDlr(message); + } + + public List getUnpushedDlrs(String systemId) { + return storage.getUnpushedDlrs(systemId); + } + + public List claimUnpushedDlrs(String systemId) { + return storage.claimUnpushedDlrs(systemId); + } + + public boolean removeUnpushedDlr(StandardMessage message) { + return storage.removeUnpushedDlr(message); + } + + public void releaseUnpushedDlrClaim(StandardMessage message) { + storage.releaseUnpushedDlrClaim(message); + } + + private MessageState.MessageStatus mapDlrState(int dlrState) { + return switch (dlrState) { + case 1, 15 -> MessageState.MessageStatus.DELIVERED; + case 5, 9 -> MessageState.MessageStatus.ACCEPTED; + default -> MessageState.MessageStatus.FAILED; + }; + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java new file mode 100644 index 0000000..9630ae5 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java @@ -0,0 +1,40 @@ +package gr.cytech.sendium.core.worker; + +import gr.cytech.sendium.core.message.StandardMessage; + +import java.util.List; +import java.util.Optional; + +/** + * Persistence boundary for delivery-receipt correlation and downstream SMPP replay state. + */ +public interface DlrStorage { + void saveInitialState(MessageState state); + + void linkOperatorId(String gatewayMsgId, String operatorMsgId); + + /** + * Resolves and removes one provider correlation and its tracked message. + * The returned state must contain the supplied status, the linked operator ID, and an updated timestamp. + */ + Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status); + + Optional getState(String gatewayMsgId); + + boolean markAsFailed(String gatewayMsgId); + + boolean saveUnpushedDlr(StandardMessage message); + + List getUnpushedDlrs(String systemId); + + /** + * Claims replayable receipts within this storage instance. V1 targets one Sendium process and does not promise + * distributed claim coordination across multiple gateway replicas. + */ + List claimUnpushedDlrs(String systemId); + + boolean removeUnpushedDlr(StandardMessage message); + + void releaseUnpushedDlrClaim(StandardMessage message); + +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryDlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java similarity index 95% rename from sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryDlrService.java rename to sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java index b829b8e..767de7c 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryDlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java @@ -5,10 +5,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import gr.cytech.sendium.core.message.StandardMessage; +import io.quarkus.arc.DefaultBean; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; import org.h2.mvstore.MVStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,8 +39,9 @@ * before returning them so concurrent reconnect callbacks for the same systemId cannot enqueue the same DLR twice. */ @ApplicationScoped -public class InMemoryDlrService { - private static final Logger logger = LoggerFactory.getLogger(InMemoryDlrService.class); +@DefaultBean +public class MvStoreDlrStorage implements DlrStorage { + private static final Logger logger = LoggerFactory.getLogger(MvStoreDlrStorage.class); private static final long SEVEN_DAYS_MILLIS = TimeUnit.DAYS.toMillis(7); private static final long THREE_DAYS_MILLIS = TimeUnit.DAYS.toMillis(3); private static final long EXPIRY_CHECK_INTERVAL = TimeUnit.HOURS.toMillis(1); @@ -52,9 +53,6 @@ public class InMemoryDlrService { private static final TypeReference> STRING_LIST_TYPE = new TypeReference<>() { }; - @Inject - ForwardDlrService forwardDlrService; - private final Object unpushedDlrStateLock = new Object(); private final Set claimedUnpushedDlrKeys = ConcurrentHashMap.newKeySet(); @@ -140,7 +138,7 @@ private void fallbackToInMemory() { @PreDestroy void onStop() { - logger.info("InMemoryDlrService shutting down"); + logger.info("MvStoreDlrStorage shutting down"); saveAndClose(); } @@ -161,6 +159,7 @@ private synchronized void saveAndClose() { } } + @Override public void saveInitialState(MessageState context) { if (primaryStore != null) { checkExpiry(); @@ -174,6 +173,7 @@ public void saveInitialState(MessageState context) { } } + @Override public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { checkExpiry(); if (primaryStore == null || correlationIndex == null) { @@ -222,7 +222,8 @@ public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { } } - public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrState) { + @Override + public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { checkExpiry(); if (correlationIndex == null || primaryStore == null) { return Optional.empty(); @@ -238,7 +239,7 @@ public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrS try { MessageState state = mapper.readValue(stateJson, MessageState.class); state.setTimestamp(System.currentTimeMillis()); - state.setStatus(mapDlrStateToMessageStatus(dlrState)); + state.setStatus(status); primaryStore.remove(gatewayMsgId); primaryTimestamps.remove(gatewayMsgId); @@ -246,11 +247,6 @@ public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrS correlationTimestamps.remove(operatorMsgId); logger.debug("Resolved and removed DLR for gatewayMsgId: {}", gatewayMsgId); - String forwardUrl = state.getForwardDlrUrl(); - if (forwardUrl != null && !forwardUrl.isEmpty()) { - forwardDlrService.forwardDlr(state); - } - return Optional.of(state); } catch (JsonProcessingException e) { logger.error("Failed to deserialize MessageState during resolve", e); @@ -261,16 +257,7 @@ public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrS return Optional.empty(); } - private MessageState.MessageStatus mapDlrStateToMessageStatus(int dlrState) { - return switch (dlrState) { - case 1 -> MessageState.MessageStatus.DELIVERED; - case 2, 3, 4, 6, 7, 8 -> MessageState.MessageStatus.FAILED; - case 5, 9 -> MessageState.MessageStatus.ACCEPTED; - case 15 -> MessageState.MessageStatus.DELIVERED; - default -> MessageState.MessageStatus.FAILED; - }; - } - + @Override public Optional getState(String gatewayMsgId) { checkExpiry(); if (primaryStore == null) { @@ -288,6 +275,7 @@ public Optional getState(String gatewayMsgId) { return Optional.empty(); } + @Override public boolean markAsFailed(String gatewayMsgId) { checkExpiry(); if (primaryStore == null) { @@ -312,6 +300,7 @@ public boolean markAsFailed(String gatewayMsgId) { /** * Persist a DLR that could not be pushed to the SMPP client. */ + @Override public boolean saveUnpushedDlr(StandardMessage msg) { checkExpiry(); if (unpushedDlrStore == null || unpushedDlrIndex == null || msg == null || msg.type != StandardMessage.MSG_DLR || @@ -338,6 +327,7 @@ public boolean saveUnpushedDlr(StandardMessage msg) { /** * Load unpushed DLRs for one SMPP systemId without marking them for replay. */ + @Override public List getUnpushedDlrs(String systemId) { return loadUnpushedDlrs(systemId, false); } @@ -345,6 +335,7 @@ public List getUnpushedDlrs(String systemId) { /** * Load and claim unpushed DLRs for replay. Claimed entries are hidden from later claims until removed or released. */ + @Override public List claimUnpushedDlrs(String systemId) { return loadUnpushedDlrs(systemId, true); } @@ -399,6 +390,7 @@ private List loadUnpushedDlrs(String systemId, boolean claimFor /** * Remove a replayed DLR from all unpushed-DLR maps. */ + @Override public boolean removeUnpushedDlr(StandardMessage msg) { if (unpushedDlrStore == null || unpushedDlrIndex == null || msg == null || msg.systemId == null || msg.systemId.isBlank()) { return false; @@ -420,6 +412,7 @@ public boolean removeUnpushedDlr(StandardMessage msg) { /** * Make a claimed but not yet removed DLR eligible for a later replay attempt. */ + @Override public void releaseUnpushedDlrClaim(StandardMessage msg) { if (msg == null || msg.systemId == null || msg.systemId.isBlank()) { return; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java index 71fcd3f..0aa715b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java @@ -4,8 +4,8 @@ import gr.cytech.sendium.core.queue.InMemoryQueueProvider; import gr.cytech.sendium.core.queue.QueueProvider; import gr.cytech.sendium.core.smpp.client.SmppClientHolder; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.ForwardMoService; -import gr.cytech.sendium.core.worker.InMemoryDlrService; import io.quarkus.arc.DefaultBean; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -25,7 +25,7 @@ public enum Visibility { INTERNAL, EXTERNAL } @Inject InMemoryQueueProvider queueProvider; @Inject CredentialFileWatcher credentialFileWatcher; - @Inject InMemoryDlrService dlrService; + @Inject DlrService dlrService; @Inject ForwardMoService forwardMoService; @Inject SmppClientHolder smppClientHolder; @@ -40,7 +40,7 @@ public CredentialFileWatcher getCredentialFileWatcher() { return credentialFileWatcher; } - public InMemoryDlrService getDlrService() { + public DlrService getDlrService() { return dlrService; } @@ -90,4 +90,4 @@ public boolean stopExecutor(ExecutorService executor, Logger errorLogger, String } return false; } -} \ No newline at end of file +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java index 634d735..62a781e 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java @@ -2,7 +2,7 @@ import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; -import gr.cytech.sendium.core.worker.InMemoryDlrService; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.routing.OutgoingWorkerManager; import gr.cytech.sendium.routing.StandardOutgoingWorkerHandler; @@ -24,7 +24,7 @@ @QuarkusTest class KannelResourceIT { static StandardOutgoingWorkerHandler outgoingWorkerHandler; - static InMemoryDlrService dlrService; + static DlrService dlrService; CaptorWorker captorWorker; private final String usernamekannel = "test2"; @@ -34,7 +34,7 @@ class KannelResourceIT { @BeforeAll static void beforeAll() { outgoingWorkerHandler = (StandardOutgoingWorkerHandler) CDI.current().select(OutgoingWorkerManager.class).get(); - dlrService = CDI.current().select(InMemoryDlrService.class).get(); + dlrService = CDI.current().select(DlrService.class).get(); } @BeforeEach diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java index e63447c..70ad610 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java @@ -1,7 +1,7 @@ package gr.cytech.sendium.core.smpp.server; import gr.cytech.sendium.core.message.StandardMessage; -import gr.cytech.sendium.core.worker.InMemoryDlrService; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.external.WorkerResourceProvider; import org.junit.jupiter.api.BeforeEach; @@ -31,7 +31,7 @@ class InMemorySmppServerMessageStoreTest { private WorkerResourceProvider workerResources; @Mock - private InMemoryDlrService dlrService; + private DlrService dlrService; private InMemorySmppServerMessageStore messageStore; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java new file mode 100644 index 0000000..d93ecda --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java @@ -0,0 +1,81 @@ +package gr.cytech.sendium.core.worker; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DlrServiceTest { + @Mock + DlrStorage storage; + + @Mock + ForwardDlrService forwardDlrService; + + @InjectMocks + DlrService service; + + @Test + void resolveAndRemoveDlrMapsDeliveredStateAndForwardsCallback() { + MessageState state = stateWithCallback(); + when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.DELIVERED)) + .thenReturn(Optional.of(state)); + + Optional result = service.resolveAndRemoveDlr("operator-1", 1); + + assertThat(result).containsSame(state); + verify(forwardDlrService).forwardDlr(state); + } + + @Test + void resolveAndRemoveDlrMapsAcceptedState() { + MessageState state = stateWithoutCallback(); + when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.ACCEPTED)) + .thenReturn(Optional.of(state)); + + Optional result = service.resolveAndRemoveDlr("operator-1", 9); + + assertThat(result).containsSame(state); + verify(forwardDlrService, never()).forwardDlr(state); + } + + @Test + void resolveAndRemoveDlrMapsUnknownStateToFailed() { + MessageState state = stateWithoutCallback(); + when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.FAILED)) + .thenReturn(Optional.of(state)); + + Optional result = service.resolveAndRemoveDlr("operator-1", 0); + + assertThat(result).containsSame(state); + } + + @Test + void resolveAndRemoveDlrDoesNotForwardMissingState() { + when(storage.resolveAndRemoveDlr("unknown", MessageState.MessageStatus.DELIVERED)) + .thenReturn(Optional.empty()); + + Optional result = service.resolveAndRemoveDlr("unknown", 15); + + assertThat(result).isEmpty(); + verify(forwardDlrService, never()).forwardDlr(org.mockito.ArgumentMatchers.any()); + } + + private MessageState stateWithCallback() { + return new MessageState("gateway-1", "account", "system", "source", "destination", + "https://example.test/dlr"); + } + + private MessageState stateWithoutCallback() { + return new MessageState("gateway-1", "account", "system", "source", "destination", null); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java index 71a211d..47a9f48 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java @@ -32,7 +32,7 @@ class InMemoryMessageTrackerTest { private WorkerResourceProvider workerResources; @Mock - private InMemoryDlrService dlrService; + private DlrService dlrService; private InMemoryMessageTracker tracker; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryDlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java similarity index 60% rename from sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryDlrServiceTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java index 9da35cd..f0b9385 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryDlrServiceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java @@ -14,9 +14,9 @@ import static org.junit.jupiter.api.Assertions.*; -class InMemoryDlrServiceTest { +class MvStoreDlrStorageTest { - private InMemoryDlrService dlrService; + private MvStoreDlrStorage storage; private Path dbPath; private String oldDbPath; @@ -26,14 +26,14 @@ void setUp() throws Exception { dbPath = Files.createTempFile("dlr-service-test", ".db"); Files.deleteIfExists(dbPath); System.setProperty("sendium.dlr.db.path", dbPath.toString()); - dlrService = new InMemoryDlrService(); - dlrService.init(); + storage = new MvStoreDlrStorage(); + storage.init(); } @AfterEach void tearDown() throws Exception { - if (dlrService != null) { - dlrService.onStop(); + if (storage != null) { + storage.onStop(); } if (oldDbPath == null) { System.clearProperty("sendium.dlr.db.path"); @@ -49,18 +49,18 @@ void tearDown() throws Exception { void saveInitialState_StoresInPrimaryStore() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - assertEquals(1, dlrService.getPrimaryStoreSize()); + assertEquals(1, storage.getPrimaryStoreSize()); } @Test void saveInitialState_SetsTimestamp() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - Optional retrieved = dlrService.getState("gw-123"); + Optional retrieved = storage.getState("gw-123"); assertTrue(retrieved.isPresent()); assertTrue(retrieved.get().getTimestamp() > 0); } @@ -68,21 +68,21 @@ void saveInitialState_SetsTimestamp() { @Test void linkOperatorId_LinksCorrelation() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - dlrService.linkOperatorId("gw-123", "op-456"); + storage.linkOperatorId("gw-123", "op-456"); - assertEquals(1, dlrService.getCorrelationIndexSize()); + assertEquals(1, storage.getCorrelationIndexSize()); } @Test void linkOperatorId_UpdatesStatusToSent() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - dlrService.linkOperatorId("gw-123", "op-456"); + storage.linkOperatorId("gw-123", "op-456"); - Optional retrieved = dlrService.getState("gw-123"); + Optional retrieved = storage.getState("gw-123"); assertTrue(retrieved.isPresent()); assertEquals(MessageState.MessageStatus.SENT, retrieved.get().getStatus()); } @@ -90,19 +90,25 @@ void linkOperatorId_UpdatesStatusToSent() { @Test void resolveAndRemoveDlr_ReturnsAndRemoves() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); - dlrService.linkOperatorId("gw-123", "op-456"); + storage.saveInitialState(state); + storage.linkOperatorId("gw-123", "op-456"); - Optional result = dlrService.resolveAndRemoveDlr("op-456", 1); + long beforeResolve = System.currentTimeMillis(); + Optional result = storage.resolveAndRemoveDlr( + "op-456", MessageState.MessageStatus.DELIVERED); assertTrue(result.isPresent()); - assertEquals(0, dlrService.getPrimaryStoreSize()); - assertEquals(0, dlrService.getCorrelationIndexSize()); + assertEquals(MessageState.MessageStatus.DELIVERED, result.get().getStatus()); + assertEquals("op-456", result.get().getOperatorMsgId()); + assertTrue(result.get().getTimestamp() >= beforeResolve); + assertEquals(0, storage.getPrimaryStoreSize()); + assertEquals(0, storage.getCorrelationIndexSize()); } @Test void resolveAndRemoveDlr_MissingId_ReturnsEmpty() { - Optional result = dlrService.resolveAndRemoveDlr("unknown", 1); + Optional result = storage.resolveAndRemoveDlr( + "unknown", MessageState.MessageStatus.DELIVERED); assertTrue(result.isEmpty()); } @@ -110,9 +116,9 @@ void resolveAndRemoveDlr_MissingId_ReturnsEmpty() { @Test void getState_ReturnsWrappedState() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - Optional result = dlrService.getState("gw-123"); + Optional result = storage.getState("gw-123"); assertTrue(result.isPresent()); assertEquals("gw-123", result.get().getGatewayMsgId()); @@ -120,7 +126,7 @@ void getState_ReturnsWrappedState() { @Test void getState_MissingId_ReturnsEmpty() { - Optional result = dlrService.getState("unknown"); + Optional result = storage.getState("unknown"); assertTrue(result.isEmpty()); } @@ -128,19 +134,19 @@ void getState_MissingId_ReturnsEmpty() { @Test void markAsFailed_UpdatesStatusToFailed() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - dlrService.saveInitialState(state); + storage.saveInitialState(state); - boolean result = dlrService.markAsFailed("gw-123"); + boolean result = storage.markAsFailed("gw-123"); assertTrue(result); - Optional updated = dlrService.getState("gw-123"); + Optional updated = storage.getState("gw-123"); assertTrue(updated.isPresent()); assertEquals(MessageState.MessageStatus.FAILED, updated.get().getStatus()); } @Test void markAsFailed_MissingId_ReturnsFalse() { - boolean result = dlrService.markAsFailed("unknown"); + boolean result = storage.markAsFailed("unknown"); assertFalse(result); } @@ -149,8 +155,8 @@ void markAsFailed_MissingId_ReturnsFalse() { void saveUnpushedDlr_StoresAndReturnsMatchingDlr() { StandardMessage dlr = createDlr("account1", "sys1"); - boolean result = dlrService.saveUnpushedDlr(dlr); - List dlrs = dlrService.getUnpushedDlrs("sys1"); + boolean result = storage.saveUnpushedDlr(dlr); + List dlrs = storage.getUnpushedDlrs("sys1"); assertTrue(result); assertTrue(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); @@ -160,14 +166,14 @@ void saveUnpushedDlr_StoresAndReturnsMatchingDlr() { assertEquals(dlr.acked, stored.acked); assertEquals(dlr.priority, stored.priority); assertEquals(dlr.reassembledParts, stored.reassembledParts); - assertEquals(1, dlrService.getUnpushedDlrIndexSize()); + assertEquals(1, storage.getUnpushedDlrIndexSize()); } @Test void saveUnpushedDlr_BlankSystemIdReturnsFalse() { StandardMessage dlr = createDlr("account1", null); - boolean result = dlrService.saveUnpushedDlr(dlr); + boolean result = storage.saveUnpushedDlr(dlr); assertFalse(result); } @@ -175,9 +181,9 @@ void saveUnpushedDlr_BlankSystemIdReturnsFalse() { @Test void getUnpushedDlrs_DifferentSystemIdDoesNotMatch() { StandardMessage dlr = createDlr("account1", "sys1"); - dlrService.saveUnpushedDlr(dlr); + storage.saveUnpushedDlr(dlr); - List dlrs = dlrService.getUnpushedDlrs("sys2"); + List dlrs = storage.getUnpushedDlrs("sys2"); assertFalse(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); } @@ -186,12 +192,12 @@ void getUnpushedDlrs_DifferentSystemIdDoesNotMatch() { void getUnpushedDlrs_UsesSystemIdIndex() { StandardMessage sys1Dlr = createDlr("account1", "sys1"); StandardMessage sys2Dlr = createDlr("account2", "sys2"); - dlrService.saveUnpushedDlr(sys1Dlr); - dlrService.saveUnpushedDlr(sys2Dlr); + storage.saveUnpushedDlr(sys1Dlr); + storage.saveUnpushedDlr(sys2Dlr); - List dlrs = dlrService.getUnpushedDlrs("sys1"); + List dlrs = storage.getUnpushedDlrs("sys1"); - assertEquals(2, dlrService.getUnpushedDlrIndexSize()); + assertEquals(2, storage.getUnpushedDlrIndexSize()); assertTrue(dlrs.stream().anyMatch(msg -> sys1Dlr.serial.equals(msg.serial))); assertFalse(dlrs.stream().anyMatch(msg -> sys2Dlr.serial.equals(msg.serial))); } @@ -199,25 +205,25 @@ void getUnpushedDlrs_UsesSystemIdIndex() { @Test void removeUnpushedDlr_RemovesStoredDlr() { StandardMessage dlr = createDlr("account1", "sys1"); - dlrService.saveUnpushedDlr(dlr); + storage.saveUnpushedDlr(dlr); - boolean result = dlrService.removeUnpushedDlr(dlr); - List dlrs = dlrService.getUnpushedDlrs("sys1"); + boolean result = storage.removeUnpushedDlr(dlr); + List dlrs = storage.getUnpushedDlrs("sys1"); assertTrue(result); assertFalse(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); - assertEquals(0, dlrService.getUnpushedDlrIndexSize()); + assertEquals(0, storage.getUnpushedDlrIndexSize()); } @Test void claimUnpushedDlrs_HidesClaimedDlrUntilReleased() { StandardMessage dlr = createDlr("account1", "sys1"); - dlrService.saveUnpushedDlr(dlr); + storage.saveUnpushedDlr(dlr); - List firstClaim = dlrService.claimUnpushedDlrs("sys1"); - List secondClaim = dlrService.claimUnpushedDlrs("sys1"); - dlrService.releaseUnpushedDlrClaim(firstClaim.getFirst()); - List afterRelease = dlrService.claimUnpushedDlrs("sys1"); + List firstClaim = storage.claimUnpushedDlrs("sys1"); + List secondClaim = storage.claimUnpushedDlrs("sys1"); + storage.releaseUnpushedDlrClaim(firstClaim.getFirst()); + List afterRelease = storage.claimUnpushedDlrs("sys1"); assertEquals(1, firstClaim.size()); assertTrue(secondClaim.isEmpty()); @@ -229,38 +235,38 @@ void claimUnpushedDlrs_HidesClaimedDlrUntilReleased() { void unpushedDlrs_SurviveRestart() throws Exception { StandardMessage dlr = createDlr("account-restart", "sys-restart"); - assertTrue(dlrService.saveUnpushedDlr(dlr)); - dlrService.onStop(); + assertTrue(storage.saveUnpushedDlr(dlr)); + storage.onStop(); - dlrService = new InMemoryDlrService(); - dlrService.init(); - List dlrs = dlrService.getUnpushedDlrs("sys-restart"); + storage = new MvStoreDlrStorage(); + storage.init(); + List dlrs = storage.getUnpushedDlrs("sys-restart"); assertTrue(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); } @Test void getPrimaryStoreSize_ReturnsCount() { - dlrService.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); - dlrService.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); - dlrService.saveInitialState(new MessageState("gw-3", "systemId", "from", "to", null)); + storage.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); + storage.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); + storage.saveInitialState(new MessageState("gw-3", "systemId", "from", "to", null)); - assertEquals(3, dlrService.getPrimaryStoreSize()); + assertEquals(3, storage.getPrimaryStoreSize()); } @Test void getCorrelationIndexSize_ReturnsCount() { - dlrService.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); - dlrService.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); - dlrService.linkOperatorId("gw-1", "op-1"); - dlrService.linkOperatorId("gw-2", "op-2"); + storage.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); + storage.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); + storage.linkOperatorId("gw-1", "op-1"); + storage.linkOperatorId("gw-2", "op-2"); - assertEquals(2, dlrService.getCorrelationIndexSize()); + assertEquals(2, storage.getCorrelationIndexSize()); } @Test void isPersistent_TrueWhenDbAvailable() { - assertTrue(dlrService.isPersistent()); + assertTrue(storage.isPersistent()); } private StandardMessage createDlr(String accountId, String systemId) { From 4ed01cf06a1ecb2d8d2d9826ff9f92e51c3d9c66 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 14:38:36 +0300 Subject: [PATCH 03/20] feat(dlr): add PostgreSQL message state storage --- .../core/worker/DlrMessageStorage.java | 19 + .../sendium/core/worker/DlrStorage.java | 17 +- .../core/worker/DlrStorageException.java | 11 + .../worker/PostgresqlMessageStateStorage.java | 406 ++++++++++++++++++ .../V1__create_sendium_dlr_schema.sql | 1 + .../core/dlr/PostgresqlMigrationTest.java | 3 +- .../PostgresqlMessageStateStorageTest.java | 323 ++++++++++++++ 7 files changed, 763 insertions(+), 17 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageException.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java new file mode 100644 index 0000000..d8ea99f --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java @@ -0,0 +1,19 @@ +package gr.cytech.sendium.core.worker; + +import java.util.Optional; + +public interface DlrMessageStorage { + void saveInitialState(MessageState state); + + void linkOperatorId(String gatewayMsgId, String operatorMsgId); + + /** + * Resolves and removes one provider correlation and its tracked message. + * The returned state must contain the supplied status, the linked operator ID, and an updated timestamp. + */ + Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status); + + Optional getState(String gatewayMsgId); + + boolean markAsFailed(String gatewayMsgId); +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java index 9630ae5..7378952 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java @@ -3,26 +3,11 @@ import gr.cytech.sendium.core.message.StandardMessage; import java.util.List; -import java.util.Optional; /** * Persistence boundary for delivery-receipt correlation and downstream SMPP replay state. */ -public interface DlrStorage { - void saveInitialState(MessageState state); - - void linkOperatorId(String gatewayMsgId, String operatorMsgId); - - /** - * Resolves and removes one provider correlation and its tracked message. - * The returned state must contain the supplied status, the linked operator ID, and an updated timestamp. - */ - Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status); - - Optional getState(String gatewayMsgId); - - boolean markAsFailed(String gatewayMsgId); - +public interface DlrStorage extends DlrMessageStorage { boolean saveUnpushedDlr(StandardMessage message); List getUnpushedDlrs(String systemId); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageException.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageException.java new file mode 100644 index 0000000..25856e5 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageException.java @@ -0,0 +1,11 @@ +package gr.cytech.sendium.core.worker; + +public class DlrStorageException extends RuntimeException { + public DlrStorageException(String message) { + super(message); + } + + public DlrStorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java new file mode 100644 index 0000000..55a1fad --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java @@ -0,0 +1,406 @@ +package gr.cytech.sendium.core.worker; + +import javax.sql.DataSource; +import java.sql.Array; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +public class PostgresqlMessageStateStorage implements DlrMessageStorage { + private static final int DEFAULT_LINK_MAX_ATTEMPTS = 20; + private static final long DEFAULT_LINK_RETRY_INTERVAL_MILLIS = 200; + private static final long EXPIRY_CHECK_INTERVAL_MILLIS = TimeUnit.HOURS.toMillis(1); + + private static final String SAVE_INITIAL_STATE_SQL = """ + INSERT INTO sendium_dlr.tracked_message + (gateway_message_id, account_id, system_id, source_address, destination_address, + operator_message_id, forward_dlr_url, reassembled_parts, status, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (gateway_message_id) DO UPDATE SET + account_id = EXCLUDED.account_id, + system_id = EXCLUDED.system_id, + source_address = EXCLUDED.source_address, + destination_address = EXCLUDED.destination_address, + operator_message_id = EXCLUDED.operator_message_id, + forward_dlr_url = EXCLUDED.forward_dlr_url, + reassembled_parts = EXCLUDED.reassembled_parts, + status = EXCLUDED.status, + created_at = CURRENT_TIMESTAMP, + updated_at = EXCLUDED.updated_at + """; + + private static final String LINK_MESSAGE_SQL = """ + UPDATE sendium_dlr.tracked_message + SET operator_message_id = ?, status = 'SENT', updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? + """; + + private static final String SAVE_CORRELATION_SQL = """ + INSERT INTO sendium_dlr.operator_correlation + (operator_message_id, gateway_message_id) + VALUES (?, ?) + ON CONFLICT (operator_message_id) DO UPDATE SET + created_at = CURRENT_TIMESTAMP + WHERE operator_correlation.gateway_message_id = EXCLUDED.gateway_message_id + """; + + private static final String DELETE_CORRELATIONS_SQL = """ + DELETE FROM sendium_dlr.operator_correlation + WHERE gateway_message_id = ? + """; + + private static final String GET_STATE_SQL = """ + SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, + tm.destination_address, tm.operator_message_id, tm.forward_dlr_url, + tm.reassembled_parts, tm.status, tm.updated_at + FROM sendium_dlr.tracked_message tm + WHERE tm.gateway_message_id = ? + """; + + private static final String RESOLVE_STATE_SQL = """ + SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, + tm.destination_address, tm.forward_dlr_url, tm.reassembled_parts, + correlation.operator_message_id, CURRENT_TIMESTAMP AS resolved_at + FROM sendium_dlr.operator_correlation correlation + JOIN sendium_dlr.tracked_message tm + ON tm.gateway_message_id = correlation.gateway_message_id + WHERE correlation.operator_message_id = ? + FOR UPDATE OF tm, correlation + """; + + private static final String DELETE_STATE_SQL = """ + DELETE FROM sendium_dlr.tracked_message + WHERE gateway_message_id = ? + """; + + private static final String MARK_FAILED_SQL = """ + UPDATE sendium_dlr.tracked_message + SET status = 'FAILED', updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? + """; + + private static final String DELETE_EXPIRED_CORRELATIONS_SQL = """ + DELETE FROM sendium_dlr.operator_correlation + WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '3 days' + """; + + private static final String DELETE_EXPIRED_MESSAGES_SQL = """ + DELETE FROM sendium_dlr.tracked_message + WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' + """; + + private final DataSource dataSource; + private final int linkMaxAttempts; + private final long linkRetryIntervalMillis; + private volatile long lastExpiryCheck; + + public PostgresqlMessageStateStorage(DataSource dataSource) { + this(dataSource, DEFAULT_LINK_MAX_ATTEMPTS, DEFAULT_LINK_RETRY_INTERVAL_MILLIS); + } + + PostgresqlMessageStateStorage(DataSource dataSource, int linkMaxAttempts, + long linkRetryIntervalMillis) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + if (linkMaxAttempts < 1 || linkRetryIntervalMillis < 0) { + throw new IllegalArgumentException("Invalid operator-link retry policy"); + } + this.linkMaxAttempts = linkMaxAttempts; + this.linkRetryIntervalMillis = linkRetryIntervalMillis; + } + + @Override + public void saveInitialState(MessageState state) { + Objects.requireNonNull(state, "state"); + checkExpiry(); + + UUID gatewayMsgId = parseGatewayId(state.getGatewayMsgId()); + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try { + saveState(connection, gatewayMsgId, state); + deleteCorrelations(connection, gatewayMsgId); + if (state.getOperatorMsgId() != null && + !saveCorrelation(connection, gatewayMsgId, state.getOperatorMsgId())) { + throw new SQLException("Operator message ID is already linked to another gateway message"); + } + connection.commit(); + } catch (SQLException e) { + rollback(connection, e); + throw e; + } + } catch (SQLException e) { + throw failure("save initial DLR state", e); + } + } + + @Override + public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { + checkExpiry(); + UUID gatewayId = parseGatewayId(gatewayMsgId); + + for (int attempt = 0; attempt < linkMaxAttempts; attempt++) { + if (tryLinkOperatorId(gatewayId, operatorMsgId)) { + return; + } + if (attempt + 1 < linkMaxAttempts) { + sleepBeforeLinkRetry(); + } + } + throw new DlrStorageException("Gateway message state not found while linking operator ID"); + } + + @Override + public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { + Objects.requireNonNull(status, "status"); + checkExpiry(); + + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try { + Optional state = lockResolvedState(connection, operatorMsgId, status); + if (state.isEmpty()) { + connection.rollback(); + return Optional.empty(); + } + deleteState(connection, parseGatewayId(state.get().getGatewayMsgId())); + connection.commit(); + return state; + } catch (SQLException e) { + rollback(connection, e); + throw e; + } + } catch (SQLException e) { + throw failure("resolve DLR state", e); + } + } + + @Override + public Optional getState(String gatewayMsgId) { + checkExpiry(); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(GET_STATE_SQL)) { + statement.setObject(1, parseGatewayId(gatewayMsgId)); + try (ResultSet resultSet = statement.executeQuery()) { + return resultSet.next() ? Optional.of(readState(resultSet)) : Optional.empty(); + } + } catch (SQLException e) { + throw failure("read DLR state", e); + } + } + + @Override + public boolean markAsFailed(String gatewayMsgId) { + checkExpiry(); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(MARK_FAILED_SQL)) { + statement.setObject(1, parseGatewayId(gatewayMsgId)); + return statement.executeUpdate() == 1; + } catch (SQLException e) { + throw failure("mark DLR state as failed", e); + } + } + + private boolean tryLinkOperatorId(UUID gatewayMsgId, String operatorMsgId) { + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try { + if (!markAsSent(connection, gatewayMsgId, operatorMsgId)) { + connection.rollback(); + return false; + } + if (!saveCorrelation(connection, gatewayMsgId, operatorMsgId)) { + throw new SQLException("Operator message ID is already linked to another gateway message"); + } + connection.commit(); + return true; + } catch (SQLException e) { + rollback(connection, e); + throw e; + } + } catch (SQLException e) { + throw failure("link operator DLR ID", e); + } + } + + private boolean markAsSent(Connection connection, UUID gatewayMsgId, + String operatorMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(LINK_MESSAGE_SQL)) { + statement.setString(1, operatorMsgId); + statement.setObject(2, gatewayMsgId); + return statement.executeUpdate() == 1; + } + } + + private void saveState(Connection connection, UUID gatewayMsgId, + MessageState state) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(SAVE_INITIAL_STATE_SQL)) { + statement.setObject(1, gatewayMsgId); + statement.setString(2, state.getAccountId()); + statement.setString(3, state.getSystemId()); + statement.setString(4, state.getSourceAddr()); + statement.setString(5, state.getDestAddr()); + statement.setString(6, state.getOperatorMsgId()); + statement.setString(7, state.getForwardDlrUrl()); + setStringArray(connection, statement, 8, state.getReassembledParts()); + statement.setString(9, state.getStatus().name()); + statement.setTimestamp(10, new Timestamp(state.getTimestamp())); + statement.executeUpdate(); + } + } + + private void deleteCorrelations(Connection connection, UUID gatewayMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(DELETE_CORRELATIONS_SQL)) { + statement.setObject(1, gatewayMsgId); + statement.executeUpdate(); + } + } + + private boolean saveCorrelation(Connection connection, UUID gatewayMsgId, + String operatorMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(SAVE_CORRELATION_SQL)) { + statement.setString(1, operatorMsgId); + statement.setObject(2, gatewayMsgId); + return statement.executeUpdate() == 1; + } + } + + private Optional lockResolvedState(Connection connection, String operatorMsgId, + MessageState.MessageStatus status) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(RESOLVE_STATE_SQL)) { + statement.setString(1, operatorMsgId); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + return Optional.empty(); + } + MessageState state = readState(resultSet); + state.setOperatorMsgId(resultSet.getString("operator_message_id")); + state.setStatus(status); + state.setTimestamp(resultSet.getTimestamp("resolved_at").getTime()); + return Optional.of(state); + } + } + } + + private void deleteState(Connection connection, UUID gatewayMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(DELETE_STATE_SQL)) { + statement.setObject(1, gatewayMsgId); + statement.executeUpdate(); + } + } + + private MessageState readState(ResultSet resultSet) throws SQLException { + MessageState state = new MessageState( + resultSet.getObject("gateway_message_id", UUID.class).toString(), + resultSet.getString("account_id"), + resultSet.getString("system_id"), + resultSet.getString("source_address"), + resultSet.getString("destination_address"), + resultSet.getString("forward_dlr_url")); + state.setOperatorMsgId(resultSet.getString("operator_message_id")); + if (hasColumn(resultSet, "status")) { + state.setStatus(MessageState.MessageStatus.valueOf(resultSet.getString("status"))); + } + state.setReassembledParts(readStringArray(resultSet, "reassembled_parts")); + if (hasColumn(resultSet, "updated_at")) { + state.setTimestamp(resultSet.getTimestamp("updated_at").getTime()); + } + return state; + } + + private boolean hasColumn(ResultSet resultSet, String columnName) throws SQLException { + for (int index = 1; index <= resultSet.getMetaData().getColumnCount(); index++) { + if (columnName.equalsIgnoreCase(resultSet.getMetaData().getColumnLabel(index))) { + return true; + } + } + return false; + } + + private List readStringArray(ResultSet resultSet, String columnName) throws SQLException { + Array array = resultSet.getArray(columnName); + if (array == null) { + return null; + } + return new ArrayList<>(List.of((String[]) array.getArray())); + } + + private void setStringArray(Connection connection, PreparedStatement statement, int index, + List values) throws SQLException { + if (values == null) { + statement.setNull(index, Types.ARRAY); + return; + } + statement.setArray(index, connection.createArrayOf("text", values.toArray(String[]::new))); + } + + private void sleepBeforeLinkRetry() { + try { + Thread.sleep(linkRetryIntervalMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DlrStorageException("Interrupted while linking operator ID", e); + } + } + + private void checkExpiry() { + long now = System.currentTimeMillis(); + if (now - lastExpiryCheck < EXPIRY_CHECK_INTERVAL_MILLIS) { + return; + } + synchronized (this) { + if (now - lastExpiryCheck < EXPIRY_CHECK_INTERVAL_MILLIS) { + return; + } + deleteExpiredState(); + lastExpiryCheck = now; + } + } + + private void deleteExpiredState() { + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); + PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL)) { + correlations.executeUpdate(); + messages.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + rollback(connection, e); + throw e; + } + } catch (SQLException e) { + throw failure("expire DLR state", e); + } + } + + private UUID parseGatewayId(String gatewayMsgId) { + try { + return UUID.fromString(gatewayMsgId); + } catch (IllegalArgumentException | NullPointerException e) { + throw new DlrStorageException("Invalid gateway message ID", e); + } + } + + private void rollback(Connection connection, SQLException failure) { + try { + connection.rollback(); + } catch (SQLException rollbackFailure) { + failure.addSuppressed(rollbackFailure); + } + } + + private DlrStorageException failure(String operation, SQLException cause) { + return new DlrStorageException("Failed to " + operation, cause); + } +} diff --git a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql index a9c052f..aac2c28 100644 --- a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql +++ b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql @@ -6,6 +6,7 @@ CREATE TABLE sendium_dlr.tracked_message ( system_id TEXT, source_address TEXT, destination_address TEXT, + operator_message_id TEXT, forward_dlr_url TEXT, reassembled_parts TEXT[], status TEXT NOT NULL, diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java index aa06b87..7081184 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java @@ -133,7 +133,7 @@ void typedColumnsStoreCurrentDlrState() throws SQLException { insertCompleteUnpushedDlr(connection); try (PreparedStatement statement = connection.prepareStatement(""" - SELECT gateway_message_id, reassembled_parts, created_at, updated_at + SELECT gateway_message_id, operator_message_id, reassembled_parts, created_at, updated_at FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? """)) { @@ -142,6 +142,7 @@ void typedColumnsStoreCurrentDlrState() throws SQLException { assertThat(resultSet.next()).isTrue(); assertThat(resultSet.getObject("gateway_message_id", UUID.class)) .isEqualTo(COMPLETE_GATEWAY_ID); + assertThat(resultSet.getString("operator_message_id")).isNull(); assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) .containsExactly("part-1", "part-2"); assertThat(resultSet.getObject("created_at")).isNotNull(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java new file mode 100644 index 0000000..e7d15f0 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java @@ -0,0 +1,323 @@ +package gr.cytech.sendium.core.worker; + +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.postgresql.PostgreSQLContainer; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") +class PostgresqlMessageStateStorageTest { + private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; + private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") + .withDatabaseName("sendium") + .withUsername("sendium") + .withPassword("sendium-test"); + + private static DataSource dataSource; + + private PostgresqlMessageStateStorage storage; + + @BeforeAll + static void startPostgresql() { + POSTGRESQL.start(); + Flyway.configure() + .dataSource(POSTGRESQL.getJdbcUrl(), POSTGRESQL.getUsername(), POSTGRESQL.getPassword()) + .locations(MIGRATION_LOCATION) + .load() + .migrate(); + + PGSimpleDataSource postgresDataSource = new PGSimpleDataSource(); + postgresDataSource.setUrl(POSTGRESQL.getJdbcUrl()); + postgresDataSource.setUser(POSTGRESQL.getUsername()); + postgresDataSource.setPassword(POSTGRESQL.getPassword()); + dataSource = postgresDataSource; + } + + @AfterAll + static void stopPostgresql() { + POSTGRESQL.stop(); + } + + @BeforeEach + void resetStorage() throws SQLException { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("TRUNCATE sendium_dlr.tracked_message CASCADE"); + } + storage = new PostgresqlMessageStateStorage(dataSource); + } + + @Test + void saveInitialStateRoundTripsAllFields() { + MessageState state = newState(); + state.setOperatorMsgId("operator-initial"); + state.setReassembledParts(List.of("part-1", "part-2")); + + storage.saveInitialState(state); + + assertThat(storage.getState(state.getGatewayMsgId())) + .get() + .usingRecursiveComparison() + .isEqualTo(state); + assertThat(storage.resolveAndRemoveDlr("operator-initial", MessageState.MessageStatus.DELIVERED)) + .isPresent(); + } + + @Test + void saveInitialStateOverwritesExistingState() throws SQLException { + MessageState initial = newState(); + storage.saveInitialState(initial); + storage.linkOperatorId(initial.getGatewayMsgId(), "operator-old"); + + MessageState replacement = new MessageState(initial.getGatewayMsgId(), "replacement-account", + "replacement-system", "replacement-source", "replacement-destination", null); + replacement.setStatus(MessageState.MessageStatus.FAILED); + storage.saveInitialState(replacement); + + assertThat(storage.getState(initial.getGatewayMsgId())) + .get() + .usingRecursiveComparison() + .isEqualTo(replacement); + assertThat(countCorrelations(initial.getGatewayMsgId())).isZero(); + assertThat(storage.resolveAndRemoveDlr("operator-old", MessageState.MessageStatus.DELIVERED)) + .isEmpty(); + } + + @Test + void saveInitialStateRollsBackOnCorrelationOwnedByAnotherMessage() throws SQLException { + MessageState owner = newState(); + storage.saveInitialState(owner); + storage.linkOperatorId(owner.getGatewayMsgId(), "shared-operator"); + + MessageState target = newState(); + storage.saveInitialState(target); + storage.linkOperatorId(target.getGatewayMsgId(), "target-operator"); + MessageState targetBeforeReplacement = storage.getState(target.getGatewayMsgId()).orElseThrow(); + + MessageState replacement = new MessageState(target.getGatewayMsgId(), "replacement-account", + "replacement-system", "replacement-source", "replacement-destination", null); + replacement.setOperatorMsgId("shared-operator"); + assertThatThrownBy(() -> storage.saveInitialState(replacement)) + .isInstanceOf(DlrStorageException.class); + + assertThat(storage.getState(target.getGatewayMsgId())) + .get() + .usingRecursiveComparison() + .isEqualTo(targetBeforeReplacement); + assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getOperatorMsgId()) + .isEqualTo("shared-operator"); + assertThat(countCorrelations(target.getGatewayMsgId())).isOne(); + assertThat(countCorrelations(owner.getGatewayMsgId())).isOne(); + + MessageState newConflict = newState(); + newConflict.setOperatorMsgId("shared-operator"); + assertThatThrownBy(() -> storage.saveInitialState(newConflict)) + .isInstanceOf(DlrStorageException.class); + assertThat(storage.getState(newConflict.getGatewayMsgId())).isEmpty(); + } + + @Test + void linkOperatorIdUpdatesStateAndKeepsMultipleCorrelations() throws SQLException { + MessageState state = newState(); + storage.saveInitialState(state); + + storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + + MessageState linked = storage.getState(state.getGatewayMsgId()).orElseThrow(); + assertThat(linked.getStatus()).isEqualTo(MessageState.MessageStatus.SENT); + assertThat(linked.getOperatorMsgId()).isEqualTo("operator-2"); + assertThat(countCorrelations(state.getGatewayMsgId())).isEqualTo(2); + } + + @Test + void linkOperatorIdRollsBackStateWhenCorrelationInsertFails() { + MessageState state = newState(); + storage.saveInitialState(state); + + assertThatThrownBy(() -> storage.linkOperatorId(state.getGatewayMsgId(), null)) + .isInstanceOf(DlrStorageException.class); + + MessageState unchanged = storage.getState(state.getGatewayMsgId()).orElseThrow(); + assertThat(unchanged.getStatus()).isEqualTo(MessageState.MessageStatus.ACCEPTED); + assertThat(unchanged.getOperatorMsgId()).isNull(); + } + + @Test + void linkOperatorIdRejectsCorrelationOwnedByAnotherMessage() throws SQLException { + MessageState first = newState(); + MessageState second = newState(); + storage.saveInitialState(first); + storage.saveInitialState(second); + storage.linkOperatorId(first.getGatewayMsgId(), "shared-operator"); + + assertThatThrownBy(() -> storage.linkOperatorId(second.getGatewayMsgId(), "shared-operator")) + .isInstanceOf(DlrStorageException.class); + + assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getOperatorMsgId()) + .isEqualTo("shared-operator"); + MessageState unchanged = storage.getState(second.getGatewayMsgId()).orElseThrow(); + assertThat(unchanged.getStatus()).isEqualTo(MessageState.MessageStatus.ACCEPTED); + assertThat(unchanged.getOperatorMsgId()).isNull(); + assertThat(countCorrelations(first.getGatewayMsgId())).isOne(); + assertThat(countCorrelations(second.getGatewayMsgId())).isZero(); + } + + @Test + void linkOperatorIdFailsWhenGatewayStateDoesNotAppear() { + PostgresqlMessageStateStorage noRetryStorage = + new PostgresqlMessageStateStorage(dataSource, 1, 0); + + assertThatThrownBy(() -> noRetryStorage.linkOperatorId(UUID.randomUUID().toString(), "operator")) + .isInstanceOf(DlrStorageException.class) + .hasMessageContaining("not found"); + } + + @Test + void resolveAndRemoveDlrReturnsUpdatedStateAndDeletesAllCorrelations() throws SQLException { + MessageState state = newState(); + storage.saveInitialState(state); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + long beforeResolve = System.currentTimeMillis(); + + Optional resolved = storage.resolveAndRemoveDlr( + "operator-1", MessageState.MessageStatus.DELIVERED); + + assertThat(resolved).isPresent(); + assertThat(resolved.orElseThrow().getStatus()).isEqualTo(MessageState.MessageStatus.DELIVERED); + assertThat(resolved.orElseThrow().getOperatorMsgId()).isEqualTo("operator-1"); + assertThat(resolved.orElseThrow().getTimestamp()).isGreaterThanOrEqualTo(beforeResolve); + assertThat(storage.getState(state.getGatewayMsgId())).isEmpty(); + assertThat(countCorrelations(state.getGatewayMsgId())).isZero(); + } + + @Test + void concurrentResolveAcrossCorrelationsReturnsStateOnlyOnce() throws Exception { + MessageState state = newState(); + storage.saveInitialState(state); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + + try { + Future> first = executor.submit( + () -> resolveWhenReleased("operator-1", ready, start)); + Future> second = executor.submit( + () -> resolveWhenReleased("operator-2", ready, start)); + ready.await(); + start.countDown(); + + assertThat(List.of(first.get(), second.get()).stream().filter(Optional::isPresent).count()) + .isOne(); + } finally { + executor.shutdownNow(); + } + } + + @Test + void markAsFailedUpdatesExistingState() { + MessageState state = newState(); + storage.saveInitialState(state); + + boolean updated = storage.markAsFailed(state.getGatewayMsgId()); + + assertThat(updated).isTrue(); + assertThat(storage.getState(state.getGatewayMsgId()).orElseThrow().getStatus()) + .isEqualTo(MessageState.MessageStatus.FAILED); + assertThat(storage.markAsFailed(UUID.randomUUID().toString())).isFalse(); + } + + @Test + void expiryRemovesOldCorrelationsAndMessages() throws SQLException { + MessageState correlationState = newState(); + storage.saveInitialState(correlationState); + storage.linkOperatorId(correlationState.getGatewayMsgId(), "old-correlation"); + ageCorrelation("old-correlation"); + + PostgresqlMessageStateStorage correlationCleanup = new PostgresqlMessageStateStorage(dataSource); + assertThat(correlationCleanup.getState(correlationState.getGatewayMsgId())).isPresent(); + assertThat(countCorrelations(correlationState.getGatewayMsgId())).isZero(); + + MessageState oldMessage = newState(); + storage.saveInitialState(oldMessage); + ageMessage(oldMessage.getGatewayMsgId()); + + PostgresqlMessageStateStorage messageCleanup = new PostgresqlMessageStateStorage(dataSource); + assertThat(messageCleanup.getState(oldMessage.getGatewayMsgId())).isEmpty(); + } + + private Optional resolveWhenReleased(String operatorMsgId, CountDownLatch ready, + CountDownLatch start) throws InterruptedException { + ready.countDown(); + start.await(); + return storage.resolveAndRemoveDlr(operatorMsgId, MessageState.MessageStatus.DELIVERED); + } + + private MessageState newState() { + return new MessageState(UUID.randomUUID().toString(), "account", "system", "source", "destination", + "https://example.test/dlr"); + } + + private int countCorrelations(String gatewayMsgId) throws SQLException { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + SELECT COUNT(*) + FROM sendium_dlr.operator_correlation + WHERE gateway_message_id = ? + """)) { + statement.setObject(1, UUID.fromString(gatewayMsgId)); + try (ResultSet resultSet = statement.executeQuery()) { + resultSet.next(); + return resultSet.getInt(1); + } + } + } + + private void ageCorrelation(String operatorMsgId) throws SQLException { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + UPDATE sendium_dlr.operator_correlation + SET created_at = CURRENT_TIMESTAMP - INTERVAL '4 days' + WHERE operator_message_id = ? + """)) { + statement.setString(1, operatorMsgId); + statement.executeUpdate(); + } + } + + private void ageMessage(String gatewayMsgId) throws SQLException { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + UPDATE sendium_dlr.tracked_message + SET created_at = CURRENT_TIMESTAMP - INTERVAL '8 days' + WHERE gateway_message_id = ? + """)) { + statement.setObject(1, UUID.fromString(gatewayMsgId)); + statement.executeUpdate(); + } + } +} From e3685b3b6fe633a01732d44e7c402f3cdc4c65e7 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 15:37:06 +0300 Subject: [PATCH 04/20] feat(dlr): persist unpushed receipts in PostgreSQL --- ...Storage.java => PostgresqlDlrStorage.java} | 232 ++++++++++++++++-- ...est.java => PostgresqlDlrStorageTest.java} | 198 ++++++++++++++- 2 files changed, 402 insertions(+), 28 deletions(-) rename sendium-core/src/main/java/gr/cytech/sendium/core/worker/{PostgresqlMessageStateStorage.java => PostgresqlDlrStorage.java} (62%) rename sendium-core/src/test/java/gr/cytech/sendium/core/worker/{PostgresqlMessageStateStorageTest.java => PostgresqlDlrStorageTest.java} (62%) diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java similarity index 62% rename from sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java rename to sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java index 55a1fad..6cf8f0d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java @@ -1,5 +1,7 @@ package gr.cytech.sendium.core.worker; +import gr.cytech.sendium.core.message.StandardMessage; + import javax.sql.DataSource; import java.sql.Array; import java.sql.Connection; @@ -12,10 +14,12 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class PostgresqlMessageStateStorage implements DlrMessageStorage { +public class PostgresqlDlrStorage implements DlrStorage { private static final int DEFAULT_LINK_MAX_ATTEMPTS = 20; private static final long DEFAULT_LINK_RETRY_INTERVAL_MILLIS = 200; private static final long EXPIRY_CHECK_INTERVAL_MILLIS = TimeUnit.HOURS.toMillis(1); @@ -98,23 +102,72 @@ ON CONFLICT (operator_message_id) DO UPDATE SET WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' """; + private static final String SAVE_UNPUSHED_DLR_SQL = """ + INSERT INTO sendium_dlr.unpushed_dlr + (dlr_key, system_id, account_id, source_address, destination_address, serial, + message_id, dlr_state, error_code, acked, priority, reassembled_parts) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (dlr_key) DO UPDATE SET + system_id = EXCLUDED.system_id, + account_id = EXCLUDED.account_id, + source_address = EXCLUDED.source_address, + destination_address = EXCLUDED.destination_address, + serial = EXCLUDED.serial, + message_id = EXCLUDED.message_id, + dlr_state = EXCLUDED.dlr_state, + error_code = EXCLUDED.error_code, + acked = EXCLUDED.acked, + priority = EXCLUDED.priority, + reassembled_parts = EXCLUDED.reassembled_parts, + created_at = CURRENT_TIMESTAMP + """; + + private static final String GET_UNPUSHED_DLRS_SQL = """ + SELECT dlr_key, system_id, account_id, source_address, destination_address, serial, + message_id, dlr_state, error_code, acked, priority, reassembled_parts + FROM sendium_dlr.unpushed_dlr + WHERE system_id = ? + ORDER BY created_at, dlr_key + """; + + private static final String DELETE_UNPUSHED_DLR_SQL = """ + DELETE FROM sendium_dlr.unpushed_dlr + WHERE dlr_key = ? + """; + + private static final String DELETE_EXPIRED_UNPUSHED_DLRS_SQL = """ + DELETE FROM sendium_dlr.unpushed_dlr + WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' + RETURNING dlr_key + """; + private final DataSource dataSource; private final int linkMaxAttempts; private final long linkRetryIntervalMillis; + private final long expiryCheckIntervalMillis; + private final Object unpushedDlrStateLock = new Object(); + private final Set claimedUnpushedDlrKeys = ConcurrentHashMap.newKeySet(); private volatile long lastExpiryCheck; - public PostgresqlMessageStateStorage(DataSource dataSource) { - this(dataSource, DEFAULT_LINK_MAX_ATTEMPTS, DEFAULT_LINK_RETRY_INTERVAL_MILLIS); + public PostgresqlDlrStorage(DataSource dataSource) { + this(dataSource, DEFAULT_LINK_MAX_ATTEMPTS, DEFAULT_LINK_RETRY_INTERVAL_MILLIS, + EXPIRY_CHECK_INTERVAL_MILLIS); + } + + PostgresqlDlrStorage(DataSource dataSource, int linkMaxAttempts, + long linkRetryIntervalMillis) { + this(dataSource, linkMaxAttempts, linkRetryIntervalMillis, EXPIRY_CHECK_INTERVAL_MILLIS); } - PostgresqlMessageStateStorage(DataSource dataSource, int linkMaxAttempts, - long linkRetryIntervalMillis) { + PostgresqlDlrStorage(DataSource dataSource, int linkMaxAttempts, + long linkRetryIntervalMillis, long expiryCheckIntervalMillis) { this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); - if (linkMaxAttempts < 1 || linkRetryIntervalMillis < 0) { - throw new IllegalArgumentException("Invalid operator-link retry policy"); + if (linkMaxAttempts < 1 || linkRetryIntervalMillis < 0 || expiryCheckIntervalMillis < 0) { + throw new IllegalArgumentException("Invalid storage retry or expiry policy"); } this.linkMaxAttempts = linkMaxAttempts; this.linkRetryIntervalMillis = linkRetryIntervalMillis; + this.expiryCheckIntervalMillis = expiryCheckIntervalMillis; } @Override @@ -211,6 +264,135 @@ public boolean markAsFailed(String gatewayMsgId) { } } + @Override + public boolean saveUnpushedDlr(StandardMessage message) { + checkExpiry(); + if (!isValidUnpushedDlr(message)) { + return false; + } + + UnpushedDlr dlr = UnpushedDlr.fromMessage(message); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(SAVE_UNPUSHED_DLR_SQL)) { + statement.setString(1, getUnpushedDlrKey(message)); + statement.setString(2, dlr.systemId); + statement.setString(3, dlr.accountId); + statement.setString(4, dlr.from); + statement.setString(5, dlr.to); + statement.setString(6, dlr.serial); + statement.setInt(7, dlr.msgId); + statement.setInt(8, dlr.state); + statement.setString(9, dlr.errcode); + statement.setBoolean(10, dlr.acked); + statement.setInt(11, dlr.priority); + setStringArray(connection, statement, 12, dlr.reassembledParts); + return statement.executeUpdate() == 1; + } catch (SQLException e) { + throw failure("save unpushed DLR", e); + } + } + + @Override + public List getUnpushedDlrs(String systemId) { + return loadUnpushedDlrs(systemId, false); + } + + @Override + public List claimUnpushedDlrs(String systemId) { + return loadUnpushedDlrs(systemId, true); + } + + @Override + public boolean removeUnpushedDlr(StandardMessage message) { + if (!isValidUnpushedDlr(message)) { + return false; + } + + String key = getUnpushedDlrKey(message); + synchronized (unpushedDlrStateLock) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(DELETE_UNPUSHED_DLR_SQL)) { + statement.setString(1, key); + boolean removed = statement.executeUpdate() == 1; + claimedUnpushedDlrKeys.remove(key); + return removed; + } catch (SQLException e) { + throw failure("remove unpushed DLR", e); + } + } + } + + @Override + public void releaseUnpushedDlrClaim(StandardMessage message) { + if (!isValidUnpushedDlr(message)) { + return; + } + + synchronized (unpushedDlrStateLock) { + claimedUnpushedDlrKeys.remove(getUnpushedDlrKey(message)); + } + } + + private List loadUnpushedDlrs(String systemId, boolean claimForReplay) { + checkExpiry(); + if (systemId == null || systemId.isBlank()) { + return List.of(); + } + + synchronized (unpushedDlrStateLock) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(GET_UNPUSHED_DLRS_SQL)) { + statement.setString(1, systemId); + try (ResultSet resultSet = statement.executeQuery()) { + List messages = new ArrayList<>(); + while (resultSet.next()) { + String key = resultSet.getString("dlr_key"); + if (!claimForReplay || claimedUnpushedDlrKeys.add(key)) { + messages.add(readUnpushedDlr(resultSet).toMessage()); + } + } + return messages; + } + } catch (SQLException e) { + throw failure("read unpushed DLRs", e); + } + } + } + + private UnpushedDlr readUnpushedDlr(ResultSet resultSet) throws SQLException { + UnpushedDlr dlr = new UnpushedDlr(); + dlr.systemId = resultSet.getString("system_id"); + dlr.accountId = resultSet.getString("account_id"); + dlr.from = resultSet.getString("source_address"); + dlr.to = resultSet.getString("destination_address"); + dlr.serial = resultSet.getString("serial"); + dlr.msgId = resultSet.getInt("message_id"); + dlr.state = resultSet.getInt("dlr_state"); + dlr.errcode = resultSet.getString("error_code"); + dlr.acked = resultSet.getBoolean("acked"); + dlr.priority = resultSet.getInt("priority"); + dlr.reassembledParts = readStringArray(resultSet, "reassembled_parts"); + return dlr; + } + + private boolean isValidUnpushedDlr(StandardMessage message) { + return message != null && message.type == StandardMessage.MSG_DLR && + message.systemId != null && !message.systemId.isBlank(); + } + + private String getUnpushedDlrKey(StandardMessage message) { + return String.join("|", + nullToEmpty(message.systemId), + nullToEmpty(message.serial), + String.valueOf(message.state), + nullToEmpty(message.errcode), + String.valueOf(message.msgId)); + } + + private String nullToEmpty(String value) { + return value == null ? "" : value; + } + private boolean tryLinkOperatorId(UUID gatewayMsgId, String operatorMsgId) { try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); @@ -355,11 +537,11 @@ private void sleepBeforeLinkRetry() { private void checkExpiry() { long now = System.currentTimeMillis(); - if (now - lastExpiryCheck < EXPIRY_CHECK_INTERVAL_MILLIS) { + if (now - lastExpiryCheck < expiryCheckIntervalMillis) { return; } synchronized (this) { - if (now - lastExpiryCheck < EXPIRY_CHECK_INTERVAL_MILLIS) { + if (now - lastExpiryCheck < expiryCheckIntervalMillis) { return; } deleteExpiredState(); @@ -368,19 +550,29 @@ private void checkExpiry() { } private void deleteExpiredState() { - try (Connection connection = dataSource.getConnection()) { - connection.setAutoCommit(false); - try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); - PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL)) { - correlations.executeUpdate(); - messages.executeUpdate(); - connection.commit(); + synchronized (unpushedDlrStateLock) { + List expiredUnpushedDlrKeys = new ArrayList<>(); + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); + PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL); + PreparedStatement unpushedDlrs = connection.prepareStatement(DELETE_EXPIRED_UNPUSHED_DLRS_SQL)) { + correlations.executeUpdate(); + messages.executeUpdate(); + try (ResultSet resultSet = unpushedDlrs.executeQuery()) { + while (resultSet.next()) { + expiredUnpushedDlrKeys.add(resultSet.getString("dlr_key")); + } + } + connection.commit(); + claimedUnpushedDlrKeys.removeAll(expiredUnpushedDlrKeys); + } catch (SQLException e) { + rollback(connection, e); + throw e; + } } catch (SQLException e) { - rollback(connection, e); - throw e; + throw failure("expire DLR state", e); } - } catch (SQLException e) { - throw failure("expire DLR state", e); } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java similarity index 62% rename from sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java index e7d15f0..d7159c4 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlMessageStateStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java @@ -1,5 +1,6 @@ package gr.cytech.sendium.core.worker; +import gr.cytech.sendium.core.message.StandardMessage; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -15,6 +16,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -27,7 +29,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; @EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") -class PostgresqlMessageStateStorageTest { +class PostgresqlDlrStorageTest { private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") .withDatabaseName("sendium") @@ -36,7 +38,7 @@ class PostgresqlMessageStateStorageTest { private static DataSource dataSource; - private PostgresqlMessageStateStorage storage; + private PostgresqlDlrStorage storage; @BeforeAll static void startPostgresql() { @@ -63,9 +65,9 @@ static void stopPostgresql() { void resetStorage() throws SQLException { try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) { - statement.execute("TRUNCATE sendium_dlr.tracked_message CASCADE"); + statement.execute("TRUNCATE sendium_dlr.tracked_message, sendium_dlr.unpushed_dlr CASCADE"); } - storage = new PostgresqlMessageStateStorage(dataSource); + storage = new PostgresqlDlrStorage(dataSource); } @Test @@ -186,8 +188,8 @@ void linkOperatorIdRejectsCorrelationOwnedByAnotherMessage() throws SQLException @Test void linkOperatorIdFailsWhenGatewayStateDoesNotAppear() { - PostgresqlMessageStateStorage noRetryStorage = - new PostgresqlMessageStateStorage(dataSource, 1, 0); + PostgresqlDlrStorage noRetryStorage = + new PostgresqlDlrStorage(dataSource, 1, 0); assertThatThrownBy(() -> noRetryStorage.linkOperatorId(UUID.randomUUID().toString(), "operator")) .isInstanceOf(DlrStorageException.class) @@ -258,7 +260,7 @@ void expiryRemovesOldCorrelationsAndMessages() throws SQLException { storage.linkOperatorId(correlationState.getGatewayMsgId(), "old-correlation"); ageCorrelation("old-correlation"); - PostgresqlMessageStateStorage correlationCleanup = new PostgresqlMessageStateStorage(dataSource); + PostgresqlDlrStorage correlationCleanup = new PostgresqlDlrStorage(dataSource); assertThat(correlationCleanup.getState(correlationState.getGatewayMsgId())).isPresent(); assertThat(countCorrelations(correlationState.getGatewayMsgId())).isZero(); @@ -266,10 +268,145 @@ void expiryRemovesOldCorrelationsAndMessages() throws SQLException { storage.saveInitialState(oldMessage); ageMessage(oldMessage.getGatewayMsgId()); - PostgresqlMessageStateStorage messageCleanup = new PostgresqlMessageStateStorage(dataSource); + PostgresqlDlrStorage messageCleanup = new PostgresqlDlrStorage(dataSource); assertThat(messageCleanup.getState(oldMessage.getGatewayMsgId())).isEmpty(); } + @Test + void saveUnpushedDlrRoundTripsAllPersistedFields() { + StandardMessage dlr = newDlr("account-1", "system-1"); + + assertThat(storage.saveUnpushedDlr(dlr)).isTrue(); + + List stored = storage.getUnpushedDlrs("system-1"); + assertThat(stored).singleElement().satisfies(actual -> { + assertThat(actual.type).isEqualTo(StandardMessage.MSG_DLR); + assertThat(actual.systemId).isEqualTo(dlr.systemId); + assertThat(actual.owner_id).isEqualTo(dlr.owner_id); + assertThat(actual.from).isEqualTo(dlr.from); + assertThat(actual.to).isEqualTo(dlr.to); + assertThat(actual.serial).isEqualTo(dlr.serial); + assertThat(actual.msgId).isEqualTo(dlr.msgId); + assertThat(actual.state).isEqualTo(dlr.state); + assertThat(actual.errcode).isEqualTo(dlr.errcode); + assertThat(actual.acked).isEqualTo(dlr.acked); + assertThat(actual.priority).isEqualTo(dlr.priority); + assertThat(actual.reassembledParts).containsExactlyElementsOf(dlr.reassembledParts); + }); + } + + @Test + void saveUnpushedDlrRejectsInvalidMessages() throws SQLException { + StandardMessage wrongType = newDlr("account-1", "system-1"); + wrongType.type = StandardMessage.MSG_TEXT; + StandardMessage blankSystem = newDlr("account-1", " "); + + assertThat(storage.saveUnpushedDlr(null)).isFalse(); + assertThat(storage.saveUnpushedDlr(wrongType)).isFalse(); + assertThat(storage.saveUnpushedDlr(blankSystem)).isFalse(); + assertThat(countUnpushedDlrs()).isZero(); + } + + @Test + void saveUnpushedDlrOverwritesSameReplayKey() throws SQLException { + StandardMessage initial = newDlr("account-1", "system-1"); + storage.saveUnpushedDlr(initial); + + StandardMessage replacement = newDlr("replacement-account", initial.systemId); + replacement.serial = initial.serial; + replacement.msgId = initial.msgId; + replacement.state = initial.state; + replacement.errcode = initial.errcode; + replacement.priority = 9; + replacement.reassembledParts = new ArrayList<>(List.of("replacement-part")); + storage.saveUnpushedDlr(replacement); + + assertThat(countUnpushedDlrs()).isOne(); + assertThat(storage.getUnpushedDlrs(initial.systemId)) + .singleElement() + .satisfies(actual -> { + assertThat(actual.owner_id).isEqualTo("replacement-account"); + assertThat(actual.priority).isEqualTo(9); + assertThat(actual.reassembledParts).containsExactly("replacement-part"); + }); + } + + @Test + void unpushedDlrsAreFilteredAndSurviveAdapterRecreation() { + StandardMessage first = newDlr("account-1", "system-1"); + StandardMessage second = newDlr("account-2", "system-2"); + storage.saveUnpushedDlr(first); + storage.saveUnpushedDlr(second); + + PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); + + assertThat(recreated.getUnpushedDlrs("system-1")) + .extracting(message -> message.serial) + .containsExactly(first.serial); + assertThat(recreated.getUnpushedDlrs("system-2")) + .extracting(message -> message.serial) + .containsExactly(second.serial); + assertThat(recreated.getUnpushedDlrs("missing-system")).isEmpty(); + } + + @Test + void claimHidesReceiptUntilReleasedOrRemoved() { + StandardMessage dlr = newDlr("account-1", "system-1"); + storage.saveUnpushedDlr(dlr); + + List firstClaim = storage.claimUnpushedDlrs(dlr.systemId); + + assertThat(firstClaim).hasSize(1); + assertThat(storage.claimUnpushedDlrs(dlr.systemId)).isEmpty(); + assertThat(storage.getUnpushedDlrs(dlr.systemId)).hasSize(1); + + storage.releaseUnpushedDlrClaim(firstClaim.getFirst()); + List releasedClaim = storage.claimUnpushedDlrs(dlr.systemId); + assertThat(releasedClaim).hasSize(1); + assertThat(storage.removeUnpushedDlr(releasedClaim.getFirst())).isTrue(); + assertThat(storage.removeUnpushedDlr(releasedClaim.getFirst())).isFalse(); + assertThat(storage.getUnpushedDlrs(dlr.systemId)).isEmpty(); + } + + @Test + void concurrentClaimsReturnReceiptOnlyOnce() throws Exception { + StandardMessage dlr = newDlr("account-1", "system-1"); + storage.saveUnpushedDlr(dlr); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + + try { + Future> first = executor.submit( + () -> claimWhenReleased(dlr.systemId, ready, start)); + Future> second = executor.submit( + () -> claimWhenReleased(dlr.systemId, ready, start)); + ready.await(); + start.countDown(); + + assertThat(List.of(first.get(), second.get()).stream().filter(claim -> !claim.isEmpty()).count()) + .isOne(); + } finally { + executor.shutdownNow(); + } + } + + @Test + void expiryRemovesOldUnpushedDlrsAndTheirClaims() throws SQLException { + StandardMessage dlr = newDlr("account-1", "system-1"); + storage.saveUnpushedDlr(dlr); + storage = new PostgresqlDlrStorage(dataSource, 20, 200, 0); + assertThat(storage.claimUnpushedDlrs(dlr.systemId)).hasSize(1); + ageUnpushedDlr(dlr.serial); + + assertThat(storage.getUnpushedDlrs(dlr.systemId)).isEmpty(); + assertThat(countUnpushedDlrs()).isZero(); + + assertThat(storage.saveUnpushedDlr(dlr)).isTrue(); + assertThat(storage.claimUnpushedDlrs(dlr.systemId)).hasSize(1); + assertThat(storage.claimUnpushedDlrs(dlr.systemId)).isEmpty(); + } + private Optional resolveWhenReleased(String operatorMsgId, CountDownLatch ready, CountDownLatch start) throws InterruptedException { ready.countDown(); @@ -277,11 +414,35 @@ private Optional resolveWhenReleased(String operatorMsgId, CountDo return storage.resolveAndRemoveDlr(operatorMsgId, MessageState.MessageStatus.DELIVERED); } + private List claimWhenReleased(String systemId, CountDownLatch ready, + CountDownLatch start) throws InterruptedException { + ready.countDown(); + start.await(); + return storage.claimUnpushedDlrs(systemId); + } + private MessageState newState() { return new MessageState(UUID.randomUUID().toString(), "account", "system", "source", "destination", "https://example.test/dlr"); } + private StandardMessage newDlr(String accountId, String systemId) { + StandardMessage dlr = new StandardMessage(); + dlr.type = StandardMessage.MSG_DLR; + dlr.systemId = systemId; + dlr.owner_id = accountId; + dlr.from = "source"; + dlr.to = "destination"; + dlr.serial = UUID.randomUUID().toString(); + dlr.msgId = 42; + dlr.state = 1; + dlr.errcode = "000"; + dlr.acked = true; + dlr.priority = 3; + dlr.reassembledParts = new ArrayList<>(List.of("part-1", "part-2")); + return dlr; + } + private int countCorrelations(String gatewayMsgId) throws SQLException { try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(""" @@ -297,6 +458,15 @@ SELECT COUNT(*) } } + private int countUnpushedDlrs() throws SQLException { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM sendium_dlr.unpushed_dlr")) { + resultSet.next(); + return resultSet.getInt(1); + } + } + private void ageCorrelation(String operatorMsgId) throws SQLException { try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(""" @@ -320,4 +490,16 @@ private void ageMessage(String gatewayMsgId) throws SQLException { statement.executeUpdate(); } } + + private void ageUnpushedDlr(String serial) throws SQLException { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + UPDATE sendium_dlr.unpushed_dlr + SET created_at = CURRENT_TIMESTAMP - INTERVAL '8 days' + WHERE serial = ? + """)) { + statement.setString(1, serial); + statement.executeUpdate(); + } + } } From c71c45bd516359c03569215a2e8c6e99f17e2d97 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 15:37:30 +0300 Subject: [PATCH 05/20] feat(dlr): wire configurable storage backend --- .../src/main/resources/application.properties | 21 ++ sendium-core/pom.xml | 31 +-- .../core/worker/ConfiguredDlrStorage.java | 187 ++++++++++++++++++ .../core/worker/DlrStorageReadinessCheck.java | 39 ++++ .../core/worker/MvStoreDlrStorage.java | 14 +- .../src/main/resources/application.properties | 23 ++- .../core/worker/ConfiguredDlrStorageTest.java | 128 ++++++++++++ .../worker/DlrStorageReadinessCheckTest.java | 62 ++++++ .../core/worker/DlrStorageRuntimeTest.java | 56 ++++++ .../PostgresqlDlrQuarkusTestResource.java | 38 ++++ .../core/worker/PostgresqlDlrRuntimeTest.java | 90 +++++++++ 11 files changed, 670 insertions(+), 19 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index d3388bf..1b78ced 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -2,6 +2,27 @@ smsg.routing.file.path=conf/routingTable.conf smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml +# DLR persistence. PostgreSQL also requires the named datasource URL and credentials. +sendium.dlr.storage=${SENDIUM_DLR_STORAGE:mvstore} +sendium.dlr.db.path=${SENDIUM_DLR_MVSTORE_PATH:data/dlr-mvstore.db} +quarkus.datasource.devservices.enabled=false +quarkus.datasource.dlr.db-kind=postgresql +quarkus.datasource.dlr.active=${SENDIUM_DLR_POSTGRESQL_ACTIVE:false} +quarkus.datasource.dlr.devservices.enabled=false +quarkus.datasource.dlr.jdbc.min-size=${SENDIUM_DLR_POSTGRESQL_POOL_MIN_SIZE:0} +quarkus.datasource.dlr.jdbc.max-size=${SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE:10} +quarkus.datasource.dlr.jdbc.acquisition-timeout=${SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT:5S} +quarkus.datasource.metrics.enabled=true +quarkus.flyway.dlr.locations=db/sendium-dlr/postgresql +quarkus.flyway.dlr.active=${quarkus.datasource.dlr.active} +quarkus.flyway.dlr.migrate-at-start=${quarkus.datasource.dlr.active} +quarkus.flyway.dlr.validate-on-migrate=true +quarkus.flyway.dlr.clean-disabled=true +quarkus.flyway.dlr.schemas=sendium_dlr +quarkus.flyway.dlr.default-schema=sendium_dlr +quarkus.flyway.dlr.table=flyway_schema_history +quarkus.flyway.dlr.create-schemas=true + #logging quarkus.log.level=${QUARKUS_LOG_LEVEL:INFO} quarkus.log.category."gr.cytech".level=${LOG_LEVEL:INFO} diff --git a/sendium-core/pom.xml b/sendium-core/pom.xml index d6bb466..f8fe1fe 100644 --- a/sendium-core/pom.xml +++ b/sendium-core/pom.xml @@ -32,6 +32,22 @@ io.quarkus quarkus-smallrye-openapi + + io.quarkus + quarkus-smallrye-health + + + io.quarkus + quarkus-jdbc-postgresql + + + io.quarkus + quarkus-flyway + + + org.flywaydb + flyway-database-postgresql + io.quarkus quarkus-micrometer-registry-prometheus @@ -80,21 +96,6 @@ assertj-core test - - org.flywaydb - flyway-core - test - - - org.flywaydb - flyway-database-postgresql - test - - - org.postgresql - postgresql - test - org.testcontainers testcontainers-postgresql diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java new file mode 100644 index 0000000..f494bd2 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java @@ -0,0 +1,187 @@ +package gr.cytech.sendium.core.worker; + +import gr.cytech.sendium.core.message.StandardMessage; +import io.agroal.api.AgroalDataSource; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.quarkus.agroal.DataSource; +import io.quarkus.arc.InjectableInstance; +import io.quarkus.runtime.Startup; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Supplier; + +@Startup +@ApplicationScoped +public class ConfiguredDlrStorage implements DlrStorage { + private static final String METRIC_NAME = "sendium.dlr.storage.operation"; + private static final String POSTGRESQL_PROBE_SQL = """ + SELECT 1 + FROM sendium_dlr.tracked_message + WHERE FALSE + """; + + @Inject + @ConfigProperty(name = "sendium.dlr.storage", defaultValue = "mvstore") + String configuredBackend; + + @Inject + @ConfigProperty(name = "quarkus.flyway.dlr.active", defaultValue = "false") + boolean flywayActive; + + @Inject + @ConfigProperty(name = "quarkus.flyway.dlr.migrate-at-start", defaultValue = "false") + boolean flywayMigrateAtStart; + + @Inject + Instance mvStoreStorage; + + @Inject + @DataSource("dlr") + InjectableInstance postgresqlDataSource; + + @Inject + MeterRegistry meterRegistry; + + private DlrStorage delegate; + private AgroalDataSource selectedPostgresqlDataSource; + private String backend; + + @PostConstruct + void initialize() { + backend = configuredBackend.strip().toLowerCase(Locale.ROOT); + boolean postgresqlActive = postgresqlDataSource.getHandle().getBean().isActive(); + delegate = switch (backend) { + case "mvstore" -> { + if (postgresqlActive || flywayActive || flywayMigrateAtStart) { + throw new IllegalStateException( + "The DLR PostgreSQL datasource and Flyway must be inactive when MVStore is selected"); + } + yield mvStoreStorage.get(); + } + case "postgresql" -> { + if (!postgresqlActive || !flywayActive || !flywayMigrateAtStart) { + throw new IllegalStateException( + "PostgreSQL DLR storage requires the active 'dlr' datasource and Flyway migration"); + } + selectedPostgresqlDataSource = postgresqlDataSource.get(); + yield new PostgresqlDlrStorage(selectedPostgresqlDataSource); + } + default -> throw new IllegalStateException("Unsupported DLR storage backend: " + backend); + }; + + Gauge.builder("sendium.dlr.storage.selected", this, ignored -> 1.0) + .description("Selected Sendium DLR storage backend") + .tag("backend", backend) + .strongReference(true) + .register(meterRegistry); + } + + String backend() { + return backend; + } + + String mode() { + if (delegate instanceof MvStoreDlrStorage mvStore) { + return mvStore.isPersistent() ? "persistent" : "memory"; + } + return "persistent"; + } + + void verifyPostgresqlSchema() throws SQLException { + if (selectedPostgresqlDataSource == null) { + return; + } + try (Connection connection = selectedPostgresqlDataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(POSTGRESQL_PROBE_SQL)) { + statement.executeQuery(); + } + } + + @Override + public void saveInitialState(MessageState state) { + timed("save_initial", () -> delegate.saveInitialState(state)); + } + + @Override + public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { + timed("link_operator", () -> delegate.linkOperatorId(gatewayMsgId, operatorMsgId)); + } + + @Override + public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { + return timed("resolve", () -> delegate.resolveAndRemoveDlr(operatorMsgId, status)); + } + + @Override + public Optional getState(String gatewayMsgId) { + return timed("get_state", () -> delegate.getState(gatewayMsgId)); + } + + @Override + public boolean markAsFailed(String gatewayMsgId) { + return timed("mark_failed", () -> delegate.markAsFailed(gatewayMsgId)); + } + + @Override + public boolean saveUnpushedDlr(StandardMessage message) { + return timed("save_unpushed", () -> delegate.saveUnpushedDlr(message)); + } + + @Override + public List getUnpushedDlrs(String systemId) { + return timed("get_unpushed", () -> delegate.getUnpushedDlrs(systemId)); + } + + @Override + public List claimUnpushedDlrs(String systemId) { + return timed("claim_unpushed", () -> delegate.claimUnpushedDlrs(systemId)); + } + + @Override + public boolean removeUnpushedDlr(StandardMessage message) { + return timed("remove_unpushed", () -> delegate.removeUnpushedDlr(message)); + } + + @Override + public void releaseUnpushedDlrClaim(StandardMessage message) { + timed("release_claim", () -> delegate.releaseUnpushedDlrClaim(message)); + } + + private T timed(String operation, Supplier action) { + Timer.Sample sample = Timer.start(meterRegistry); + try { + T result = action.get(); + sample.stop(timer(operation, "success")); + return result; + } catch (RuntimeException | Error e) { + sample.stop(timer(operation, "error")); + throw e; + } + } + + private void timed(String operation, Runnable action) { + timed(operation, () -> { + action.run(); + return null; + }); + } + + private Timer timer(String operation, String outcome) { + return Timer.builder(METRIC_NAME) + .description("Sendium DLR storage operation latency") + .tags("backend", backend, "operation", operation, "outcome", outcome) + .register(meterRegistry); + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java new file mode 100644 index 0000000..0a46533 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java @@ -0,0 +1,39 @@ +package gr.cytech.sendium.core.worker; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.health.HealthCheck; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.HealthCheckResponseBuilder; +import org.eclipse.microprofile.health.Readiness; + +import java.sql.SQLException; + +@Readiness +@ApplicationScoped +public class DlrStorageReadinessCheck implements HealthCheck { + private static final String CHECK_NAME = "sendium-dlr-storage"; + + @Inject + ConfiguredDlrStorage storage; + + @Override + public HealthCheckResponse call() { + HealthCheckResponseBuilder response = HealthCheckResponse.named(CHECK_NAME) + .withData("backend", storage.backend()); + if (!"postgresql".equals(storage.backend())) { + return response.up() + .withData("mode", storage.mode()) + .build(); + } + + try { + storage.verifyPostgresqlSchema(); + return response.up().build(); + } catch (SQLException e) { + return response.down() + .withData("reason", "unavailable") + .build(); + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java index 767de7c..f247f9b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java @@ -5,10 +5,12 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import gr.cytech.sendium.core.message.StandardMessage; -import io.quarkus.arc.DefaultBean; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Typed; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.h2.mvstore.MVStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +41,7 @@ * before returning them so concurrent reconnect callbacks for the same systemId cannot enqueue the same DLR twice. */ @ApplicationScoped -@DefaultBean +@Typed(MvStoreDlrStorage.class) public class MvStoreDlrStorage implements DlrStorage { private static final Logger logger = LoggerFactory.getLogger(MvStoreDlrStorage.class); private static final long SEVEN_DAYS_MILLIS = TimeUnit.DAYS.toMillis(7); @@ -56,6 +58,10 @@ public class MvStoreDlrStorage implements DlrStorage { private final Object unpushedDlrStateLock = new Object(); private final Set claimedUnpushedDlrKeys = ConcurrentHashMap.newKeySet(); + @Inject + @ConfigProperty(name = DB_PATH_PROPERTY, defaultValue = DEFAULT_DB_PATH) + private String configuredDbPath; + private MVStore store; private Map primaryStore; @@ -72,7 +78,9 @@ public class MvStoreDlrStorage implements DlrStorage { @PostConstruct void init() { - String dbPath = System.getProperty(DB_PATH_PROPERTY, DEFAULT_DB_PATH); + String dbPath = configuredDbPath != null ? + configuredDbPath + : System.getProperty(DB_PATH_PROPERTY, DEFAULT_DB_PATH); File dbFile = new File(dbPath); File dbDir = dbFile.getParentFile(); diff --git a/sendium-core/src/main/resources/application.properties b/sendium-core/src/main/resources/application.properties index 014b868..aa00fff 100644 --- a/sendium-core/src/main/resources/application.properties +++ b/sendium-core/src/main/resources/application.properties @@ -1,3 +1,24 @@ %test.smsg.routing.file.path=src/test/resources/routingTable.conf %test.smsg.properties.file.path=src/test/resources/smsg.properties -%test.smsg.credentials.file.path=src/test/resources/credentials.yml \ No newline at end of file +%test.smsg.credentials.file.path=src/test/resources/credentials.yml +%test.sendium.dlr.db.path=${java.io.tmpdir}/sendium-dlr-${quarkus.uuid}.db + +# The PostgreSQL DLR datasource stays inactive unless explicitly selected and activated. +sendium.dlr.storage=mvstore +quarkus.datasource.devservices.enabled=false +quarkus.datasource.dlr.db-kind=postgresql +quarkus.datasource.dlr.active=false +quarkus.datasource.dlr.devservices.enabled=false +quarkus.datasource.dlr.jdbc.min-size=0 +quarkus.datasource.dlr.jdbc.max-size=10 +quarkus.datasource.dlr.jdbc.acquisition-timeout=5S +quarkus.datasource.metrics.enabled=true +quarkus.flyway.dlr.locations=db/sendium-dlr/postgresql +quarkus.flyway.dlr.active=${quarkus.datasource.dlr.active} +quarkus.flyway.dlr.migrate-at-start=${quarkus.datasource.dlr.active} +quarkus.flyway.dlr.validate-on-migrate=true +quarkus.flyway.dlr.clean-disabled=true +quarkus.flyway.dlr.schemas=sendium_dlr +quarkus.flyway.dlr.default-schema=sendium_dlr +quarkus.flyway.dlr.table=flyway_schema_history +quarkus.flyway.dlr.create-schemas=true diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java new file mode 100644 index 0000000..eeed462 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java @@ -0,0 +1,128 @@ +package gr.cytech.sendium.core.worker; + +import io.agroal.api.AgroalDataSource; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import io.quarkus.arc.InjectableInstance; +import jakarta.enterprise.inject.Instance; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ConfiguredDlrStorageTest { + private Instance mvStoreInstance; + private MvStoreDlrStorage mvStore; + private InjectableInstance postgresqlDataSource; + private SimpleMeterRegistry meterRegistry; + private ConfiguredDlrStorage storage; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + mvStoreInstance = mock(Instance.class); + mvStore = mock(MvStoreDlrStorage.class); + postgresqlDataSource = mock(InjectableInstance.class, RETURNS_DEEP_STUBS); + meterRegistry = new SimpleMeterRegistry(); + when(mvStoreInstance.get()).thenReturn(mvStore); + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(false); + + storage = new ConfiguredDlrStorage(); + storage.configuredBackend = "mvstore"; + storage.mvStoreStorage = mvStoreInstance; + storage.postgresqlDataSource = postgresqlDataSource; + storage.meterRegistry = meterRegistry; + } + + @Test + void selectsMvStoreAndRecordsLowCardinalityMetrics() { + when(mvStore.getState("sensitive-gateway-id")).thenReturn(Optional.empty()); + + storage.initialize(); + storage.getState("sensitive-gateway-id"); + + verify(mvStore).getState("sensitive-gateway-id"); + assertThat(storage.backend()).isEqualTo("mvstore"); + assertThat(meterRegistry.find("sendium.dlr.storage.selected") + .tag("backend", "mvstore").gauge().value()).isEqualTo(1.0); + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "mvstore", "operation", "get_state", "outcome", "success") + .timer().count()).isOne(); + assertThat(meterRegistry.getMeters()) + .flatExtracting(meter -> meter.getId().getTags()) + .noneMatch(tag -> tag.getValue().contains("sensitive")); + } + + @Test + void recordsThrownStorageFailureAsError() { + when(mvStore.markAsFailed("gateway-id")).thenThrow(new DlrStorageException("failure")); + storage.initialize(); + + assertThatThrownBy(() -> storage.markAsFailed("gateway-id")) + .isInstanceOf(DlrStorageException.class); + + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "mvstore", "operation", "mark_failed", "outcome", "error") + .timer().count()).isOne(); + } + + @Test + void rejectsUnknownBackend() { + storage.configuredBackend = "unknown"; + + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Unsupported DLR storage backend: unknown"); + } + + @Test + void rejectsActivePostgresqlDatasourceForMvStore() { + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); + + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must be inactive"); + } + + @Test + void rejectsInactivePostgresqlDatasourceWhenSelected() { + storage.configuredBackend = "postgresql"; + + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); + } + + @Test + void rejectsPostgresqlDatasourceWithoutFlyway() { + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); + storage.configuredBackend = "postgresql"; + + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); + } + + @Test + void selectsActivePostgresqlDatasourceWithoutMvStoreFallback() { + AgroalDataSource dataSource = mock(AgroalDataSource.class); + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); + when(postgresqlDataSource.get()).thenReturn(dataSource); + storage.configuredBackend = " POSTGRESQL "; + storage.flywayActive = true; + storage.flywayMigrateAtStart = true; + + storage.initialize(); + + assertThat(storage.backend()).isEqualTo("postgresql"); + assertThat(storage.mode()).isEqualTo("persistent"); + verify(postgresqlDataSource).get(); + verify(mvStoreInstance, org.mockito.Mockito.never()).get(); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java new file mode 100644 index 0000000..4065b1b --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java @@ -0,0 +1,62 @@ +package gr.cytech.sendium.core.worker; + +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.junit.jupiter.api.Test; + +import java.sql.SQLException; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DlrStorageReadinessCheckTest { + @Test + void reportsMvStoreMode() { + ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); + when(storage.backend()).thenReturn("mvstore"); + when(storage.mode()).thenReturn("memory"); + DlrStorageReadinessCheck check = new DlrStorageReadinessCheck(); + check.storage = storage; + + HealthCheckResponse response = check.call(); + Map data = response.getData().orElseThrow(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); + assertThat(data).containsEntry("backend", "mvstore").containsEntry("mode", "memory"); + } + + @Test + void reportsPostgresqlSchemaAsReady() throws SQLException { + ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); + when(storage.backend()).thenReturn("postgresql"); + DlrStorageReadinessCheck check = new DlrStorageReadinessCheck(); + check.storage = storage; + + HealthCheckResponse response = check.call(); + Map data = response.getData().orElseThrow(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); + assertThat(data).containsOnlyKeys("backend"); + } + + @Test + void reportsSanitizedPostgresqlFailure() throws SQLException { + ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); + when(storage.backend()).thenReturn("postgresql"); + doThrow(new SQLException("jdbc:postgresql://secret-host/database")) + .when(storage).verifyPostgresqlSchema(); + DlrStorageReadinessCheck check = new DlrStorageReadinessCheck(); + check.storage = storage; + + HealthCheckResponse response = check.call(); + Map data = response.getData().orElseThrow(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(data) + .containsEntry("backend", "postgresql") + .containsEntry("reason", "unavailable"); + assertThat(data.toString()).doesNotContain("secret-host"); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java new file mode 100644 index 0000000..cd6fa70 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java @@ -0,0 +1,56 @@ +package gr.cytech.sendium.core.worker; + +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +@QuarkusTest +class DlrStorageRuntimeTest { + @Inject + Instance storageInstance; + + @Inject + ConfiguredDlrStorage configuredStorage; + + @Inject + MeterRegistry meterRegistry; + + @Test + void selectsExactlyOneMvStoreBackendByDefault() { + assertThat(storageInstance.isResolvable()).isTrue(); + assertThat(storageInstance.stream()).hasSize(1); + assertThat(storageInstance.get()).isSameAs(configuredStorage); + assertThat(configuredStorage.backend()).isEqualTo("mvstore"); + } + + @Test + void exposesReadinessAndStorageMetrics() { + configuredStorage.getState(UUID.randomUUID().toString()); + + given() + .when().get("/q/health/ready") + .then() + .statusCode(200) + .body("status", equalTo("UP")) + .body("checks.find { it.name == 'sendium-dlr-storage' }.data.backend", equalTo("mvstore")); + + given() + .when().get("/q/metrics") + .then() + .statusCode(200) + .body(containsString("sendium_dlr_storage_selected")) + .body(containsString("sendium_dlr_storage_operation_seconds_count")); + + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tag("operation", "get_state").timer().count()).isGreaterThanOrEqualTo(1); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java new file mode 100644 index 0000000..bf8f60e --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java @@ -0,0 +1,38 @@ +package gr.cytech.sendium.core.worker; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import org.testcontainers.postgresql.PostgreSQLContainer; + +import java.util.Map; + +public class PostgresqlDlrQuarkusTestResource implements QuarkusTestResourceLifecycleManager { + private PostgreSQLContainer postgresql; + + @Override + public Map start() { + if (!Boolean.getBoolean("sendium.postgresql.tests")) { + return Map.of(); + } + + postgresql = new PostgreSQLContainer("postgres:17-alpine") + .withDatabaseName("sendium") + .withUsername("sendium") + .withPassword("sendium-test"); + postgresql.start(); + return Map.of( + "sendium.dlr.storage", "postgresql", + "quarkus.datasource.dlr.active", "true", + "quarkus.flyway.dlr.active", "true", + "quarkus.flyway.dlr.migrate-at-start", "true", + "quarkus.datasource.dlr.jdbc.url", postgresql.getJdbcUrl(), + "quarkus.datasource.dlr.username", postgresql.getUsername(), + "quarkus.datasource.dlr.password", postgresql.getPassword()); + } + + @Override + public void stop() { + if (postgresql != null) { + postgresql.stop(); + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java new file mode 100644 index 0000000..4f58699 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java @@ -0,0 +1,90 @@ +package gr.cytech.sendium.core.worker; + +import io.agroal.api.AgroalDataSource; +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.agroal.DataSource; +import io.quarkus.arc.InjectableInstance; +import io.quarkus.flyway.FlywayDataSource; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.equalTo; + +@QuarkusTest +@QuarkusTestResource(value = PostgresqlDlrQuarkusTestResource.class, restrictToAnnotatedClass = true) +@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") +class PostgresqlDlrRuntimeTest { + @Inject + DlrStorage storage; + + @Inject + ConfiguredDlrStorage configuredStorage; + + @Inject + @DataSource("dlr") + InjectableInstance dataSource; + + @Inject + @FlywayDataSource("dlr") + InjectableInstance flyway; + + @Inject + MeterRegistry meterRegistry; + + @Test + void wiresPoolMigrationStorageHealthAndMetrics() throws SQLException { + assertThat(storage).isSameAs(configuredStorage); + assertThat(configuredStorage.backend()).isEqualTo("postgresql"); + assertThat(dataSource.getHandle().getBean().isActive()).isTrue(); + assertThat(flyway.getHandle().getBean().isActive()).isTrue(); + assertThat(flyway.get().info().current().getVersion().getVersion()).isEqualTo("1"); + assertThat(flywayHistoryCount()).isOne(); + + MessageState state = new MessageState(UUID.randomUUID().toString(), "account", "system", + "source", "destination", null); + storage.saveInitialState(state); + assertThat(storage.getState(state.getGatewayMsgId())).isPresent(); + + given() + .when().get("/q/health/ready") + .then() + .statusCode(200) + .body("status", equalTo("UP")) + .body("checks.find { it.name == 'sendium-dlr-storage' }.data.backend", + equalTo("postgresql")); + + assertThat(meterRegistry.find("sendium.dlr.storage.selected") + .tag("backend", "postgresql").gauge().value()).isEqualTo(1.0); + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "postgresql", "operation", "save_initial", "outcome", "success") + .timer().count()).isOne(); + assertThat(meterRegistry.getMeters()) + .extracting(meter -> meter.getId().getName()) + .anyMatch(name -> name.startsWith("agroal")); + } + + private int flywayHistoryCount() throws SQLException { + try (Connection connection = dataSource.get().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(""" + SELECT COUNT(*) + FROM sendium_dlr.flyway_schema_history + WHERE success AND version = '1' + """)) { + resultSet.next(); + return resultSet.getInt(1); + } + } +} From 6b1b549fdcd5224aa5bce3d952b4b019027ac7cf Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 17 Aug 2026 15:54:22 +0300 Subject: [PATCH 06/20] fix(http): persist DLR state before queueing --- docs/06-http-api.md | 4 +- .../sendium/core/http/KannelResource.java | 16 ++- .../sendium/core/http/KannelResourceTest.java | 104 ++++++++++++++++++ 3 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java diff --git a/docs/06-http-api.md b/docs/06-http-api.md index ae7d3db..bed2dd2 100644 --- a/docs/06-http-api.md +++ b/docs/06-http-api.md @@ -61,7 +61,7 @@ The API returns standard HTTP status codes along with a plain-text response body | **`400 Bad Request`** | **Error** | Missing a required parameter (`to`, `from`, or `text`). The response body details which parameter is missing. | | **`401 Unauthorized`** | **Error** | Invalid or missing credentials. | | **`500 Server Error`** | **Error** | An internal error occurred while parsing or processing the message payload. | -| **`503 Unavailable`** | **Error** | Temporal failure (e.g., the internal queue was interrupted). The client should retry later. | +| **`503 Unavailable`** | **Error** | Required delivery-receipt state could not be persisted or internal queue admission was interrupted. The message was not queued; the client should retry later. | --- @@ -90,7 +90,7 @@ For a manual installation, replace the environment variables with the HTTP `syst 123e4567-e89b-12d3-a456-426614174000 ``` -`202 Accepted` means Sendium validated the message and inserted it into the router queue. It does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. +`202 Accepted` means Sendium validated the message, persisted its required delivery-receipt state, and inserted it into the router queue. It does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. ## Related Documentation diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index 6f4a02f..8e1306f 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -6,6 +6,7 @@ import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; import gr.cytech.sendium.core.worker.DlrService; +import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.util.MessageTrace; import jakarta.annotation.security.PermitAll; @@ -78,7 +79,7 @@ public class KannelResource { ), @APIResponse( responseCode = "503", - description = "Service Unavailable. Temporal failure, usually due to a queue enqueue interruption.", + description = "Service Unavailable. Required DLR state could not be persisted or queue admission was interrupted.", content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(examples = "Temporal failure, try again later.")) ) }) @@ -210,20 +211,25 @@ public Response receiveSms( } msg.acked = true; msg.serial = UUID.randomUUID().toString(); + MessageState state = new MessageState(msg.serial, usr, msg.from, msg.to, dlrUrl); + dlrService.saveInitialState(state); + queueProvider.getRouterQueue().enqueue(msg); if (MessageTrace.shouldLog(configurationHandler, MessageTrace.EVENT_ACCEPTED)) { logger.info("message.accepted ingress=http {}", MessageTrace.identifiers(msg)); } - queueProvider.getRouterQueue().enqueue(msg); - MessageState state = new MessageState(msg.serial, usr, msg.from, msg.to, dlrUrl); - dlrService.saveInitialState(state); return Response.status(Response.Status.ACCEPTED) .entity(msg.serial) .build(); + } catch (DlrStorageException e) { + logger.error("HTTP submission rejected: DLR storage unavailable"); + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity("Temporal failure, try again later.") + .build(); } catch (InterruptedException e) { logger.error("Failed to enqueue message", e); - return Response.status(503) + return Response.status(Response.Status.SERVICE_UNAVAILABLE) .entity("Temporal failure, try again later.") .build(); } catch (Exception e) { diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java new file mode 100644 index 0000000..b5d6762 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java @@ -0,0 +1,104 @@ +package gr.cytech.sendium.core.http; + +import gr.cytech.sendium.auth.CredentialFileWatcher; +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.worker.DlrService; +import gr.cytech.sendium.core.worker.DlrStorageException; +import gr.cytech.sendium.core.worker.MessageState; +import jakarta.ws.rs.core.Response; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; + +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class KannelResourceTest { + private static final String USERNAME = "http-user"; + private static final String PASSWORD = "secret"; + + private Queue routerQueue; + private DlrService dlrService; + private KannelResource resource; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + InMemoryQueueProvider queueProvider = mock(InMemoryQueueProvider.class); + routerQueue = mock(Queue.class); + when(queueProvider.getRouterQueue()).thenReturn(routerQueue); + + CredentialFileWatcher credentials = mock(CredentialFileWatcher.class); + CredentialFileWatcher.Credential credential = new CredentialFileWatcher.Credential( + CredentialFileWatcher.CredentialType.HTTP, null, null, USERNAME, PASSWORD, null, Set.of()); + when(credentials.getValidCredentials()).thenReturn(Map.of(USERNAME, credential)); + + resource = new KannelResource(); + resource.queueProvider = queueProvider; + resource.credentialFileWatcher = credentials; + resource.configurationHandler = mock(SendiumConfigurationHandler.class); + dlrService = mock(DlrService.class); + resource.dlrService = dlrService; + } + + @Test + void persistsStateBeforeQueueAdmissionForEverySubmission() throws InterruptedException { + ArgumentCaptor stateCaptor = ArgumentCaptor.forClass(MessageState.class); + ArgumentCaptor messageCaptor = ArgumentCaptor.forClass(StandardMessage.class); + InOrder order = inOrder(dlrService, routerQueue); + + Response response = submit(null); + + order.verify(dlrService).saveInitialState(stateCaptor.capture()); + order.verify(routerQueue).enqueue(messageCaptor.capture()); + MessageState state = stateCaptor.getValue(); + StandardMessage message = messageCaptor.getValue(); + assertThat(response.getStatus()).isEqualTo(Response.Status.ACCEPTED.getStatusCode()); + assertThat(response.getEntity()).isEqualTo(message.serial).isEqualTo(state.getGatewayMsgId()); + assertThat(message.acked).isTrue(); + assertThat(state.getForwardDlrUrl()).isNull(); + } + + @Test + void rejectsBeforeQueueAdmissionWhenPersistenceFails() throws InterruptedException { + doThrow(new DlrStorageException("database details")) + .when(dlrService).saveInitialState(any(MessageState.class)); + + Response response = submit("https://callback.test/dlr"); + + assertThat(response.getStatus()).isEqualTo(Response.Status.SERVICE_UNAVAILABLE.getStatusCode()); + assertThat(response.getEntity()).isEqualTo("Temporal failure, try again later."); + verify(routerQueue, never()).enqueue(any(StandardMessage.class)); + } + + @Test + void returnsRetryableFailureWhenQueueAdmissionIsInterruptedAfterPersistence() throws InterruptedException { + doThrow(new InterruptedException("interrupted")) + .when(routerQueue).enqueue(any(StandardMessage.class)); + + Response response = submit(null); + + assertThat(response.getStatus()).isEqualTo(Response.Status.SERVICE_UNAVAILABLE.getStatusCode()); + assertThat(response.getEntity()).isEqualTo("Temporal failure, try again later."); + verify(dlrService).saveInitialState(any(MessageState.class)); + } + + private Response submit(String dlrUrl) { + return resource.receiveSms( + USERNAME, PASSWORD, "Sender", "306910000000", "Hello", null, null, null, + null, null, null, null, dlrUrl, null, null, null, null, null, null, null, null); + } +} From 4765b69c4285d62aa7d10820c68a6cb0b8815771 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 10:28:40 +0300 Subject: [PATCH 07/20] fix(smpp): persist submissions before acknowledgement --- .../sendium/core/smpp/server/InEvent.java | 3 + .../InMemorySmppServerMessageStore.java | 68 ++++++-- .../smpp/server/SmppServerMessageStore.java | 4 + .../core/smpp/server/SmppServerWorker.java | 145 +++++++++++++--- .../core/worker/ConfiguredDlrStorage.java | 5 + .../core/worker/DlrMessageStorage.java | 5 + .../sendium/core/worker/DlrService.java | 4 + .../core/worker/PostgresqlDlrStorage.java | 88 ++++++---- .../InMemorySmppServerMessageStoreTest.java | 72 ++++++-- .../SmppServerWorkerReassemblyTest.java | 163 +++++++++++++++++- .../core/worker/ConfiguredDlrStorageTest.java | 15 ++ .../core/worker/PostgresqlDlrStorageTest.java | 31 ++++ 12 files changed, 509 insertions(+), 94 deletions(-) diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java index 1554093..473c7a2 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java @@ -14,6 +14,7 @@ public class InEvent { public boolean concatenated; public boolean waitingForResponse; public String responseMessageId; + public int persistenceAttempts; public InEvent(M pMsg, SubmitSm submitSm, int mpid, Timestamp localTimestamp) { this(pMsg, submitSm, mpid, localTimestamp, true, null); @@ -28,6 +29,7 @@ public InEvent(M pMsg, SubmitSm submitSm, int mpid, Timestamp localTimestamp, bo this.concatenated = false; this.waitingForResponse = waitingForResponse; this.responseMessageId = messageId; + this.persistenceAttempts = 0; } public String toString() { @@ -41,6 +43,7 @@ public String toString() { ", concatenated=" + concatenated + ", waitingForResponse=" + waitingForResponse + ", responseMessageId=" + responseMessageId + + ", persistenceAttempts=" + persistenceAttempts + '}'; } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java index 05c57ce..f3fe714 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java @@ -2,6 +2,7 @@ import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.worker.DlrService; +import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.util.MessageTrace; import gr.cytech.sendium.util.SensitiveLogSanitizer; @@ -9,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -35,25 +37,59 @@ public void stop() { @Override public Future persistMessages(List> eventsQueue) { + boolean persisted = true; + int start = 0; + while (start < eventsQueue.size()) { + boolean clientSubmission = isClientSubmission(eventsQueue.get(start)); + int end = start + 1; + while (end < eventsQueue.size() && isClientSubmission(eventsQueue.get(end)) == clientSubmission) { + end++; + } + persisted = persistBatch(eventsQueue.subList(start, end)) && persisted; + start = end; + } + return CompletableFuture.completedFuture(persisted); + } + + private boolean isClientSubmission(InEvent event) { + return event != null && event.submitSm != null; + } + + private boolean persistBatch(List> eventsQueue) { + if (eventsQueue.isEmpty()) { + return true; + } + List states = new ArrayList<>(eventsQueue.size()); for (InEvent event : eventsQueue) { - try { - StandardMessage msg = event.pMsg; - if (msg != null) { - String gatewayMsgId = msg.serial; - String accountId = msg.owner_id; - String systemId = msg.systemId; - String sourceAddr = msg.from; - String destAddr = msg.to; - - MessageState state = new MessageState(gatewayMsgId, accountId, systemId, sourceAddr, destAddr, null); - state.setReassembledParts(msg.reassembledParts); - worker.getWorkerResources().getDlrService().saveInitialState(state); - } - } catch (Exception e) { - logger.error("Failed to persist message state", e); + if (event == null) { + continue; } + StandardMessage msg = event.pMsg; + if (msg != null) { + MessageState state = new MessageState(msg.serial, msg.owner_id, msg.systemId, msg.from, msg.to, null); + state.setReassembledParts(msg.reassembledParts); + states.add(state); + } + } + + try { + getDlrService().saveInitialStates(states); + } catch (DlrStorageException e) { + logger.error("SMPP submission batch rejected: DLR storage unavailable"); + worker.handleMessagePersistenceFailure(eventsQueue); + return false; + } catch (Exception e) { + logger.error("Failed to persist SMPP submission batch", e); + worker.handleMessagePersistenceFailure(eventsQueue); + return false; } - return CompletableFuture.completedFuture(true); + worker.handlePersistedMessages(eventsQueue); + return true; + } + + @Override + public boolean persistsBeforeAcknowledgement() { + return true; } @Override diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java index 775099b..c779a99 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java @@ -22,6 +22,10 @@ public interface SmppServerMessageStore { */ Future persistMessages(List> eventsQueue); + default boolean persistsBeforeAcknowledgement() { + return false; + } + /** * Mark a message as unpushed to retry it later. * @return true if successfully handled by the store, false if the worker should handle the retry in-memory. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java index 187dc43..720e87a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java @@ -238,9 +238,12 @@ public Thread start() { @Override public boolean stop() { logger.info("Stopping SMPP server..."); - messagePartsHandler.stop(); keepOnRunning = false; - messageStore.stop(); + messagePartsHandler.stop(); + boolean drainPersistedIngress = messageStore != null && messageStore.persistsBeforeAcknowledgement(); + if (!drainPersistedIngress) { + messageStore.stop(); + } if (inactivityTimeFuture != null) { try { inactivityTimeFuture.cancel(true); @@ -252,7 +255,14 @@ public boolean stop() { inExecutorRunnable.die(); } stopExecutor(inExecutor, "in"); - stopExecutor(monitorExecutor, "monitor"); + boolean ingressDrained = true; + if (drainPersistedIngress) { + stopExecutor(monitorExecutor, "monitor"); + ingressDrained = drainPersistedIngress(); + messageStore.stop(); + } else { + stopExecutor(monitorExecutor, "monitor"); + } stopExecutor(outExecutor, "out"); destroyServer(server); destroyServer(tlsServer); @@ -272,7 +282,36 @@ public boolean stop() { tlsServer != null ? tlsServer.getCounters() : null, proxyServer != null ? proxyServer.getCounters() : null); } - return super.stop(); + boolean workerStopped = super.stop(); + return ingressDrained && workerStopped; + } + + private boolean drainPersistedIngress() { + int batchSize = Math.max(1, messageStore.getInsertBatchSize()); + long deadline = System.currentTimeMillis() + Math.max(1_000, getResponseTimeout()); + do { + List> pendingEvents = new ArrayList<>(); + inEventQueue.drainTo(pendingEvents); + for (int offset = 0; offset < pendingEvents.size(); offset += batchSize) { + int end = Math.min(offset + batchSize, pendingEvents.size()); + persistMessagesIn(new ArrayList<>(pendingEvents.subList(offset, end))); + } + if (!inEventQueue.isEmpty() && !isFastUnsafeStop) { + long remainingMillis = deadline - System.currentTimeMillis(); + if (remainingMillis <= 0) { + logger.error("Timed out draining {} acknowledged SMPP ingress events", inEventQueue.size()); + return false; + } + try { + Thread.sleep(Math.min(1_000, remainingMillis)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Interrupted while draining acknowledged SMPP ingress events"); + return false; + } + } + } while (!inEventQueue.isEmpty() && !isFastUnsafeStop); + return inEventQueue.isEmpty(); } protected void destroyServer(com.cloudhopper.smpp.SmppServer smppServer) { @@ -828,34 +867,19 @@ public void enqueueIn(InEvent ine) { if (printMsgs) { logger.debug("IN: {}", ine); } + if (!keepOnRunning) { + enqueueOut(SmppServerUtil.createSubmitRsp(ine.submitSm, SmppConstants.STATUS_SYSERR, null)); + return; + } InEvent filtered = handleBeforeInsertMessageFiltering(ine); if (filtered == null) { return; } filtered.pMsg.serial = UUID.randomUUID().toString(); - if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_ACCEPTED)) { - logger.info("message.accepted ingress=smppserver worker={} {}", getFullName(), MessageTrace.identifiers(filtered.pMsg)); - } - filtered.waitingForResponse = false; filtered.pMsg.ctstamp = ine.localTimestamp.getTime(); filtered.pMsg.onetwork = ine.mpid; - - // Check reassembling logic - if (!Strings.isNullOrEmpty(ine.pMsg.binheader)) { - messagePartsHandler.addMessagePart(ine.pMsg); - enqueueOut(SmppServerUtil.createSubmitRsp(filtered.submitSm, SmppConstants.STATUS_OK, filtered.pMsg.serial)); - return; - } - try { - enqueueToRouter(ine.pMsg); - enqueueOut(SmppServerUtil.createSubmitRsp(filtered.submitSm, SmppConstants.STATUS_OK, filtered.pMsg.serial)); - } catch (InterruptedException e) { - logger.error("Interrupted while waiting for submit RSP", e); - enqueueOut(SmppServerUtil.createSubmitRsp(filtered.submitSm, SmppConstants.STATUS_UNKNOWNERR, filtered.pMsg.serial)); - throw new RuntimeException(e); - } - inEventQueue.add(ine); + inEventQueue.add(filtered); } protected boolean checkReassembling(M msg) { @@ -867,10 +891,75 @@ protected boolean checkReassembling(M msg) { } public void reEnqueueIn(List> inEvents) { - inEvents.forEach(event -> enqueueToRouterNoExceptions(event.pMsg)); inEventQueue.addAll(inEvents); } + public void handlePersistedMessages(List> events) { + for (InEvent event : events) { + if (event == null) { + continue; + } + if (event.pMsg == null) { + handleMessagePersistenceFailure(List.of(event)); + continue; + } + + try { + if (event.submitSm != null && !Strings.isNullOrEmpty(event.pMsg.binheader)) { + messagePartsHandler.addMessagePart(event.pMsg); + } else { + enqueueToRouter(event.pMsg); + } + if (event.submitSm != null) { + if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_ACCEPTED)) { + logger.info("message.accepted ingress=smppserver worker={} {}", getFullName(), + MessageTrace.identifiers(event.pMsg)); + } + enqueueOut(SmppServerUtil.createSubmitRsp(event.submitSm, SmppConstants.STATUS_OK, event.pMsg.serial)); + event.waitingForResponse = false; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("SMPP submission rejected: router admission interrupted"); + handleMessagePersistenceFailure(List.of(event)); + } catch (Exception e) { + logger.error("SMPP submission rejected after persistence", e); + handleMessagePersistenceFailure(List.of(event)); + } + } + } + + public void handleMessagePersistenceFailure(List> events) { + for (InEvent event : events) { + if (event == null) { + continue; + } + if (event.submitSm == null) { + if (event.pMsg != null) { + schedulePersistenceRetry(event); + } + } else if (event.waitingForResponse) { + try { + enqueueOut(SmppServerUtil.createSubmitRsp(event.submitSm, SmppConstants.STATUS_SYSERR, null)); + event.waitingForResponse = false; + } catch (Exception e) { + logger.error("Failed to enqueue SMPP submission failure response", e); + } + } + } + } + + private void schedulePersistenceRetry(InEvent event) { + event.persistenceAttempts++; + if (keepOnRunning && monitorExecutor != null && !monitorExecutor.isShutdown()) { + long delayMillis = Math.min(TimeUnit.SECONDS.toMillis(30), + TimeUnit.SECONDS.toMillis(event.persistenceAttempts)); + monitorExecutor.schedule(() -> reEnqueueIn(List.of(event)), delayMillis, TimeUnit.MILLISECONDS); + return; + } + reEnqueueIn(List.of(event)); + } + public InEvent handleBeforeInsertMessageFiltering(InEvent ine) { List filters = getBeforeInsertMessageFilters(); if (filters == null || filters.isEmpty()) { @@ -1157,7 +1246,11 @@ public void onMessagePartsHandlingEvent(MessagePartsHandler.MessagePartsEventTyp reEnqueueIn(List.of(event)); } else { var messages = parts.stream().map(m -> new InEvent(m, null, m.onetwork, new Timestamp(m.ctstamp))).collect(Collectors.toList()); - reEnqueueIn(messages); + if (messageStore != null && messageStore.persistsBeforeAcknowledgement()) { + handlePersistedMessages(messages); + } else { + reEnqueueIn(messages); + } } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java index f494bd2..120d8d6 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java @@ -114,6 +114,11 @@ public void saveInitialState(MessageState state) { timed("save_initial", () -> delegate.saveInitialState(state)); } + @Override + public void saveInitialStates(List states) { + timed("save_initial_batch", () -> delegate.saveInitialStates(states)); + } + @Override public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { timed("link_operator", () -> delegate.linkOperatorId(gatewayMsgId, operatorMsgId)); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java index d8ea99f..763624d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java @@ -1,10 +1,15 @@ package gr.cytech.sendium.core.worker; +import java.util.List; import java.util.Optional; public interface DlrMessageStorage { void saveInitialState(MessageState state); + default void saveInitialStates(List states) { + states.forEach(this::saveInitialState); + } + void linkOperatorId(String gatewayMsgId, String operatorMsgId); /** diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java index 7810b86..6877a43 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java @@ -19,6 +19,10 @@ public void saveInitialState(MessageState state) { storage.saveInitialState(state); } + public void saveInitialStates(List states) { + storage.saveInitialStates(states); + } + public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { storage.linkOperatorId(gatewayMsgId, operatorMsgId); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java index 6cf8f0d..88edc67 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java @@ -8,6 +8,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; @@ -172,18 +173,53 @@ public PostgresqlDlrStorage(DataSource dataSource) { @Override public void saveInitialState(MessageState state) { - Objects.requireNonNull(state, "state"); + saveInitialStates(List.of(state)); + } + + @Override + public void saveInitialStates(List states) { + Objects.requireNonNull(states, "states"); + if (states.isEmpty()) { + return; + } + + List checkedStates = states.stream() + .map(state -> Objects.requireNonNull(state, "state")) + .toList(); + List gatewayMsgIds = checkedStates.stream() + .map(state -> parseGatewayId(state.getGatewayMsgId())) + .toList(); checkExpiry(); - UUID gatewayMsgId = parseGatewayId(state.getGatewayMsgId()); try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); - try { - saveState(connection, gatewayMsgId, state); - deleteCorrelations(connection, gatewayMsgId); - if (state.getOperatorMsgId() != null && - !saveCorrelation(connection, gatewayMsgId, state.getOperatorMsgId())) { - throw new SQLException("Operator message ID is already linked to another gateway message"); + try (PreparedStatement saveStates = connection.prepareStatement(SAVE_INITIAL_STATE_SQL); + PreparedStatement deleteCorrelations = connection.prepareStatement(DELETE_CORRELATIONS_SQL); + PreparedStatement saveCorrelations = connection.prepareStatement(SAVE_CORRELATION_SQL)) { + int correlationCount = 0; + for (int index = 0; index < checkedStates.size(); index++) { + MessageState state = checkedStates.get(index); + UUID gatewayMsgId = gatewayMsgIds.get(index); + setStateParameters(connection, saveStates, gatewayMsgId, state); + saveStates.addBatch(); + deleteCorrelations.setObject(1, gatewayMsgId); + deleteCorrelations.addBatch(); + if (state.getOperatorMsgId() != null) { + saveCorrelations.setString(1, state.getOperatorMsgId()); + saveCorrelations.setObject(2, gatewayMsgId); + saveCorrelations.addBatch(); + correlationCount++; + } + } + + saveStates.executeBatch(); + deleteCorrelations.executeBatch(); + if (correlationCount > 0) { + for (int result : saveCorrelations.executeBatch()) { + if (result == 0 || result == Statement.EXECUTE_FAILED) { + throw new SQLException("Operator message ID is already linked to another gateway message"); + } + } } connection.commit(); } catch (SQLException e) { @@ -191,7 +227,7 @@ public void saveInitialState(MessageState state) { throw e; } } catch (SQLException e) { - throw failure("save initial DLR state", e); + throw failure("save initial DLR states", e); } } @@ -424,28 +460,18 @@ private boolean markAsSent(Connection connection, UUID gatewayMsgId, } } - private void saveState(Connection connection, UUID gatewayMsgId, - MessageState state) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(SAVE_INITIAL_STATE_SQL)) { - statement.setObject(1, gatewayMsgId); - statement.setString(2, state.getAccountId()); - statement.setString(3, state.getSystemId()); - statement.setString(4, state.getSourceAddr()); - statement.setString(5, state.getDestAddr()); - statement.setString(6, state.getOperatorMsgId()); - statement.setString(7, state.getForwardDlrUrl()); - setStringArray(connection, statement, 8, state.getReassembledParts()); - statement.setString(9, state.getStatus().name()); - statement.setTimestamp(10, new Timestamp(state.getTimestamp())); - statement.executeUpdate(); - } - } - - private void deleteCorrelations(Connection connection, UUID gatewayMsgId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(DELETE_CORRELATIONS_SQL)) { - statement.setObject(1, gatewayMsgId); - statement.executeUpdate(); - } + private void setStateParameters(Connection connection, PreparedStatement statement, UUID gatewayMsgId, + MessageState state) throws SQLException { + statement.setObject(1, gatewayMsgId); + statement.setString(2, state.getAccountId()); + statement.setString(3, state.getSystemId()); + statement.setString(4, state.getSourceAddr()); + statement.setString(5, state.getDestAddr()); + statement.setString(6, state.getOperatorMsgId()); + statement.setString(7, state.getForwardDlrUrl()); + setStringArray(connection, statement, 8, state.getReassembledParts()); + statement.setString(9, state.getStatus().name()); + statement.setTimestamp(10, new Timestamp(state.getTimestamp())); } private boolean saveCorrelation(Connection connection, UUID gatewayMsgId, diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java index 70ad610..47f46c8 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java @@ -1,13 +1,16 @@ package gr.cytech.sendium.core.smpp.server; +import com.cloudhopper.smpp.pdu.SubmitSm; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.worker.DlrService; +import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.external.WorkerResourceProvider; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -45,7 +48,7 @@ void setUp() { } @Test - void persistMessages_SavesStateForEachMessage() { + void persistMessages_SavesStatesAsOneBatchBeforeNotifyingWorker() { List> events = new ArrayList<>(); StandardMessage msg1 = new StandardMessage(); @@ -62,20 +65,24 @@ void persistMessages_SavesStateForEachMessage() { msg2.from = "from2"; msg2.to = "to2"; - InEvent event1 = new InEvent<>(msg1, null, 1, new Timestamp(System.currentTimeMillis())); - InEvent event2 = new InEvent<>(msg2, null, 2, new Timestamp(System.currentTimeMillis())); + InEvent event1 = new InEvent<>(msg1, new SubmitSm(), 1, + new Timestamp(System.currentTimeMillis())); + InEvent event2 = new InEvent<>(msg2, new SubmitSm(), 2, + new Timestamp(System.currentTimeMillis())); events.add(event1); events.add(event2); messageStore.persistMessages(events); - ArgumentCaptor captor = ArgumentCaptor.forClass(MessageState.class); - verify(dlrService, times(2)).saveInitialState(captor.capture()); - assertEquals("account1", captor.getAllValues().get(0).getAccountId()); - assertEquals("sys1", captor.getAllValues().get(0).getSystemId()); - assertEquals("account2", captor.getAllValues().get(1).getAccountId()); - assertEquals("sys2", captor.getAllValues().get(1).getSystemId()); + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + InOrder order = inOrder(dlrService, worker); + order.verify(dlrService).saveInitialStates(captor.capture()); + order.verify(worker).handlePersistedMessages(events); + assertEquals("account1", captor.getValue().get(0).getAccountId()); + assertEquals("sys1", captor.getValue().get(0).getSystemId()); + assertEquals("account2", captor.getValue().get(1).getAccountId()); + assertEquals("sys2", captor.getValue().get(1).getSystemId()); } @Test @@ -90,9 +97,9 @@ void persistMessages_SavesReassembledPartIds() { messageStore.persistMessages(List.of(new InEvent<>(msg, null, 1, new Timestamp(System.currentTimeMillis())))); - ArgumentCaptor captor = ArgumentCaptor.forClass(MessageState.class); - verify(dlrService).saveInitialState(captor.capture()); - assertEquals(List.of("part-1", "part-2"), captor.getValue().getReassembledParts()); + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(dlrService).saveInitialStates(captor.capture()); + assertEquals(List.of("part-1", "part-2"), captor.getValue().getFirst().getReassembledParts()); } @Test @@ -105,7 +112,40 @@ void persistMessages_WithNullMessage_Skips() { messageStore.persistMessages(events); - verify(dlrService, never()).saveInitialState(any(MessageState.class)); + verify(dlrService).saveInitialStates(List.of()); + verify(worker).handlePersistedMessages(events); + } + + @Test + void persistMessages_WhenStorageFails_NotifiesWorkerFailure() { + StandardMessage msg = new StandardMessage(); + msg.serial = "gw-1"; + List> events = List.of( + new InEvent<>(msg, new SubmitSm(), 1, new Timestamp(System.currentTimeMillis()))); + doThrow(new DlrStorageException("database details")) + .when(dlrService).saveInitialStates(anyList()); + + assertFalse(messageStore.persistMessages(events).resultNow()); + + verify(worker).handleMessagePersistenceFailure(events); + verify(worker, never()).handlePersistedMessages(anyList()); + } + + @Test + void persistMessages_IsolatesInternalEventsWithoutReorderingCallbacks() { + InEvent firstClient = event("first-client", new SubmitSm()); + InEvent internal = event("internal", null); + InEvent secondClient = event("second-client", new SubmitSm()); + + messageStore.persistMessages(List.of(firstClient, internal, secondClient)); + + InOrder order = inOrder(dlrService, worker); + order.verify(dlrService).saveInitialStates(anyList()); + order.verify(worker).handlePersistedMessages(List.of(firstClient)); + order.verify(dlrService).saveInitialStates(anyList()); + order.verify(worker).handlePersistedMessages(List.of(internal)); + order.verify(dlrService).saveInitialStates(anyList()); + order.verify(worker).handlePersistedMessages(List.of(secondClient)); } @Test @@ -124,6 +164,12 @@ void getMaxAttempts_DefaultsTo3_WhenNoWorker() { assertEquals(3, result); } + private InEvent event(String serial, SubmitSm submitSm) { + StandardMessage message = new StandardMessage(); + message.serial = serial; + return new InEvent<>(message, submitSm, 1, new Timestamp(System.currentTimeMillis())); + } + @Test void markAsUnpushed_Dlr_SavesToDlrService() { StandardMessage msg = new StandardMessage(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java index a2227f6..8a34417 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java @@ -3,12 +3,16 @@ import com.cloudhopper.commons.charset.CharsetUtil; import com.cloudhopper.smpp.SmppConstants; import com.cloudhopper.smpp.pdu.DeliverSm; +import com.cloudhopper.smpp.pdu.Pdu; +import com.cloudhopper.smpp.pdu.SubmitSm; +import com.cloudhopper.smpp.pdu.SubmitSmResp; import gr.cytech.sendium.conf.PropertyChangeListener; import gr.cytech.sendium.conf.SendiumConfigurationProvider; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; import org.junit.jupiter.api.Test; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -19,6 +23,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class SmppServerWorkerReassemblyTest { @@ -26,6 +32,9 @@ class SmppServerWorkerReassemblyTest { void completeUdhPartsAreReassembledAndRoutedToRouterQueue() throws Exception { Queue routerQueue = new Queue<>(); TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + when(store.persistsBeforeAcknowledgement()).thenReturn(true); + worker.setMessageStore(store); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); MessagePartsHandler handler = new MessagePartsHandler<>( worker.new CcatMessagePartsEventsListener(), TimeUnit.SECONDS.toMillis(30), executor); @@ -34,16 +43,16 @@ void completeUdhPartsAreReassembledAndRoutedToRouterQueue() throws Exception { handler.addMessagePart(messagePart("0500037F0202", "World", "part-2")); handler.addMessagePart(messagePart("0500037F0201", "Hello ", "part-1")); + assertThat(routerQueue.dequeue(10)).isNull(); + InEvent persisted = worker.getInEventQueue().poll(1_000, TimeUnit.MILLISECONDS); + assertThat(persisted).isNotNull(); + worker.handlePersistedMessages(List.of(persisted)); StandardMessage routed = routerQueue.dequeue(1_000); - - assertThat(routed).isNotNull(); assertThat(routed.body).isEqualTo("Hello World"); assertThat(routed.binheader).isNull(); assertThat(routed.reassembledParts).containsExactly("part-1", "part-2"); assertThat(worker.workerQueueMessages).isEmpty(); - InEvent persisted = worker.getInEventQueue().poll(1_000, TimeUnit.MILLISECONDS); - assertThat(persisted).isNotNull(); assertThat(persisted.pMsg).isSameAs(routed); } finally { executor.shutdownNow(); @@ -54,6 +63,9 @@ void completeUdhPartsAreReassembledAndRoutedToRouterQueue() throws Exception { void delayedUdhPartsAreRoutedToRouterQueueWithoutBecomingDeliverSm() throws Exception { Queue routerQueue = new Queue<>(); TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + when(store.persistsBeforeAcknowledgement()).thenReturn(true); + worker.setMessageStore(store); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); MessagePartsHandler handler = new MessagePartsHandler<>( worker.new CcatMessagePartsEventsListener(), 10, executor); @@ -63,16 +75,13 @@ void delayedUdhPartsAreRoutedToRouterQueueWithoutBecomingDeliverSm() throws Exce handler.addMessagePart(part); StandardMessage routed = routerQueue.dequeue(1_000); - assertThat(routed).isSameAs(part); assertThat(routed.body).isEqualTo("Hello "); assertThat(routed.binheader).isEqualTo("0500037F0201"); assertThat(routed.reassembledParts).isNull(); assertThat(worker.workerQueueMessages).isEmpty(); - InEvent persisted = worker.getInEventQueue().poll(1_000, TimeUnit.MILLISECONDS); - assertThat(persisted).isNotNull(); - assertThat(persisted.pMsg).isSameAs(part); + assertThat(worker.getInEventQueue()).isEmpty(); } finally { executor.shutdownNow(); } @@ -106,6 +115,137 @@ void reassembledDlrGeneratesDeliverSmPerOriginalPartIdWithSameStatus() throws Ex assertThat(bodies).anySatisfy(body -> assertThat(body).contains("id:part-3")); } + @Test + void normalSubmissionRoutesAndAcknowledgesOnlyAfterPersistence() throws Exception { + Queue routerQueue = new Queue<>(); + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + StandardMessage message = messagePart(null, "hello", null); + SubmitSm submitSm = new SubmitSm(); + submitSm.setSequenceNumber(42); + InEvent event = new InEvent<>(message, submitSm, 1, + new Timestamp(System.currentTimeMillis())); + + worker.enqueueIn(event); + + InEvent queued = worker.getInEventQueue().poll(); + assertThat(queued).isSameAs(event); + assertThat(routerQueue.dequeue(10)).isNull(); + assertThat(worker.outgoingPdus).isEmpty(); + + worker.handlePersistedMessages(List.of(queued)); + + assertThat(routerQueue.dequeue(1_000)).isSameAs(message); + assertThat(worker.outgoingPdus).singleElement().satisfies(pdu -> { + assertThat(pdu).isInstanceOf(SubmitSmResp.class); + SubmitSmResp response = (SubmitSmResp) pdu; + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK); + assertThat(response.getMessageId()).isEqualTo(message.serial); + }); + } + + @Test + void persistenceFailureReturnsSystemErrorWithoutRouting() throws Exception { + Queue routerQueue = new Queue<>(); + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + StandardMessage message = messagePart(null, "hello", null); + SubmitSm submitSm = new SubmitSm(); + InEvent event = new InEvent<>(message, submitSm, 1, + new Timestamp(System.currentTimeMillis())); + worker.enqueueIn(event); + InEvent queued = worker.getInEventQueue().poll(); + + worker.handleMessagePersistenceFailure(List.of(queued)); + + assertThat(routerQueue.dequeue(10)).isNull(); + assertThat(worker.outgoingPdus).singleElement().satisfies(pdu -> { + assertThat(pdu).isInstanceOf(SubmitSmResp.class); + assertThat(pdu.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR); + }); + } + + @Test + void submissionDuringShutdownIsRejectedWithoutQueueAdmission() throws Exception { + Queue routerQueue = new Queue<>(); + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + worker.setKeepOnRunning(false); + InEvent event = new InEvent<>(messagePart(null, "hello", null), new SubmitSm(), 1, + new Timestamp(System.currentTimeMillis())); + + worker.enqueueIn(event); + + assertThat(worker.getInEventQueue()).isEmpty(); + assertThat(routerQueue.dequeue(10)).isNull(); + assertThat(worker.outgoingPdus).singleElement() + .satisfies(pdu -> assertThat(pdu.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR)); + } + + @Test + void routerAdmissionFailureReturnsSystemErrorAfterPersistence() throws Exception { + Queue routerQueue = new Queue<>() { + @Override + public void enqueue(StandardMessage message) throws InterruptedException { + throw new InterruptedException("router unavailable"); + } + }; + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + StandardMessage message = messagePart(null, "hello", null); + SubmitSm submitSm = new SubmitSm(); + InEvent event = new InEvent<>(message, submitSm, 1, + new Timestamp(System.currentTimeMillis())); + worker.enqueueIn(event); + InEvent queued = worker.getInEventQueue().poll(); + + try { + worker.handlePersistedMessages(List.of(queued)); + + assertThat(worker.outgoingPdus).singleElement() + .satisfies(pdu -> assertThat(pdu.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR)); + } finally { + Thread.interrupted(); + } + } + + @Test + void failedAggregatePersistenceRequeuesWithoutAnotherClientResponse() { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + StandardMessage aggregate = messagePart(null, "Hello World", "part-1"); + aggregate.reassembledParts = new ArrayList<>(List.of("part-1", "part-2")); + InEvent event = new InEvent<>(aggregate, null, 1, + new Timestamp(System.currentTimeMillis())); + + worker.handleMessagePersistenceFailure(List.of(event)); + + assertThat(worker.getInEventQueue()).containsExactly(event); + assertThat(worker.outgoingPdus).isEmpty(); + } + + @Test + void multipartPartIsBufferedAndAcknowledgedOnlyAfterProvisionalPersistence() throws Exception { + Queue routerQueue = new Queue<>(); + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), routerQueue); + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); + worker.setMessagePartsHandler(new MessagePartsHandler<>( + worker.new CcatMessagePartsEventsListener(), TimeUnit.SECONDS.toMillis(30), executor)); + StandardMessage part = messagePart("0500037F0201", "Hello ", null); + SubmitSm submitSm = new SubmitSm(); + InEvent event = new InEvent<>(part, submitSm, 1, + new Timestamp(System.currentTimeMillis())); + + try { + worker.enqueueIn(event); + InEvent queued = worker.getInEventQueue().poll(); + assertThat(worker.outgoingPdus).isEmpty(); + + worker.handlePersistedMessages(List.of(queued)); + + assertThat(routerQueue.dequeue(10)).isNull(); + assertThat(worker.outgoingPdus).singleElement() + .satisfies(pdu -> assertThat(pdu.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK)); + } finally { + executor.shutdownNow(); + } + } + private static StandardMessage messagePart(String udh, String body, String serial) { StandardMessage message = new StandardMessage(); message.owner_id = "account-a"; @@ -122,6 +262,7 @@ private static StandardMessage messagePart(String udh, String body, String seria private static class TestSmppServerWorker extends SmppServerWorker { private final List workerQueueMessages = new ArrayList<>(); + private final List outgoingPdus = new ArrayList<>(); TestSmppServerWorker(SendiumConfigurationProvider configurationProvider, Queue routerQueue) { super(configurationProvider, "smpp", routerQueue); @@ -131,6 +272,12 @@ private static class TestSmppServerWorker extends SmppServerWorker states = List.of( + new MessageState("gateway-id", "system", "source", "destination", null)); + storage.initialize(); + + storage.saveInitialStates(states); + + verify(mvStore).saveInitialStates(states); + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "mvstore", "operation", "save_initial_batch", "outcome", "success") + .timer().count()).isOne(); + } + @Test void rejectsUnknownBackend() { storage.configuredBackend = "unknown"; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java index d7159c4..2dbd527 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java @@ -86,6 +86,37 @@ void saveInitialStateRoundTripsAllFields() { .isPresent(); } + @Test + void saveInitialStatesCommitsWholeBatch() { + List states = List.of(newState(), newState(), newState()); + + storage.saveInitialStates(states); + + for (MessageState state : states) { + assertThat(storage.getState(state.getGatewayMsgId())) + .get() + .usingRecursiveComparison() + .isEqualTo(state); + } + } + + @Test + void saveInitialStatesRollsBackWholeBatchOnCorrelationConflict() { + MessageState owner = newState(); + owner.setOperatorMsgId("shared-operator"); + storage.saveInitialState(owner); + MessageState innocent = newState(); + MessageState conflict = newState(); + conflict.setOperatorMsgId("shared-operator"); + + assertThatThrownBy(() -> storage.saveInitialStates(List.of(innocent, conflict))) + .isInstanceOf(DlrStorageException.class); + + assertThat(storage.getState(innocent.getGatewayMsgId())).isEmpty(); + assertThat(storage.getState(conflict.getGatewayMsgId())).isEmpty(); + assertThat(storage.getState(owner.getGatewayMsgId())).isPresent(); + } + @Test void saveInitialStateOverwritesExistingState() throws SQLException { MessageState initial = newState(); From 93aeb462fec28372cb7a0da2b38a135ed126f7c5 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 10:58:22 +0300 Subject: [PATCH 08/20] test(dlr): verify PostgreSQL outage recovery --- .../PostgresqlDlrQuarkusTestResource.java | 70 ++++++- .../core/worker/PostgresqlDlrRuntimeTest.java | 176 +++++++++++++++++- .../core/worker/PostgresqlDlrStorageTest.java | 23 +++ 3 files changed, 257 insertions(+), 12 deletions(-) diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java index bf8f60e..19b72fb 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java @@ -3,10 +3,16 @@ import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; import org.testcontainers.postgresql.PostgreSQLContainer; +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; public class PostgresqlDlrQuarkusTestResource implements QuarkusTestResourceLifecycleManager { - private PostgreSQLContainer postgresql; + private static PostgreSQLContainer postgresql; + private static int smppPort; + private Path smppConfiguration; @Override public Map start() { @@ -19,20 +25,66 @@ public Map start() { .withUsername("sendium") .withPassword("sendium-test"); postgresql.start(); - return Map.of( - "sendium.dlr.storage", "postgresql", - "quarkus.datasource.dlr.active", "true", - "quarkus.flyway.dlr.active", "true", - "quarkus.flyway.dlr.migrate-at-start", "true", - "quarkus.datasource.dlr.jdbc.url", postgresql.getJdbcUrl(), - "quarkus.datasource.dlr.username", postgresql.getUsername(), - "quarkus.datasource.dlr.password", postgresql.getPassword()); + smppPort = findFreePort(); + smppConfiguration = createSmppConfiguration(smppPort); + String jdbcUrl = postgresql.getJdbcUrl() + "&connectTimeout=2&socketTimeout=2"; + return Map.ofEntries( + Map.entry("sendium.dlr.storage", "postgresql"), + Map.entry("quarkus.datasource.dlr.active", "true"), + Map.entry("quarkus.flyway.dlr.active", "true"), + Map.entry("quarkus.flyway.dlr.migrate-at-start", "true"), + Map.entry("quarkus.datasource.dlr.jdbc.url", jdbcUrl), + Map.entry("quarkus.datasource.dlr.username", postgresql.getUsername()), + Map.entry("quarkus.datasource.dlr.password", postgresql.getPassword()), + Map.entry("smsg.properties.file.path", smppConfiguration.toString())); } @Override public void stop() { if (postgresql != null) { postgresql.stop(); + postgresql = null; + } + if (smppConfiguration != null) { + try { + Files.deleteIfExists(smppConfiguration); + } catch (IOException ignored) { + // Temporary test configuration is also cleaned by the operating system. + } + } + } + + static void pausePostgresql() { + postgresql.getDockerClient().pauseContainerCmd(postgresql.getContainerId()).exec(); + } + + static void resumePostgresql() { + postgresql.getDockerClient().unpauseContainerCmd(postgresql.getContainerId()).exec(); + } + + static int getSmppPort() { + return smppPort; + } + + private static int findFreePort() { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } catch (IOException e) { + throw new IllegalStateException("Could not allocate an SMPP test port", e); + } + } + + private static Path createSmppConfiguration(int port) { + try { + String configuration = Files.readString(Path.of("src", "test", "resources", "smsg.properties")) + .replace("outSms.instance.smpp.enable = truef", "outSms.instance.smpp.enable = true") + .replace("outSms.instance.smpp.srv.port = 27777", + "outSms.instance.smpp.srv.port = " + port); + Path temporaryConfiguration = Files.createTempFile("sendium-postgresql-", ".properties"); + Files.writeString(temporaryConfiguration, configuration); + return temporaryConfiguration; + } catch (IOException e) { + throw new IllegalStateException("Could not create the SMPP test configuration", e); } } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java index 4f58699..252fd40 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java @@ -1,5 +1,17 @@ package gr.cytech.sendium.core.worker; +import com.cloudhopper.smpp.SmppConstants; +import com.cloudhopper.smpp.SmppBindType; +import com.cloudhopper.smpp.SmppSession; +import com.cloudhopper.smpp.SmppSessionConfiguration; +import com.cloudhopper.smpp.impl.DefaultSmppClient; +import com.cloudhopper.smpp.impl.DefaultSmppSessionHandler; +import com.cloudhopper.smpp.pdu.SubmitSm; +import com.cloudhopper.smpp.pdu.SubmitSmResp; +import com.cloudhopper.smpp.type.Address; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import gr.cytech.sendium.routing.OutgoingWorkerManager; +import gr.cytech.sendium.routing.StandardOutgoingWorkerHandler; import io.agroal.api.AgroalDataSource; import io.micrometer.core.instrument.MeterRegistry; import io.quarkus.agroal.DataSource; @@ -11,12 +23,17 @@ import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import utils.CaptorWorker; +import io.netty.channel.nio.NioEventLoopGroup; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; @@ -43,8 +60,12 @@ class PostgresqlDlrRuntimeTest { @Inject MeterRegistry meterRegistry; + @Inject + OutgoingWorkerManager outgoingWorkerManager; + @Test void wiresPoolMigrationStorageHealthAndMetrics() throws SQLException { + long successfulSavesBefore = metricCount("save_initial", "success"); assertThat(storage).isSameAs(configuredStorage); assertThat(configuredStorage.backend()).isEqualTo("postgresql"); assertThat(dataSource.getHandle().getBean().isActive()).isTrue(); @@ -67,14 +88,80 @@ void wiresPoolMigrationStorageHealthAndMetrics() throws SQLException { assertThat(meterRegistry.find("sendium.dlr.storage.selected") .tag("backend", "postgresql").gauge().value()).isEqualTo(1.0); - assertThat(meterRegistry.find("sendium.dlr.storage.operation") - .tags("backend", "postgresql", "operation", "save_initial", "outcome", "success") - .timer().count()).isOne(); + assertThat(metricCount("save_initial", "success")).isEqualTo(successfulSavesBefore + 1); assertThat(meterRegistry.getMeters()) .extracting(meter -> meter.getId().getName()) .anyMatch(name -> name.startsWith("agroal")); } + @Test + void databaseOutageRejectsHttpAndSmppWithoutRoutingOrFallback() throws Exception { + StandardOutgoingWorkerHandler outgoingWorkerHandler = (StandardOutgoingWorkerHandler) outgoingWorkerManager; + CaptorWorker captorWorker = (CaptorWorker) outgoingWorkerHandler.getWorkers().get("captorTest"); + captorWorker.captures.clear(); + try (DownstreamSmppClient smppClient = new DownstreamSmppClient( + PostgresqlDlrQuarkusTestResource.getSmppPort())) { + smppClient.start(); + PostgresqlDlrQuarkusTestResource.pausePostgresql(); + try { + given() + .queryParam("username", "test2") + .queryParam("password", "123qwe") + .queryParam("from", "Sender") + .queryParam("to", "306910000000") + .queryParam("text", "database outage http") + .when().get("/sendsms") + .then() + .statusCode(503) + .body(equalTo("Temporal failure, try again later.")); + + SubmitSmResp failedSmpp = smppClient.sendSms( + "Sender", "306910000001", "database outage smpp"); + assertThat(failedSmpp.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR); + assertThat(failedSmpp.getMessageId()).isBlank(); + assertThat(captorWorker.captures).isEmpty(); + assertThat(configuredStorage.backend()).isEqualTo("postgresql"); + + given() + .when().get("/q/health/ready") + .then() + .statusCode(503) + .body("status", equalTo("DOWN")) + .body("checks.find { it.name == 'sendium-dlr-storage' }.data.reason", + equalTo("unavailable")); + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "postgresql", "operation", "save_initial", "outcome", "error") + .timer().count()).isGreaterThanOrEqualTo(1); + assertThat(meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "postgresql", "operation", "save_initial_batch", "outcome", "error") + .timer().count()).isGreaterThanOrEqualTo(1); + } finally { + PostgresqlDlrQuarkusTestResource.resumePostgresql(); + awaitPostgresqlRecovery(); + } + + String httpGatewayId = submitHttpAfterRecovery(); + SubmitSmResp recoveredSmpp = smppClient.sendSms( + "Sender", "306910000003", "database recovered smpp"); + assertThat(recoveredSmpp.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK); + assertThat(recoveredSmpp.getMessageId()).isNotBlank(); + assertThat(storage.getState(httpGatewayId)).isPresent(); + assertThat(storage.getState(recoveredSmpp.getMessageId())).isPresent(); + var firstRouted = captorWorker.captures.poll(5, TimeUnit.SECONDS); + var secondRouted = captorWorker.captures.poll(5, TimeUnit.SECONDS); + assertThat(firstRouted).isNotNull(); + assertThat(secondRouted).isNotNull(); + assertThat(List.of(firstRouted.body, secondRouted.body)) + .containsExactlyInAnyOrder("database recovered http", "database recovered smpp"); + + given() + .when().get("/q/health/ready") + .then() + .statusCode(200) + .body("status", equalTo("UP")); + } + } + private int flywayHistoryCount() throws SQLException { try (Connection connection = dataSource.get().getConnection(); Statement statement = connection.createStatement(); @@ -87,4 +174,87 @@ SELECT COUNT(*) return resultSet.getInt(1); } } + + private long metricCount(String operation, String outcome) { + var timer = meterRegistry.find("sendium.dlr.storage.operation") + .tags("backend", "postgresql", "operation", operation, "outcome", outcome) + .timer(); + return timer == null ? 0 : timer.count(); + } + + private void awaitPostgresqlRecovery() throws InterruptedException { + DlrStorageException lastFailure = null; + for (int attempt = 0; attempt < 20; attempt++) { + try { + storage.getState(UUID.randomUUID().toString()); + return; + } catch (DlrStorageException e) { + lastFailure = e; + Thread.sleep(250); + } + } + throw new AssertionError("PostgreSQL storage did not recover", lastFailure); + } + + private String submitHttpAfterRecovery() { + return given() + .queryParam("username", "test2") + .queryParam("password", "123qwe") + .queryParam("from", "Sender") + .queryParam("to", "306910000002") + .queryParam("text", "database recovered http") + .when().get("/sendsms") + .then() + .statusCode(202) + .extract().asString(); + } + + private static class DownstreamSmppClient implements AutoCloseable { + private final int port; + private final DefaultSmppClient client; + private SmppSession session; + + DownstreamSmppClient(int port) { + this.port = port; + this.client = new DefaultSmppClient(new NioEventLoopGroup( + 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("postgresql-smpp-client-%d").build())); + } + + void start() throws Exception { + SmppSessionConfiguration configuration = new SmppSessionConfiguration( + SmppBindType.TRANSCEIVER, "test1", "123qwe"); + configuration.setHost("127.0.0.1"); + configuration.setPort(port); + configuration.setWindowSize(10); + Exception lastFailure = null; + for (int attempt = 0; attempt < 20; attempt++) { + try { + session = client.bind(configuration, new DefaultSmppSessionHandler()); + return; + } catch (Exception e) { + lastFailure = e; + Thread.sleep(250); + } + } + throw new IllegalStateException("Could not bind to the Sendium SMPP test server", lastFailure); + } + + SubmitSmResp sendSms(String from, String to, String text) throws Exception { + SubmitSm submit = new SubmitSm(); + submit.setSourceAddress(new Address((byte) 0, (byte) 0, from)); + submit.setDestAddress(new Address((byte) 0, (byte) 0, to)); + submit.setRegisteredDelivery(SmppConstants.REGISTERED_DELIVERY_SMSC_RECEIPT_REQUESTED); + submit.setDataCoding(SmppConstants.DATA_CODING_DEFAULT); + submit.setShortMessage(text.getBytes(StandardCharsets.UTF_8)); + return session.submit(submit, 10_000); + } + + @Override + public void close() { + if (session != null) { + session.destroy(); + } + client.destroy(0, 0); + } + } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java index 2dbd527..f8677bb 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java @@ -117,6 +117,25 @@ void saveInitialStatesRollsBackWholeBatchOnCorrelationConflict() { assertThat(storage.getState(owner.getGatewayMsgId())).isPresent(); } + @Test + void trackedStateAndCorrelationSurviveAdapterRecreation() { + MessageState state = newState(); + storage.saveInitialState(state); + storage.linkOperatorId(state.getGatewayMsgId(), "operator-after-restart"); + + PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); + + assertThat(recreated.getState(state.getGatewayMsgId())) + .get() + .extracting(MessageState::getOperatorMsgId, MessageState::getStatus) + .containsExactly("operator-after-restart", MessageState.MessageStatus.SENT); + assertThat(recreated.resolveAndRemoveDlr( + "operator-after-restart", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId, MessageState::getStatus) + .containsExactly(state.getGatewayMsgId(), MessageState.MessageStatus.DELIVERED); + } + @Test void saveInitialStateOverwritesExistingState() throws SQLException { MessageState initial = newState(); @@ -378,6 +397,10 @@ void unpushedDlrsAreFilteredAndSurviveAdapterRecreation() { .extracting(message -> message.serial) .containsExactly(second.serial); assertThat(recreated.getUnpushedDlrs("missing-system")).isEmpty(); + assertThat(recreated.claimUnpushedDlrs("system-1")) + .extracting(message -> message.serial) + .containsExactly(first.serial); + assertThat(recreated.claimUnpushedDlrs("system-1")).isEmpty(); } @Test From 1965fe6322c686514d35cfc183067b4ef75a84de Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 12:04:17 +0300 Subject: [PATCH 09/20] feat(deploy): add PostgreSQL quick start --- quick-start.sh | 106 +++++++++++++++++- .../src/main/resources/application.properties | 3 + tests/quick-start-test.sh | 46 +++++++- 3 files changed, 150 insertions(+), 5 deletions(-) diff --git a/quick-start.sh b/quick-start.sh index a132849..ce788f6 100644 --- a/quick-start.sh +++ b/quick-start.sh @@ -15,7 +15,11 @@ force=false provider='' allow_windows_mount=false upstream_password_from_environment=${SENDIUM_UPSTREAM_PASSWORD-} +database_jdbc_url=${SENDIUM_DLR_POSTGRESQL_JDBC_URL-} +database_username=${SENDIUM_DLR_POSTGRESQL_USERNAME-} +database_password=${SENDIUM_DLR_POSTGRESQL_PASSWORD-} unset SENDIUM_UPSTREAM_PASSWORD +unset SENDIUM_DLR_POSTGRESQL_PASSWORD usage() { cat <<'EOF' @@ -37,6 +41,11 @@ Non-interactive provider configuration: Set SENDIUM_UPSTREAM_USERNAME and SENDIUM_UPSTREAM_PASSWORD for ProSMS. Custom SMPP also uses SENDIUM_UPSTREAM_HOST, SENDIUM_UPSTREAM_PORT, and SENDIUM_UPSTREAM_TLS. + +External PostgreSQL configuration: + Set SENDIUM_DLR_POSTGRESQL_JDBC_URL, SENDIUM_DLR_POSTGRESQL_USERNAME, + and SENDIUM_DLR_POSTGRESQL_PASSWORD together. Otherwise Quick Start creates + a private PostgreSQL 17 service with a persistent Docker volume. EOF } @@ -171,6 +180,15 @@ validate_port() { [ "$port_value" -ge 1 ] && [ "$port_value" -le 65535 ] || fail "SMPP port must be between 1 and 65535" } +validate_environment_value() { + label=$1 + environment_value=$2 + validate_property_value "$label" "$environment_value" + case "$environment_value" in + *"'"*) fail "$label cannot contain a single quote" ;; + esac +} + while [ "$#" -gt 0 ]; do case "$1" in --directory) @@ -376,6 +394,32 @@ if [ "$upstream_enabled" = true ]; then upstream_password=$(escape_property_value "$upstream_password") fi +external_database=false +if [ -n "$database_jdbc_url" ] || [ -n "$database_username" ] || [ -n "$database_password" ]; then + [ -n "$database_jdbc_url" ] && [ -n "$database_username" ] && [ -n "$database_password" ] || \ + fail "external PostgreSQL requires JDBC URL, username, and password" + case "$database_jdbc_url" in + jdbc:postgresql://*) ;; + *) fail "PostgreSQL JDBC URL must start with jdbc:postgresql://" ;; + esac + validate_environment_value "PostgreSQL JDBC URL" "$database_jdbc_url" + validate_environment_value "PostgreSQL username" "$database_username" + validate_environment_value "PostgreSQL password" "$database_password" + external_database=true +else + database_jdbc_url='jdbc:postgresql://postgres:5432/sendium' + database_username='sendium' + if [ "$force" = true ] && [ -f "$target_dir/.sendium.env" ]; then + existing_database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$target_dir/.sendium.env") + if [ "${#existing_database_password}" -eq 64 ]; then + database_password=$existing_database_password + fi + fi + if [ -z "$database_password" ]; then + database_password=$(generate_secret 32) + fi +fi + mkdir -p "$target_dir/conf" "$target_dir/data" "$target_dir/logs" staging_dir=$(mktemp -d "$target_dir/.quick-start.XXXXXX") || fail "could not create a staging directory" mkdir -p "$staging_dir/conf" @@ -410,8 +454,21 @@ SENDIUM_HTTP_USER='$http_user' SENDIUM_HTTP_PASSWORD='$http_password' SENDIUM_SMPP_USER='$smpp_user' SENDIUM_SMPP_PASSWORD='$smpp_password' +SENDIUM_DLR_STORAGE='postgresql' +SENDIUM_DLR_POSTGRESQL_ACTIVE='true' +SENDIUM_DLR_POSTGRESQL_JDBC_URL='$database_jdbc_url' +SENDIUM_DLR_POSTGRESQL_USERNAME='$database_username' +SENDIUM_DLR_POSTGRESQL_PASSWORD='$database_password' EOF +if [ "$external_database" != true ]; then + cat >> "$staging_dir/.sendium.env" < "$staging_dir/.gitignore" <<'EOF' .sendium.env conf/credentials.yml @@ -481,10 +538,43 @@ if [ "$upstream_enabled" = true ]; then printf 'upstream::default:\n' >> "$staging_dir/conf/routingTable.conf" fi -cat > "$staging_dir/compose.yml" < "$staging_dir/compose.yml" <<'EOF' services: +EOF + +if [ "$external_database" != true ]; then + cat >> "$staging_dir/compose.yml" <<'EOF' + postgres: + image: postgres:17-alpine + env_file: + - ./.sendium.env + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 2s + timeout: 3s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + +EOF +fi + +cat >> "$staging_dir/compose.yml" <> "$staging_dir/compose.yml" <<'EOF' + depends_on: + postgres: + condition: service_healthy +EOF +fi + +cat >> "$staging_dir/compose.yml" <<'EOF' environment: QUARKUS_LOG_FILE_ENABLE: "true" QUARKUS_LOG_CONSOLE_ENABLE: "true" @@ -501,6 +591,14 @@ services: - ./logs:/work/logs EOF +if [ "$external_database" != true ]; then + cat >> "$staging_dir/compose.yml" <<'EOF' + +volumes: + postgres-data: +EOF +fi + chmod 600 \ "$staging_dir/.sendium.env" \ "$staging_dir/.gitignore" \ @@ -565,7 +663,7 @@ fi if [ "$start_sendium" != true ]; then if [ "$force" = true ]; then printf '\nRecreate Sendium later to apply the regenerated configuration and credentials:\n' - printf ' docker compose -f "%s/compose.yml" --project-directory "%s" up -d --force-recreate\n' "$absolute_target" "$absolute_target" + printf ' docker compose -f "%s/compose.yml" --project-directory "%s" up -d --force-recreate --remove-orphans\n' "$absolute_target" "$absolute_target" else printf '\nStart Sendium later with:\n' printf ' docker compose -f "%s/compose.yml" --project-directory "%s" up -d\n' "$absolute_target" "$absolute_target" @@ -575,7 +673,7 @@ fi if [ "$force" = true ]; then printf '\nRecreating Sendium to apply the regenerated configuration and credentials...\n' - docker compose -f "$absolute_target/compose.yml" --project-directory "$absolute_target" up -d --force-recreate + docker compose -f "$absolute_target/compose.yml" --project-directory "$absolute_target" up -d --force-recreate --remove-orphans else printf '\nStarting Sendium...\n' docker compose -f "$absolute_target/compose.yml" --project-directory "$absolute_target" up -d @@ -583,7 +681,7 @@ fi attempt=0 while [ "$attempt" -lt 60 ]; do - if curl -fsS --connect-timeout 2 --max-time 3 http://127.0.0.1:8080/openapi.json >/dev/null 2>&1; then + if curl -fsS --connect-timeout 2 --max-time 3 http://127.0.0.1:8080/q/health/ready >/dev/null 2>&1; then printf '\nSendium is ready.\n' printf 'Swagger UI: http://127.0.0.1:8080/swagger-ui\n' printf '\nFollow live logs with:\n' diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index 1b78ced..08f4535 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -9,6 +9,9 @@ quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql quarkus.datasource.dlr.active=${SENDIUM_DLR_POSTGRESQL_ACTIVE:false} quarkus.datasource.dlr.devservices.enabled=false +quarkus.datasource.dlr.jdbc.url=${SENDIUM_DLR_POSTGRESQL_JDBC_URL:} +quarkus.datasource.dlr.username=${SENDIUM_DLR_POSTGRESQL_USERNAME:} +quarkus.datasource.dlr.password=${SENDIUM_DLR_POSTGRESQL_PASSWORD:} quarkus.datasource.dlr.jdbc.min-size=${SENDIUM_DLR_POSTGRESQL_POOL_MIN_SIZE:0} quarkus.datasource.dlr.jdbc.max-size=${SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE:10} quarkus.datasource.dlr.jdbc.acquisition-timeout=${SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT:5S} diff --git a/tests/quick-start-test.sh b/tests/quick-start-test.sh index 5044fda..2ce214a 100644 --- a/tests/quick-start-test.sh +++ b/tests/quick-start-test.sh @@ -10,9 +10,13 @@ integration_dir='' integration_project='' tests_run=0 +unset SENDIUM_DLR_POSTGRESQL_JDBC_URL +unset SENDIUM_DLR_POSTGRESQL_USERNAME +unset SENDIUM_DLR_POSTGRESQL_PASSWORD + cleanup() { if [ -n "$integration_dir" ] && [ -f "$integration_dir/compose.yml" ] && command -v docker >/dev/null 2>&1; then - docker compose -p "$integration_project" -f "$integration_dir/compose.yml" --project-directory "$integration_dir" down >/dev/null 2>&1 || true + docker compose -p "$integration_project" -f "$integration_dir/compose.yml" --project-directory "$integration_dir" down --volumes --remove-orphans >/dev/null 2>&1 || true fi rm -rf "$test_root" } @@ -95,6 +99,11 @@ assert_file "$local_dir/conf/smsg.properties" assert_file "$local_dir/conf/routingTable.conf" assert_contains '127.0.0.1:8080:8080' "$local_dir/compose.yml" assert_contains '127.0.0.1:27777:27777' "$local_dir/compose.yml" +assert_contains 'image: postgres:17-alpine' "$local_dir/compose.yml" +assert_contains 'pg_isready' "$local_dir/compose.yml" +assert_contains 'condition: service_healthy' "$local_dir/compose.yml" +assert_contains 'postgres-data:/var/lib/postgresql/data' "$local_dir/compose.yml" +assert_contains 'postgres-data:' "$local_dir/compose.yml" assert_not_contains 'outSms.instance.upstream' "$local_dir/conf/smsg.properties" assert_not_contains 'upstream::default:' "$local_dir/conf/routingTable.conf" assert_equals 600 "$(file_mode "$local_dir/.sendium.env")" ".sendium.env mode" @@ -104,10 +113,18 @@ http_user=$(sed -n "s/^SENDIUM_HTTP_USER='\([^']*\)'$/\1/p" "$local_dir/.sendium http_password=$(sed -n "s/^SENDIUM_HTTP_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$local_dir/.sendium.env") smpp_user=$(sed -n "s/^SENDIUM_SMPP_USER='\([^']*\)'$/\1/p" "$local_dir/.sendium.env") smpp_password=$(sed -n "s/^SENDIUM_SMPP_PASSWORD='\([A-Za-z0-9][A-Za-z0-9]*\)'$/\1/p" "$local_dir/.sendium.env") +database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$local_dir/.sendium.env") +postgres_password=$(sed -n "s/^POSTGRES_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$local_dir/.sendium.env") assert_equals 'sendium-http' "$http_user" "HTTP environment username" assert_equals 'sendium-smpp' "$smpp_user" "SMPP environment username" assert_equals 48 "${#http_password}" "HTTP password length" assert_equals 8 "${#smpp_password}" "SMPP password length" +assert_equals 64 "${#database_password}" "PostgreSQL password length" +assert_equals "$database_password" "$postgres_password" "PostgreSQL container password" +assert_contains "SENDIUM_DLR_STORAGE='postgresql'" "$local_dir/.sendium.env" +assert_contains "SENDIUM_DLR_POSTGRESQL_ACTIVE='true'" "$local_dir/.sendium.env" +assert_contains "SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://postgres:5432/sendium'" "$local_dir/.sendium.env" +assert_not_contains "$database_password" "$local_dir/compose.yml" assert_equals "$http_user" "$(credential_value HTTP systemId "$local_dir/conf/credentials.yml")" "HTTP credential username" assert_equals "$http_password" "$(credential_value HTTP password "$local_dir/conf/credentials.yml")" "HTTP credential password" assert_equals "$smpp_user" "$(credential_value SMPP systemId "$local_dir/conf/credentials.yml")" "SMPP credential username" @@ -121,16 +138,39 @@ pass "non-empty target protection" printf 'keep me\n' > "$local_dir/user-file.txt" old_http_password=$http_password +old_database_password=$database_password sh "$quick_start" --directory "$local_dir" --provider local --force --no-start > "$test_root/force.out" 2>&1 assert_file "$local_dir/user-file.txt" assert_contains 'keep me' "$local_dir/user-file.txt" assert_contains 'up -d --force-recreate' "$test_root/force.out" http_password=$(sed -n "s/^SENDIUM_HTTP_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$local_dir/.sendium.env") +database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$local_dir/.sendium.env") [ -n "$http_password" ] || fail "forced regeneration did not produce an HTTP password" assert_equals 48 "${#http_password}" "regenerated HTTP password length" [ "$old_http_password" != "$http_password" ] || fail "forced regeneration did not rotate the HTTP password" +assert_equals "$old_database_password" "$database_password" "preserved PostgreSQL password" pass "explicit regeneration preserves unrelated files" +external_dir="$test_root/external-postgresql" +SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium?sslmode=require' \ +SENDIUM_DLR_POSTGRESQL_USERNAME='external-user' \ +SENDIUM_DLR_POSTGRESQL_PASSWORD='external-password' \ + sh "$quick_start" --directory "$external_dir" --provider local --no-start > "$test_root/external-postgresql.out" 2>&1 +assert_contains "SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium?sslmode=require'" "$external_dir/.sendium.env" +assert_contains "SENDIUM_DLR_POSTGRESQL_USERNAME='external-user'" "$external_dir/.sendium.env" +assert_contains "SENDIUM_DLR_POSTGRESQL_PASSWORD='external-password'" "$external_dir/.sendium.env" +assert_not_contains 'image: postgres:17-alpine' "$external_dir/compose.yml" +assert_not_contains 'condition: service_healthy' "$external_dir/compose.yml" +assert_not_contains 'postgres-data:' "$external_dir/compose.yml" +assert_not_contains 'external-password' "$external_dir/compose.yml" +pass "external PostgreSQL configuration" + +expect_failure "partial external PostgreSQL configuration" "$test_root/partial-postgresql.out" \ + env SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium' \ + sh "$quick_start" --directory "$test_root/partial-postgresql" --provider local --no-start +assert_contains 'external PostgreSQL requires JDBC URL, username, and password' "$test_root/partial-postgresql.out" +pass "incomplete external PostgreSQL configuration" + regenerated_dir="$test_root/regenerated-upstream" sh "$quick_start" --directory "$regenerated_dir" --provider local --no-start > "$test_root/regenerated-local.out" 2>&1 SENDIUM_UPSTREAM_USERNAME='prosms-user' \ @@ -206,6 +246,7 @@ if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; docker compose -f "$local_dir/compose.yml" --project-directory "$local_dir" config --quiet docker compose -f "$prosms_dir/compose.yml" --project-directory "$prosms_dir" config --quiet docker compose -f "$custom_dir/compose.yml" --project-directory "$custom_dir" config --quiet + docker compose -f "$external_dir/compose.yml" --project-directory "$external_dir" config --quiet pass "Docker Compose parsing" else printf 'skip - Docker Compose parsing (Docker Compose unavailable)\n' @@ -222,6 +263,7 @@ if [ -n "${SENDIUM_TEST_IMAGE-}" ]; then assert_contains 'Follow live logs with:' "$test_root/integration.out" integration_http_user=$(sed -n "s/^SENDIUM_HTTP_USER='\([^']*\)'$/\1/p" "$integration_dir/.sendium.env") integration_http_password=$(sed -n "s/^SENDIUM_HTTP_PASSWORD='\([^']*\)'$/\1/p" "$integration_dir/.sendium.env") + integration_database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([^']*\)'$/\1/p" "$integration_dir/.sendium.env") http_status=$(curl -sS -o "$test_root/sendsms.out" -w '%{http_code}' -G http://127.0.0.1:8080/sendsms \ --data-urlencode "username=$integration_http_user" \ --data-urlencode "password=$integration_http_password" \ @@ -243,8 +285,10 @@ if [ -n "${SENDIUM_TEST_IMAGE-}" ]; then --image "$SENDIUM_TEST_IMAGE" \ --force > "$test_root/integration-force.out" 2>&1 recreated_container_id=$(docker compose -p "$integration_project" -f "$integration_dir/compose.yml" --project-directory "$integration_dir" ps -q sendium) + recreated_database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([^']*\)'$/\1/p" "$integration_dir/.sendium.env") [ -n "$recreated_container_id" ] || fail "forced regeneration did not leave a running container" [ "$original_container_id" != "$recreated_container_id" ] || fail "forced regeneration did not recreate the container" + assert_equals "$integration_database_password" "$recreated_database_password" "integration PostgreSQL password" assert_contains 'Recreating Sendium' "$test_root/integration-force.out" attempt=0 while ! grep -F 'Starting: (smppclient.upstream)' "$integration_dir/logs/smsg.log" >/dev/null 2>&1 && [ "$attempt" -lt 10 ]; do From b5ee6bdf12a0ea9b3eb1e3fb303339507d22e5b2 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 12:04:24 +0300 Subject: [PATCH 10/20] test(dlr): verify PostgreSQL container restarts --- .github/workflows/native_e2e.yml | 20 +- .github/workflows/run_tests.yml | 30 +- .../src/test/java/utils/NativeE2eSmoke.java | 274 ++++++++++++------ 3 files changed, 235 insertions(+), 89 deletions(-) diff --git a/.github/workflows/native_e2e.yml b/.github/workflows/native_e2e.yml index 9985a9b..427b3d1 100644 --- a/.github/workflows/native_e2e.yml +++ b/.github/workflows/native_e2e.yml @@ -9,6 +9,7 @@ on: - ".mvn/**" - "mvnw" - "mvnw.cmd" + - ".github/workflows/run_tests.yml" - ".github/workflows/native_e2e.yml" workflow_dispatch: @@ -24,6 +25,20 @@ jobs: if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'release-please--') runs-on: ubuntu-latest timeout-minutes: 60 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_DB: sendium + POSTGRES_USER: sendium + POSTGRES_PASSWORD: sendium-test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U sendium -d sendium" + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - name: Checkout code uses: actions/checkout@v7 @@ -46,7 +61,10 @@ jobs: - name: Run native E2E smoke tests env: - SENDIUM_NATIVE_IMAGE: sendium:native-e2e + SENDIUM_E2E_IMAGE: sendium:native-e2e + SENDIUM_DLR_POSTGRESQL_JDBC_URL: jdbc:postgresql://host.docker.internal:5432/sendium + SENDIUM_DLR_POSTGRESQL_USERNAME: sendium + SENDIUM_DLR_POSTGRESQL_PASSWORD: sendium-test run: | ./mvnw -B -pl sendium-core -DskipTests test-compile org.codehaus.mojo:exec-maven-plugin:3.5.1:java \ -Dexec.classpathScope=test \ diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0ece6a4..fdec117 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -26,6 +26,20 @@ jobs: permissions: contents: read runs-on: ubuntu-latest + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_DB: sendium + POSTGRES_USER: sendium + POSTGRES_PASSWORD: sendium-test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U sendium -d sendium" + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - name: Checkout code uses: actions/checkout@v7 @@ -40,19 +54,29 @@ jobs: run: | export QUARKUS_LOG_LEVEL=ERROR chmod +x ./mvnw - ./mvnw verify + ./mvnw verify -Ppostgresql-tests - name: Build quick-start test image run: docker build -t sendium:quick-start-test -f sendium-app/src/main/docker/Dockerfile.jvm sendium-app - name: Test generated container startup env: SENDIUM_TEST_IMAGE: sendium:quick-start-test run: sh tests/quick-start-test.sh + - name: Test PostgreSQL restart behavior on JVM + env: + SENDIUM_E2E_IMAGE: sendium:quick-start-test + SENDIUM_DLR_POSTGRESQL_JDBC_URL: jdbc:postgresql://host.docker.internal:5432/sendium + SENDIUM_DLR_POSTGRESQL_USERNAME: sendium + SENDIUM_DLR_POSTGRESQL_PASSWORD: sendium-test + run: | + ./mvnw -B -pl sendium-core -DskipTests test-compile org.codehaus.mojo:exec-maven-plugin:3.5.1:java \ + -Dexec.classpathScope=test \ + -Dexec.mainClass=utils.NativeE2eSmoke - id: result name: Sendium Test Result if: always() run: | - echo "result=${{job.status}}" >> $GITHUB_ENV - echo "result=${{job.status}}" >> $GITHUB_OUTPUT + echo "result=${{job.status}}" >> "$GITHUB_ENV" + echo "result=${{job.status}}" >> "$GITHUB_OUTPUT" - name: Cancel current workflow run if: failure() uses: actions/github-script@v9 diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index db9737c..e4ad9a3 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -25,7 +25,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.sun.net.httpserver.HttpServer; import io.netty.channel.nio.NioEventLoopGroup; -import org.h2.mvstore.MVStore; import java.io.IOException; import java.net.HttpURLConnection; @@ -52,14 +51,21 @@ import java.util.concurrent.atomic.AtomicInteger; public class NativeE2eSmoke { - private static final String IMAGE = System.getenv().getOrDefault("SENDIUM_NATIVE_IMAGE", "sendium:native-e2e"); + private static final String IMAGE = System.getenv().getOrDefault("SENDIUM_E2E_IMAGE", "sendium:native-e2e"); + private static final String POSTGRESQL_JDBC_URL = System.getenv().getOrDefault( + "SENDIUM_DLR_POSTGRESQL_JDBC_URL", "jdbc:postgresql://host.docker.internal:5432/sendium"); + private static final String POSTGRESQL_USERNAME = System.getenv().getOrDefault( + "SENDIUM_DLR_POSTGRESQL_USERNAME", "sendium"); + private static final String POSTGRESQL_PASSWORD = System.getenv().getOrDefault( + "SENDIUM_DLR_POSTGRESQL_PASSWORD", "sendium-test"); private static final int SENDIUM_HTTP_PORT = 18080; private static final int SENDIUM_SMPP_PORT = 27777; private static final int UPSTREAM_SMPP_PORT = 27779; private static final Duration TIMEOUT = Duration.ofSeconds(90); + private static final Duration NO_DELIVERY_TIMEOUT = Duration.ofSeconds(3); public static void main(String[] args) throws Exception { - String containerName = "sendium-native-e2e-" + UUID.randomUUID().toString().substring(0, 8); + String containerName = "sendium-e2e-" + UUID.randomUUID().toString().substring(0, 8); Process container = null; try (UpstreamSmppServer upstream = new UpstreamSmppServer(UPSTREAM_SMPP_PORT); @@ -67,17 +73,16 @@ public static void main(String[] args) throws Exception { upstream.start(); callbackServer.start(); - Path workDir = Files.createTempDirectory("sendium-native-e2e-"); + Path workDir = Files.createTempDirectory("sendium-e2e-"); writeRuntimeConfig(workDir); container = startSendiumContainer(containerName, workDir); - waitForPort("localhost", SENDIUM_HTTP_PORT, TIMEOUT); + waitForPostgresqlReadiness(); waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); - require(upstream.awaitSessionBound(), "Sendium native container did not bind to the upstream SMPP server"); - Thread.sleep(3_000); + require(upstream.awaitSessionBound(), "Sendium container did not bind to the upstream SMPP server"); container = verifyUnpushedDlrSurvivesRestart(containerName, workDir, upstream); - verifySmppSubmitGetsDeliverSm(upstream, 2); - verifyHttpSubmitGetsDlrCallback(upstream, callbackServer, 3); + container = verifyHttpCorrelationSurvivesRestart(containerName, workDir, upstream, callbackServer, 2); + verifySmppSubmitGetsDeliverSm(upstream, 3); } catch (Throwable t) { printDockerLogs(containerName); throw t; @@ -92,7 +97,7 @@ public static void main(String[] args) throws Exception { private static void verifySmppSubmitGetsDeliverSm(UpstreamSmppServer upstream, int expectedSubmitCount) throws Exception { try (DownstreamSmppClient client = new DownstreamSmppClient()) { client.start(); - SubmitSmResp response = client.sendSms("smpp-sender", "306900000001", "native smpp e2e"); + SubmitSmResp response = client.sendSms("smpp-sender", "306900000001", "container smpp e2e"); require(response.getCommandStatus() == SmppConstants.STATUS_OK, "SMPP submit_sm_resp status was " + response.getCommandStatus()); require(response.getMessageId() != null && !response.getMessageId().isBlank(), @@ -102,63 +107,87 @@ private static void verifySmppSubmitGetsDeliverSm(UpstreamSmppServer upstream, i require(deliverSm != null, "Downstream SMPP client did not receive deliver_sm"); String body = new String(deliverSm.getShortMessage(), StandardCharsets.UTF_8); require(body.contains("DELIVRD"), "Downstream deliver_sm was not delivered: " + body); + require(body.contains("id:" + response.getMessageId()), + "Downstream deliver_sm did not contain gateway id " + response.getMessageId() + ": " + body); Thread.sleep(500); } } - private static void verifyHttpSubmitGetsDlrCallback(UpstreamSmppServer upstream, CallbackServer callbackServer, - int expectedSubmitCount) throws Exception { + private static Process verifyHttpCorrelationSurvivesRestart(String containerName, Path workDir, + UpstreamSmppServer upstream, + CallbackServer callbackServer, + int expectedSubmitCount) throws Exception { + upstream.setAutomaticDelivery(false); String dlrUrl = "http://host.docker.internal:" + callbackServer.port() + "/dlr?status=%d&id=%s"; String query = "username=http-user" + "&password=http-pass" + "&from=http-sender" + "&to=306900000002" - + "&text=" + URLEncoder.encode("native http e2e", StandardCharsets.UTF_8) + + "&text=" + URLEncoder.encode("container restart http e2e", StandardCharsets.UTF_8) + "&dlr-url=" + URLEncoder.encode(dlrUrl, StandardCharsets.UTF_8); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create("http://localhost:" + SENDIUM_HTTP_PORT + "/sendsms?" + query)) - .timeout(Duration.ofSeconds(10)) - .GET() - .build(); - HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); - require(response.statusCode() == HttpURLConnection.HTTP_ACCEPTED, - "HTTP /sendsms returned " + response.statusCode() + ": " + response.body()); - - String gatewayId = response.body().trim(); - require(!gatewayId.isBlank(), "HTTP /sendsms did not return a gateway message id"); - require(upstream.awaitSubmitCount(expectedSubmitCount), "Upstream SMPP server did not receive the HTTP-originated message"); - - String callbackQuery = callbackServer.awaitCallback(); - require(callbackQuery != null, "DLR callback URL was not called"); - require(callbackQuery.contains("status=1"), "DLR callback did not contain delivered status: " + callbackQuery); - require(callbackQuery.contains("id=" + gatewayId), "DLR callback did not contain gateway id " + gatewayId + ": " + callbackQuery); + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + SENDIUM_HTTP_PORT + "/sendsms?" + query)) + .timeout(Duration.ofSeconds(10)) + .GET() + .build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + require(response.statusCode() == HttpURLConnection.HTTP_ACCEPTED, + "HTTP /sendsms returned " + response.statusCode() + ": " + response.body()); + + String gatewayId = response.body().trim(); + require(!gatewayId.isBlank(), "HTTP /sendsms did not return a gateway message id"); + require(upstream.awaitSubmitCount(expectedSubmitCount), + "Upstream SMPP server did not receive the HTTP-originated message"); + awaitSuccessfulStorageOperation("link_operator"); + + int boundSessionsBeforeRestart = upstream.boundSessionCount(); + stopContainer(containerName); + Process restartedContainer = startSendiumContainer(containerName, workDir); + waitForPostgresqlReadiness(); + waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); + require(upstream.awaitSessionBoundAfter(boundSessionsBeforeRestart), + "Sendium container did not rebind before the HTTP DLR replay"); + + upstream.sendDeliveryReceipt(expectedSubmitCount); + String callbackQuery = callbackServer.awaitCallback(); + require(callbackQuery != null, "DLR callback URL was not called after restart"); + require(callbackQuery.contains("status=1"), "DLR callback did not contain delivered status: " + callbackQuery); + require(callbackQuery.contains("id=" + gatewayId), + "DLR callback did not contain gateway id " + gatewayId + ": " + callbackQuery); + return restartedContainer; + } finally { + upstream.setAutomaticDelivery(true); + } } private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Path workDir, UpstreamSmppServer upstream) throws Exception { - upstream.setDeliveryReceiptDelayMillis(2_500); + upstream.setAutomaticDelivery(false); String gatewayId; try (DownstreamSmppClient client = new DownstreamSmppClient()) { client.start(); - SubmitSmResp response = client.sendSms("smpp-sender", "306900000003", "native restart dlr e2e"); + SubmitSmResp response = client.sendSms("smpp-sender", "306900000003", "container restart dlr e2e"); require(response.getCommandStatus() == SmppConstants.STATUS_OK, "SMPP restart submit_sm_resp status was " + response.getCommandStatus()); gatewayId = response.getMessageId(); require(gatewayId != null && !gatewayId.isBlank(), "SMPP restart submit_sm_resp did not contain a message id"); require(upstream.awaitSubmitCount(1), "Upstream SMPP server did not receive the restart test message"); - } finally { - upstream.setDeliveryReceiptDelayMillis(1_000); + awaitSuccessfulStorageOperation("link_operator"); } - Thread.sleep(6_000); + Thread.sleep(500); + upstream.sendDeliveryReceipt(1); + awaitSuccessfulStorageOperation("save_unpushed"); + upstream.setAutomaticDelivery(true); + int boundSessionsBeforeRestart = upstream.boundSessionCount(); stopContainer(containerName); - assertUnpushedDlrPersisted(workDir, gatewayId, "before restart"); Process replayContainer = startSendiumContainer(containerName, workDir); - waitForPort("localhost", SENDIUM_HTTP_PORT, TIMEOUT); + waitForPostgresqlReadiness(); waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); - require(upstream.awaitSessionBoundCount(2), "Sendium native container did not rebind to upstream after restart"); + require(upstream.awaitSessionBoundAfter(boundSessionsBeforeRestart), + "Sendium container did not rebind to upstream after restart"); try (DownstreamSmppClient reconnectedClient = new DownstreamSmppClient()) { reconnectedClient.start(); @@ -166,15 +195,25 @@ private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Pa require(deliverSm != null, "Reconnected downstream SMPP client did not receive persisted unpushed DLR"); String body = new String(deliverSm.getShortMessage(), StandardCharsets.UTF_8); require(body.contains("DELIVRD"), "Persisted unpushed DLR was not delivered: " + body); + require(body.contains("id:" + gatewayId), + "Persisted unpushed DLR did not contain gateway id " + gatewayId + ": " + body); + awaitSuccessfulStorageOperation("remove_unpushed"); } + boundSessionsBeforeRestart = upstream.boundSessionCount(); stopContainer(containerName); replayContainer.destroyForcibly(); - assertUnpushedDlrRemoved(workDir, gatewayId, "after replay"); Process container = startSendiumContainer(containerName, workDir); - waitForPort("localhost", SENDIUM_HTTP_PORT, TIMEOUT); + waitForPostgresqlReadiness(); waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); - require(upstream.awaitSessionBoundCount(3), "Sendium native container did not rebind to upstream after replay check"); + require(upstream.awaitSessionBoundAfter(boundSessionsBeforeRestart), + "Sendium container did not rebind to upstream after replay check"); + try (DownstreamSmppClient client = new DownstreamSmppClient()) { + client.start(); + awaitSuccessfulStorageOperation("claim_unpushed"); + require(client.awaitDeliverSm(NO_DELIVERY_TIMEOUT) == null, + "Replayed unpushed DLR was delivered more than once after restart"); + } return container; } @@ -190,6 +229,11 @@ private static Process startSendiumContainer(String containerName, Path workDir) "-v", workDir.resolve("logs").toAbsolutePath() + ":/work/logs", "-e", "QUARKUS_LOG_LEVEL=INFO", "-e", "LOG_LEVEL=INFO", + "-e", "SENDIUM_DLR_STORAGE=postgresql", + "-e", "SENDIUM_DLR_POSTGRESQL_ACTIVE=true", + "-e", "SENDIUM_DLR_POSTGRESQL_JDBC_URL=" + POSTGRESQL_JDBC_URL, + "-e", "SENDIUM_DLR_POSTGRESQL_USERNAME=" + POSTGRESQL_USERNAME, + "-e", "SENDIUM_DLR_POSTGRESQL_PASSWORD=" + POSTGRESQL_PASSWORD, IMAGE ); Process process = run(command, true); @@ -270,6 +314,57 @@ private static void waitForPort(String host, int port, Duration timeout) throws throw new IllegalStateException("Timed out waiting for " + host + ':' + port); } + private static void waitForPostgresqlReadiness() throws Exception { + long deadline = System.nanoTime() + TIMEOUT.toNanos(); + while (System.nanoTime() < deadline) { + try { + HttpResponse response = get("/q/health/ready"); + String body = response.body().replaceAll("\\s", ""); + if (response.statusCode() == HttpURLConnection.HTTP_OK + && body.contains("\"name\":\"sendium-dlr-storage\"") + && body.contains("\"backend\":\"postgresql\"")) { + return; + } + } catch (IOException ignored) { + } + Thread.sleep(500); + } + throw new IllegalStateException("Timed out waiting for PostgreSQL-backed Sendium readiness"); + } + + private static void awaitSuccessfulStorageOperation(String operation) throws Exception { + long deadline = System.nanoTime() + TIMEOUT.toNanos(); + while (System.nanoTime() < deadline) { + HttpResponse response = get("/q/metrics"); + if (response.statusCode() == HttpURLConnection.HTTP_OK + && hasSuccessfulStorageOperation(response.body(), operation)) { + return; + } + Thread.sleep(250); + } + throw new IllegalStateException("Timed out waiting for successful DLR storage operation: " + operation); + } + + private static boolean hasSuccessfulStorageOperation(String metrics, String operation) { + return metrics.lines() + .filter(line -> line.startsWith("sendium_dlr_storage_operation_seconds_count")) + .filter(line -> line.contains("backend=\"postgresql\"")) + .filter(line -> line.contains("operation=\"" + operation + "\"")) + .filter(line -> line.contains("outcome=\"success\"")) + .map(line -> line.substring(line.lastIndexOf(' ') + 1)) + .mapToDouble(Double::parseDouble) + .anyMatch(count -> count >= 1.0); + } + + private static HttpResponse get(String path) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + SENDIUM_HTTP_PORT + path)) + .timeout(Duration.ofSeconds(5)) + .GET() + .build(); + return HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + } + private static Process run(List command, boolean inheritOutput) throws IOException { ProcessBuilder builder = new ProcessBuilder(command); if (inheritOutput) { @@ -298,31 +393,6 @@ private static void require(boolean condition, String message) { } } - private static void assertUnpushedDlrPersisted(Path workDir, String gatewayId, String phase) { - requireUnpushedDlrPresence(workDir, gatewayId, true, phase); - } - - private static void assertUnpushedDlrRemoved(Path workDir, String gatewayId, String phase) { - requireUnpushedDlrPresence(workDir, gatewayId, false, phase); - } - - private static void requireUnpushedDlrPresence(Path workDir, String gatewayId, boolean expectedPresent, String phase) { - Path dbPath = workDir.resolve("data").resolve("dlr-mvstore.db"); - require(Files.exists(dbPath), "DLR MVStore does not exist " + phase + ": " + dbPath); - try (MVStore store = new MVStore.Builder().fileName(dbPath.toAbsolutePath().toString()).readOnly().open()) { - Map dlrStore = store.openMap("unpushedDlrStore"); - Map dlrIndex = store.openMap("unpushedDlrIndex"); - boolean present = dlrStore.values().stream().anyMatch(value -> value.contains("\"serial\":\"" + gatewayId + "\"")); - require(present == expectedPresent, - "Unexpected unpushed DLR presence " + phase + " for gatewayId " + gatewayId - + ": " + present + " expected " + expectedPresent - + " storeSize=" + dlrStore.size() + " indexSize=" + dlrIndex.size()); - if (expectedPresent) { - require(dlrIndex.containsKey("smpp-user"), "Unpushed DLR index did not contain smpp-user " + phase); - } - } - } - private static final class CallbackServer implements AutoCloseable { private final CountDownLatch latch = new CountDownLatch(1); private final List queries = Collections.synchronizedList(new ArrayList<>()); @@ -366,10 +436,10 @@ private static final class UpstreamSmppServer implements AutoCloseable { private final AtomicInteger receivedSubmits = new AtomicInteger(); private final AtomicInteger boundSessions = new AtomicInteger(); private final CountDownLatch sessionBound = new CountDownLatch(1); - private final CountDownLatch firstTwoSubmits = new CountDownLatch(2); private final Set sessions = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Map pendingDeliveryReceipts = new ConcurrentHashMap<>(); private final DefaultSmppServer server; - private volatile long deliveryReceiptDelayMillis = 1_000; + private volatile boolean automaticDelivery = true; private UpstreamSmppServer(int port) { this.port = port; @@ -383,12 +453,18 @@ public PduResponse firePduRequestReceived(PduRequest pduRequest) { if (!(pduRequest instanceof SubmitSm submitSm)) { return pduRequest.createResponse(); } + require(submitSm.getRegisteredDelivery() == SmppConstants.REGISTERED_DELIVERY_SMSC_RECEIPT_REQUESTED, + "Sendium did not request an upstream delivery receipt"); - String messageId = "e2e-" + receivedSubmits.incrementAndGet(); - firstTwoSubmits.countDown(); + int submitNumber = receivedSubmits.incrementAndGet(); + String messageId = "e2e-" + submitNumber; + pendingDeliveryReceipts.put(submitNumber, + new PendingDeliveryReceipt(submitSm.getDestAddress(), submitSm.getSourceAddress(), messageId)); SubmitSmResp response = (SubmitSmResp) pduRequest.createResponse(); response.setMessageId(messageId); - Thread.ofVirtual().start(() -> sendDeliveryReceipt(submitSm, messageId)); + if (automaticDelivery) { + Thread.ofVirtual().start(() -> sendDeliveryReceiptAfterDelay(submitNumber)); + } return response; } }; @@ -403,10 +479,10 @@ public void sessionBindRequested(Long sessionId, SmppSessionConfiguration sessio @Override public void sessionCreated(Long sessionId, SmppServerSession session, BaseBindResp preparedBindResponse) throws SmppProcessingException { + session.serverReady(sessionHandler); sessions.add(session); boundSessions.incrementAndGet(); sessionBound.countDown(); - session.serverReady(sessionHandler); } @Override @@ -437,7 +513,7 @@ private boolean awaitSubmitCount(int expected) throws InterruptedException { if (receivedSubmits.get() >= expected) { return true; } - firstTwoSubmits.await(500, TimeUnit.MILLISECONDS); + Thread.sleep(100); } return false; } @@ -446,10 +522,14 @@ private boolean awaitSessionBound() throws InterruptedException { return sessionBound.await(TIMEOUT.toSeconds(), TimeUnit.SECONDS); } - private boolean awaitSessionBoundCount(int expected) throws InterruptedException { + private int boundSessionCount() { + return boundSessions.get(); + } + + private boolean awaitSessionBoundAfter(int previousCount) throws InterruptedException { long deadline = System.nanoTime() + TIMEOUT.toNanos(); while (System.nanoTime() < deadline) { - if (boundSessions.get() >= expected) { + if (boundSessions.get() > previousCount) { return true; } Thread.sleep(500); @@ -457,32 +537,52 @@ private boolean awaitSessionBoundCount(int expected) throws InterruptedException return false; } - private void setDeliveryReceiptDelayMillis(long deliveryReceiptDelayMillis) { - this.deliveryReceiptDelayMillis = deliveryReceiptDelayMillis; + private void setAutomaticDelivery(boolean automaticDelivery) { + this.automaticDelivery = automaticDelivery; + } + + private void sendDeliveryReceiptAfterDelay(int submitNumber) { + try { + Thread.sleep(1_000); + sendDeliveryReceipt(submitNumber); + } catch (Exception e) { + throw new IllegalStateException("Failed to send delayed upstream delivery receipt on port " + port, e); + } } - private void sendDeliveryReceipt(SubmitSm submitSm, String messageId) { + private void sendDeliveryReceipt(int submitNumber) { + PendingDeliveryReceipt pendingReceipt = pendingDeliveryReceipts.get(submitNumber); + require(pendingReceipt != null, "No pending delivery receipt for upstream submit " + submitNumber); try { - Thread.sleep(deliveryReceiptDelayMillis); DeliverSm deliverSm = new DeliverSm(); deliverSm.setDataCoding(SmppConstants.DATA_CODING_DEFAULT); deliverSm.setEsmClass(SmppConstants.ESM_CLASS_MT_SMSC_DELIVERY_RECEIPT); - deliverSm.setSourceAddress(submitSm.getDestAddress()); - deliverSm.setDestAddress(submitSm.getSourceAddress()); - String dlr = "id:" + messageId + deliverSm.setSourceAddress(pendingReceipt.sourceAddress()); + deliverSm.setDestAddress(pendingReceipt.destinationAddress()); + String dlr = "id:" + pendingReceipt.messageId() + " sub:001 dlvrd:001 submit date:2605191200 done date:2605191200 stat:DELIVRD err:000 text:e2e"; deliverSm.setShortMessage(dlr.getBytes(StandardCharsets.UTF_8)); for (SmppSession session : sessions) { if (session.isBound() && session.getBindType() != SmppBindType.TRANSMITTER) { - session.sendRequestPdu(deliverSm, 30_000, true); + var future = session.sendRequestPdu(deliverSm, 30_000, true); + require(future.await(30_000), "Timed out waiting for Sendium to acknowledge upstream delivery receipt"); + require(future.isSuccess(), "Sendium failed to acknowledge upstream delivery receipt: " + future.getCause()); + PduResponse response = future.getResponse(); + require(response.getCommandStatus() == SmppConstants.STATUS_OK, + "Sendium rejected upstream delivery receipt with status " + response.getCommandStatus()); + pendingDeliveryReceipts.remove(submitNumber, pendingReceipt); return; } } + throw new IllegalStateException("No bound Sendium session for upstream delivery receipt"); } catch (Exception e) { throw new IllegalStateException("Failed to send upstream delivery receipt on port " + port, e); } } + private record PendingDeliveryReceipt(Address sourceAddress, Address destinationAddress, String messageId) { + } + @Override public void close() { server.destroy(0, 100); @@ -559,7 +659,11 @@ private SubmitSmResp sendSms(String from, String to, String text) throws SmppInv } private DeliverSm awaitDeliverSm() throws InterruptedException { - if (!deliverSmLatch.await(TIMEOUT.toSeconds(), TimeUnit.SECONDS)) { + return awaitDeliverSm(TIMEOUT); + } + + private DeliverSm awaitDeliverSm(Duration timeout) throws InterruptedException { + if (!deliverSmLatch.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) { return null; } return deliverSm; From 062a30881605e9fd4ac33a877b936b355f3b88df Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 14:27:37 +0300 Subject: [PATCH 11/20] docs(dlr): document PostgreSQL operations --- README.md | 4 +- docs/01-architecture.md | 20 ++-- docs/02-docker-deployment.md | 14 ++- docs/07-webhooks.md | 4 +- docs/09-configuration-reference.md | 19 ++++ docs/13-dlr-persistence.md | 160 +++++++++++++++++++++++++++++ docs/DocumentationMap.md | 8 +- 7 files changed, 213 insertions(+), 16 deletions(-) create mode 100644 docs/13-dlr-persistence.md diff --git a/README.md b/README.md index d912517..51b790e 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Download and run the setup script: curl -fsSLo quick-start.sh https://raw.githubusercontent.com/cytechmobile/sendium/main/quick-start.sh && sh quick-start.sh ``` -The script creates a `sendium/` runtime directory, generates random HTTP and SMPP credentials, writes Docker Compose and all required configuration files, starts Sendium, and waits for the HTTP API. +The script creates a `sendium/` runtime directory, generates random HTTP, SMPP, and database credentials, writes Docker Compose and all required configuration files, starts PostgreSQL and Sendium, and waits for PostgreSQL-backed readiness. It asks you to choose one upstream option: @@ -72,7 +72,7 @@ It asks you to choose one upstream option: 2. **Existing SMPP provider:** Enter your provider host, port, credentials, and TLS choice. Quick Start uses a transceiver connection. 3. **Local setup only:** Starts Sendium's local HTTP and SMPP interfaces without an outbound provider. You can explore the API, but messages cannot be delivered until an upstream route is configured. -HTTP and SMPP ports are bound to `127.0.0.1` by default. Use the [Docker deployment guide](docs/02-docker-deployment.md) for generated-file details, manual setup, native images, and non-local deployments. +HTTP and SMPP ports are bound to `127.0.0.1` by default. Use the [Docker deployment guide](docs/02-docker-deployment.md) for generated-file details, manual setup, native images, and non-local deployments. Review [DLR persistence](docs/13-dlr-persistence.md) before changing database backends or volume handling. When startup completes, the script prints the Swagger URL and an exact command for following live logs diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 5dbb179..6f413c8 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -23,7 +23,7 @@ flowchart LR workerQueues[Worker queues] smppClients["SMPP client workers
smppclient instances"] carriers["Upstream SMSCs
carriers or SMPP providers"] - dlrStore["DLR correlation store
InMemoryDlrService"] + dlrStore["DLR storage
PostgreSQL or MVStore"] webhooks["HTTP webhooks
DLR and MO callbacks"] config["Runtime config files
credentials.yml
smsg.properties
routingTable.conf"] @@ -87,7 +87,7 @@ sequenceDiagram participant Client as HTTP client participant API as KannelResource participant Creds as CredentialFileWatcher - participant DLR as InMemoryDlrService + participant DLR as DlrStorage participant Queue as Router queue participant Router as StandardRoutingManager participant Worker as SmppClientWorker @@ -116,6 +116,8 @@ sequenceDiagram participant Server as SmppServerWorker participant Auth as BasicSmppAuthenticationProvider participant Submit as BasicSubmitSmProcessor + participant Store as SMPP message store + participant DLR as DlrStorage participant Queue as Router queue participant Router as StandardRoutingManager @@ -124,7 +126,12 @@ sequenceDiagram Auth-->>Server: Bind accepted or rejected Client->>Server: submit_sm Server->>Submit: Validate and convert PDU - Submit->>Queue: Enqueue StandardMessage + Submit-->>Server: Valid submission event + Server->>Store: Add event to persistence batch + Store->>DLR: Persist initial DLR state + DLR-->>Store: Commit successful + Store-->>Server: Handle persisted event + Server->>Queue: Enqueue StandardMessage Server-->>Client: submit_sm_resp Router->>Queue: Dequeue and route message ``` @@ -151,7 +158,7 @@ sequenceDiagram participant SMSC as Upstream SMSC participant Worker as SmppClientWorker participant Tracker as InMemoryMessageTracker - participant Store as InMemoryDlrService + participant Store as DlrStorage participant Router as Router queue participant DLRHook as ForwardDlrService participant App as Originating application @@ -209,9 +216,9 @@ Sendium expects runtime files in the configured `conf` directory. ## Persistence Boundaries -Most runtime queues are in-memory. The DLR correlation service uses H2 MVStore at `data/dlr-mvstore.db` by default and falls back to in-memory maps if the store cannot be opened. +Most runtime queues are in memory. DLR tracking, provider correlations, and unpushed downstream SMPP receipts can use PostgreSQL or the compatibility MVStore backend. Sendium completes the selected storage operation before HTTP routing or successful downstream SMPP acknowledgement; PostgreSQL makes that state durable, while MVStore can fall back to memory if its file cannot be opened. Queued and in-flight messages remain process-local. -This means operators should treat queued, in-flight messages as process-local state, while DLR correlation has lightweight local persistence. +PostgreSQL does not make multipart assembly, replay claims, callback retries, or router and worker queues durable. See [DLR Persistence](13-dlr-persistence.md) for retention, restart guarantees, cutover, rollback, and the remaining crash windows. ## Related Documentation @@ -221,3 +228,4 @@ This means operators should treat queued, in-flight messages as process-local st * [Routing Engine](05-routing-engine.md) * [Webhooks](07-webhooks.md) * [Docker Deployment](02-docker-deployment.md) +* [DLR Persistence](13-dlr-persistence.md) diff --git a/docs/02-docker-deployment.md b/docs/02-docker-deployment.md index 6df85cf..34a3261 100644 --- a/docs/02-docker-deployment.md +++ b/docs/02-docker-deployment.md @@ -4,7 +4,7 @@ This guide explains how to run Sendium with Docker for local testing or simple d ## Generated Quick Start -The recommended evaluation path generates random local credentials, Docker Compose, and the three required configuration files: +The recommended evaluation path generates random local credentials, Docker Compose, PostgreSQL 17 with a persistent named volume, and the three required configuration files: ```bash curl -fsSLo quick-start.sh \ @@ -43,9 +43,11 @@ sendium/ logs/ ``` -`.sendium.env`, `credentials.yml`, and `smsg.properties` contain secrets. The generated `.gitignore` excludes them, but they still require access-controlled storage and backups. +`.sendium.env`, `credentials.yml`, and `smsg.properties` contain secrets. The generated `.gitignore` excludes them, but they still require access-controlled storage and backups. The local PostgreSQL service is private to the Compose network and does not publish a database port. -Using `--force` regenerates the local credentials and configuration. When startup is enabled, Quick Start recreates the container so the new credentials and worker configuration take effect together. With `--no-start`, it prints the required `docker compose up -d --force-recreate` command instead. +Using `--force` regenerates the HTTP/SMPP credentials and configuration while preserving the generated local database password required by the existing PostgreSQL volume. When startup is enabled, Quick Start recreates the containers so the new credentials and worker configuration take effect together. With `--no-start`, it prints the required `docker compose up -d --force-recreate --remove-orphans` command instead. + +To use an operator-managed PostgreSQL database, set `SENDIUM_DLR_POSTGRESQL_JDBC_URL`, `SENDIUM_DLR_POSTGRESQL_USERNAME`, and `SENDIUM_DLR_POSTGRESQL_PASSWORD` together before running Quick Start. The generated Compose file then omits the local PostgreSQL service. See [DLR Persistence](13-dlr-persistence.md) for TLS, permissions, retention, cutover, and rollback guidance. To generate a separate runtime using the native image, first stop any generated runtime using the same local ports: @@ -132,7 +134,7 @@ After starting the container: 1. Check container status with `docker ps`. 2. Open `http://localhost:8080/swagger-ui` to confirm the HTTP API is available. -3. Open `http://localhost:8080/openapi.json` to confirm OpenAPI is available. +3. Check `http://localhost:8080/q/health/ready` and confirm the `sendium-dlr-storage` check is `UP` before sending traffic. 4. Inspect `logs/smsg.log`, `logs/smppclient.log`, and `logs/smppserver.log` if startup fails. ## Configuration Files @@ -162,9 +164,13 @@ docker compose logs -f docker compose down ``` +`docker compose down` retains the generated PostgreSQL volume. Adding `--volumes` permanently removes it and should only be used when database deletion is intended. + For the manual `docker run` example: ```bash docker stop sendium docker rm sendium ``` + +See [DLR Persistence](13-dlr-persistence.md) before changing storage backends or changing how the database volume is managed. diff --git a/docs/07-webhooks.md b/docs/07-webhooks.md index 42e7306..802db88 100644 --- a/docs/07-webhooks.md +++ b/docs/07-webhooks.md @@ -34,7 +34,7 @@ curl -G http://localhost:8080/sendsms \ | `4` | Buffered or accepted for processing. | | `8` | Submitted to SMSC. | -DLR callbacks are sent as HTTP `GET` requests. HTTP status codes from `200` to `399` are treated as successful. Failed callback attempts are retried up to 10 times with a 120 second delay between attempts. +DLR callbacks are sent as HTTP `GET` requests. HTTP status codes from `200` to `399` are treated as successful. Sendium makes up to 10 attempts with a 120 second delay between failed attempts. The retry schedule is process-local and does not survive a Sendium restart. ## Mobile-Originated Message Forwarding @@ -85,7 +85,7 @@ outSms.instance.testRoute.forward.mo.url = https://example.com/mo?from=%p&to=%P& outSms.instance.testRoute.forward.mo.format = FORM ``` -MO callbacks are sent as HTTP `POST` requests. HTTP status codes from `200` to `399` are treated as successful. Failed callback attempts are retried up to 10 times with a 120 second delay between attempts. +MO callbacks are sent as HTTP `POST` requests. HTTP status codes from `200` to `399` are treated as successful. Sendium makes up to 10 attempts with a 120 second delay between failed attempts. The retry schedule is process-local and does not survive a Sendium restart. ## Security Notes diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index dc00230..5754eef 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -43,6 +43,22 @@ In the Docker image, the working directory is `/work`, so the default configurat | `QUARKUS_HTTP_ACCESS_LOG_ENABLE` | `true` | Enables HTTP access logging. | | `QUARKUS_HTTP_ACCESS_LOG_DIRECTORY` | `/work/logs` | HTTP access log directory. | +## DLR Storage Environment Variables + +| Variable | Default | Description | +| :--- | :--- | :--- | +| `SENDIUM_DLR_STORAGE` | `mvstore` | Selects `mvstore` or `postgresql`. Generated Quick Start runtimes explicitly select PostgreSQL. | +| `SENDIUM_DLR_MVSTORE_PATH` | `data/dlr-mvstore.db` | MVStore compatibility file path. | +| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `false` | Activates the named PostgreSQL datasource and Flyway migration. Must be `true` when PostgreSQL is selected. | +| `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | Empty | PostgreSQL JDBC URL. | +| `SENDIUM_DLR_POSTGRESQL_USERNAME` | Empty | PostgreSQL role name. | +| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Empty | PostgreSQL password; provide through an access-controlled environment or secret. | +| `SENDIUM_DLR_POSTGRESQL_POOL_MIN_SIZE` | `0` | Minimum datasource pool size. | +| `SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE` | `10` | Maximum datasource pool size. | +| `SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT` | `5S` | Maximum wait for a pooled connection. | + +PostgreSQL selection is fail-closed and requires the URL, username, password, active datasource, and Flyway migration to agree. See [DLR Persistence](13-dlr-persistence.md) before switching an existing deployment; Sendium does not transfer pending state between MVStore and PostgreSQL. + ## Logs | Log | Description | @@ -68,6 +84,8 @@ When the HTTP server is running, Sendium exposes: | :--- | :--- | | `/swagger-ui` | Interactive Swagger UI. | | `/openapi.json` | OpenAPI JSON document. | +| `/q/health/ready` | Readiness status and selected DLR backend. | +| `/q/metrics` | Prometheus metrics, including DLR storage and datasource metrics. | ## Related Documentation @@ -75,3 +93,4 @@ When the HTTP server is running, Sendium exposes: - [Authentication and Security](03-auth-security.md) - [SMPP Configuration](04-smpp-configuration.md) - [Routing Engine](05-routing-engine.md) +- [DLR Persistence](13-dlr-persistence.md) diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md new file mode 100644 index 0000000..779ae04 --- /dev/null +++ b/docs/13-dlr-persistence.md @@ -0,0 +1,160 @@ +# DLR Persistence + +Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client. PostgreSQL is the recommended backend for new deployments. MVStore remains available for compatibility with existing installations. + +This storage boundary does not make Sendium's message queues or all delivery processing durable. Review [Durability Boundaries](#durability-boundaries) before using restart recovery as a delivery guarantee. + +## Quick Start PostgreSQL + +The generated Quick Start runtime selects PostgreSQL and creates: + +- A private `postgres:17-alpine` service with no published database port. +- A named Docker volume for `/var/lib/postgresql/data`. +- A generated 256-bit database password in `.sendium.env`, with mode `600` where the filesystem can enforce Unix permissions. +- A PostgreSQL health check that gates Sendium startup. +- A readiness check that verifies the selected DLR schema is available. + +Run Quick Start normally: + +```bash +sh quick-start.sh +``` + +`docker compose down` removes the containers and network but retains the PostgreSQL volume. Do not use `docker compose down --volumes` or manually delete the volume unless permanent database deletion is intended. + +Quick Start preserves the local database password during `--force` regeneration. PostgreSQL initialization variables cannot rotate the password of a role that already exists in a persistent data volume. + +## External PostgreSQL + +To omit the local PostgreSQL service and connect Sendium to an operator-managed database, first export `SENDIUM_DLR_POSTGRESQL_PASSWORD` from an access-controlled secret source without placing its value in shell history. Then provide the URL and username when generating the runtime: + +```bash +SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://db.example.com:5432/sendium?sslmode=verify-full' \ +SENDIUM_DLR_POSTGRESQL_USERNAME='sendium' \ + sh quick-start.sh --directory sendium --provider local + +unset SENDIUM_DLR_POSTGRESQL_PASSWORD +``` + +The three values are an all-or-nothing override. Partial configuration is rejected instead of mixing local and external settings. + +For a manual deployment, configure: + +| Variable | Required value or example | Purpose | +| :--- | :--- | :--- | +| `SENDIUM_DLR_STORAGE` | `postgresql` | Selects the PostgreSQL storage adapter. | +| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` | Activates the named datasource and its Flyway migrations. | +| `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | `jdbc:postgresql://db.example.com:5432/sendium` | JDBC connection URL. Add PostgreSQL JDBC TLS parameters for external networks. | +| `SENDIUM_DLR_POSTGRESQL_USERNAME` | `sendium` | Database role used by Sendium and Flyway. | +| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Secret value | Database password. Supply it through an access-controlled environment or secret mechanism. | + +The database role must be able to connect to the database and create and manage the `sendium_dlr` schema. Flyway creates or validates the schema during startup. Keep migration privileges available for future application upgrades. + +Use TLS with certificate verification when the database connection crosses an untrusted network. The exact JDBC parameters and certificate path depend on the PostgreSQL service. Protect database credentials from shell history, logs, source control, screenshots, and unauthorized container inspection. + +## Pool Settings + +These optional settings retain their shown defaults: + +| Variable | Default | Purpose | +| :--- | :--- | :--- | +| `SENDIUM_DLR_POSTGRESQL_POOL_MIN_SIZE` | `0` | Minimum number of datasource connections. | +| `SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE` | `10` | Maximum number of datasource connections. | +| `SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT` | `5S` | Maximum wait for a pooled connection. | + +Size the pool against measured gateway concurrency and the database connection budget. Do not increase it beyond the database's safe connection capacity. + +## Startup And Monitoring + +Check readiness rather than only checking whether the HTTP listener is open: + +```bash +curl -fsS http://127.0.0.1:8080/q/health/ready +``` + +The `sendium-dlr-storage` readiness check reports the selected backend. It returns `DOWN` with a sanitized `unavailable` reason when the selected PostgreSQL schema cannot be queried. + +Inspect storage and datasource metrics with: + +```bash +curl -fsS http://127.0.0.1:8080/q/metrics | grep -E 'sendium_dlr_storage|agroal' +``` + +Relevant metrics include the selected backend and storage-operation latency/counts tagged by operation and success or error outcome. PostgreSQL pool metrics use the Agroal metric prefix. + +PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP submissions return the retryable `503` response and new SMPP submissions return `ESME_RSYSERR`; Sendium does not fall back to MVStore or memory. + +## Retention + +The V1 retention thresholds are fixed application behavior, not environment settings: + +| State | Eligible for cleanup after | +| :--- | :--- | +| Provider/operator correlation | 3 days | +| Tracked gateway message | 7 days | +| Unpushed downstream SMPP receipt | 7 days | + +Cleanup is triggered by storage activity and runs no more than once per hour. These values are therefore eligibility thresholds, not exact physical deletion deadlines: idle records can remain in the database longer, and an active deployment can retain newly eligible state until the next cleanup pass. A provider receipt cannot be matched after its correlation has been removed. Making the thresholds or cleanup schedule configurable is outside the V1 storage replacement. + +## Durability Boundaries + +| State or transition | PostgreSQL guarantee | Remaining limit | +| :--- | :--- | :--- | +| Initial DLR state for HTTP and downstream SMPP submissions | Persisted before HTTP routing or a successful SMPP acknowledgement. | Router and worker queues remain in memory. A process crash can lose queued outbound work even though its DLR row remains until cleanup. | +| Gateway-to-provider message correlation | Survives Sendium restart after the provider message ID is linked. | Resolving a provider receipt consumes the correlation before callback or downstream delivery completes. A crash in that window can lose the resulting receipt. | +| Unpushed downstream SMPP receipt | Survives restart and is replayed when the same system ID binds again. | The row is removed after admission to the worker queue, not after confirmed downstream delivery. A crash in that window can lose the receipt. | +| Replay claim | Prevents duplicate replay within one Sendium process. | Claims are process-local. Multiple active Sendium replicas can claim and deliver the same database row. V1 supports one active gateway process. | +| Multipart submission | Each acknowledged segment has provisional DLR state; completed aggregates update the primary state. | Multipart assembly and its pending timers are process-local and are not reconstructed after restart. | +| HTTP DLR callback retry | The resolved callback is attempted up to 10 times while the process remains running. | The retry schedule is in memory and is lost on restart. There is no durable callback outbox. | +| Database files | The Quick Start named volume survives normal container replacement and `docker compose down`. | Volume deletion, host-disk loss, and disaster recovery require backups or external PostgreSQL replication managed by the operator. | + +These limits are intentional V1 boundaries. PostgreSQL replaces the existing DLR persistence store; it is not a durable queue, distributed claim coordinator, or delivery outbox. + +## MVStore Compatibility + +For a manual deployment that must remain on MVStore, use: + +```text +SENDIUM_DLR_STORAGE=mvstore +SENDIUM_DLR_POSTGRESQL_ACTIVE=false +SENDIUM_DLR_MVSTORE_PATH=/work/data/dlr-mvstore.db +``` + +The MVStore path must be on persistent storage. If the file cannot be opened, MVStore compatibility behavior can fall back to in-memory storage; check readiness data for `mode=persistent` rather than assuming the mount is working. + +## Cut Over From MVStore + +There is no MVStore-to-PostgreSQL importer, dual-read period, or live migration. Existing correlations and unpushed receipts do not move when the backend changes. + +1. Confirm whether pending DLR state can be allowed to expire or be abandoned. The safest compatibility choice is to remain on MVStore until a deliberate maintenance window is acceptable. +2. Stop accepting new HTTP and SMPP submissions. +3. Allow in-flight provider receipts and downstream replay to drain. The longest cleanup threshold is seven days, and cleanup is opportunistic rather than an exact deadline; continuous-traffic installations cannot obtain a lossless cutover without an importer. +4. Stop Sendium and back up the complete runtime, including `data/dlr-mvstore.db`. +5. Provision PostgreSQL, backups, access controls, and TLS where required. +6. Configure the five PostgreSQL variables described above and start exactly one Sendium instance. +7. Require `/q/health/ready` to report `UP` with `backend=postgresql` before reopening traffic. +8. Submit controlled HTTP and SMPP messages and verify provider correlation, callbacks, and downstream receipts. +9. Retain the MVStore backup and PostgreSQL database until the rollback decision window has closed. + +Provider receipts for messages that existed only in MVStore will be unknown after the switch. Do not run MVStore-backed and PostgreSQL-backed Sendium instances concurrently against the same traffic as a migration strategy. + +## Roll Back To MVStore + +Rollback is also non-seamless. State written to PostgreSQL is not copied back to MVStore. + +1. Stop accepting traffic and stop every Sendium instance. +2. Preserve the PostgreSQL database; do not drop its schema or volume. +3. Restore the previous MVStore file and runtime configuration. +4. Set `SENDIUM_DLR_STORAGE=mvstore` and `SENDIUM_DLR_POSTGRESQL_ACTIVE=false`. Remove the PostgreSQL URL and credentials from the Sendium container environment when they are no longer needed. +5. Start one Sendium instance and require readiness to report `backend=mvstore` and `mode=persistent`. +6. Reopen traffic only after controlled HTTP/SMPP checks pass. + +Messages accepted while PostgreSQL was active remain only in PostgreSQL. A later switch back to PostgreSQL can see still-retained PostgreSQL rows, so preserve both stores and record the exact cutover times during any rollback. + +## Related Documentation + +- [Architecture Overview](01-architecture.md) +- [Docker Deployment](02-docker-deployment.md) +- [Monitoring And Observability](08-monitoring-observability.md) +- [Configuration Reference](09-configuration-reference.md) +- [Troubleshooting](10-troubleshooting.md) diff --git a/docs/DocumentationMap.md b/docs/DocumentationMap.md index a7f5fe7..a18750b 100644 --- a/docs/DocumentationMap.md +++ b/docs/DocumentationMap.md @@ -21,6 +21,7 @@ Sendium is an open-source, headless SMS gateway for high-throughput messaging. I | Start migrating from Kannel config | [Kannel migration converter](https://cytechmobile.github.io/sendium/) | | Understand releases and publishing | [11. Release Process](11-release-process.md) | | Review current features and roadmap | [12. Features And Roadmap](12-features-roadmap.md) | +| Configure PostgreSQL DLR persistence or plan a cutover | [13. DLR Persistence](13-dlr-persistence.md) | | Contribute code or docs | [Contributing](../.github/CONTRIBUTING.md) | ## Core Concepts @@ -61,6 +62,7 @@ Sendium expects these files in the configured `conf` directory. The Docker quick | [10. Troubleshooting](10-troubleshooting.md) | Common startup, authentication, routing, SMPP, webhook, and logging issues. | | [11. Release Process](11-release-process.md) | Release Please flow, Conventional Commit rules, release PR handling, GitHub Packages, and Docker publishing. | | [12. Features And Roadmap](12-features-roadmap.md) | Current product capabilities, planned roadmap phases, and related feature documentation. | +| [13. DLR Persistence](13-dlr-persistence.md) | PostgreSQL and MVStore setup, retention, restart guarantees, cutover, rollback, and durability limits. | | [Kannel migration converter](https://cytechmobile.github.io/sendium/) | Browser-only helper for turning a legacy `kannel.conf` into Sendium starter files. | ## API Discovery @@ -73,6 +75,7 @@ When Sendium is running, the HTTP API can be inspected through: | `/swagger-ui` | Interactive Swagger UI. | | `/openapi.json` | OpenAPI specification. | | `/q/metrics` | Prometheus-compatible Micrometer metrics endpoint. | +| `/q/health/ready` | Readiness status, including the selected DLR storage backend. | ## Community And Project Files @@ -96,8 +99,9 @@ When Sendium is running, the HTTP API can be inspected through: 7. Submit a test message using [HTTP API](06-http-api.md). 8. Add delivery callbacks using [Webhooks](07-webhooks.md). 9. Monitor the service using [Monitoring And Observability](08-monitoring-observability.md). -10. Review current and planned product scope in [Features And Roadmap](12-features-roadmap.md). -11. Learn how releases are created and published in [Release Process](11-release-process.md). +10. Review storage guarantees in [DLR Persistence](13-dlr-persistence.md). +11. Review current and planned product scope in [Features And Roadmap](12-features-roadmap.md). +12. Learn how releases are created and published in [Release Process](11-release-process.md). ## Documentation Gaps To Improve Next From ba2c5826472a36c8fa5110149b4fc2b0d94d8ce3 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 18 Aug 2026 15:15:10 +0300 Subject: [PATCH 12/20] feat(dlr): make PostgreSQL the default backend --- .github/CONTRIBUTING.md | 22 ++++++-- docs/02-docker-deployment.md | 4 +- docs/09-configuration-reference.md | 10 ++-- docs/13-dlr-persistence.md | 12 ++--- .../src/main/resources/application.properties | 9 ++-- .../core/worker/ConfiguredDlrStorage.java | 2 +- .../src/main/resources/application.properties | 8 +-- .../core/worker/DlrStorageRuntimeTest.java | 2 +- .../src/test/java/utils/NativeE2eSmoke.java | 50 +++++++++++++++---- 9 files changed, 85 insertions(+), 34 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e38afaa..4ce6c92 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -26,21 +26,37 @@ git clone https://github.com/cytechmobile/sendium.git cd sendium ``` **2. Start the application in development mode:** + +PostgreSQL is the runtime default. To run locally without a database, explicitly select MVStore compatibility mode: ```bash -./mvnw -pl sendium-app -am quarkus:dev +SENDIUM_DLR_STORAGE=mvstore \ +SENDIUM_DLR_POSTGRESQL_ACTIVE=false \ + ./mvnw -pl sendium-app -am quarkus:dev ``` Note: This will start the server with live reload enabled. Any changes you make to the Java code will automatically trigger a compilation and reload. -On Windows PowerShell, replace `./mvnw` with `.\mvnw.cmd`. +On Windows PowerShell: + +```powershell +$env:SENDIUM_DLR_STORAGE = "mvstore" +$env:SENDIUM_DLR_POSTGRESQL_ACTIVE = "false" +.\mvnw.cmd -pl sendium-app -am quarkus:dev +``` **3. 🧪 Testing** We value reliability. Before submitting any changes, please ensure all tests pass. -You do not need Docker running locally to execute the test suite. Simply run the following command to execute all unit and integration tests: +You do not need Docker running locally to execute the default unit and integration suite. PostgreSQL-specific tests are skipped: ```bash ./mvnw verify ``` + +To include the PostgreSQL migration, adapter, and outage tests, start Docker and run: + +```bash +./mvnw verify -Ppostgresql-tests +``` **4. 💅 Code Style & Linting** We enforce a consistent code style across the project using Checkstyle. Our rules are defined in the checkstyle.xml file located in the root of the repository. diff --git a/docs/02-docker-deployment.md b/docs/02-docker-deployment.md index 34a3261..165c9e8 100644 --- a/docs/02-docker-deployment.md +++ b/docs/02-docker-deployment.md @@ -108,10 +108,12 @@ Sendium publishes two Docker image variants: ### Run Command -This command starts the default JVM image: +This standalone example explicitly uses MVStore compatibility mode. For the default PostgreSQL backend, use Generated Quick Start or configure an external database as described in [DLR Persistence](13-dlr-persistence.md). ```bash docker run -d --name sendium \ + -e SENDIUM_DLR_STORAGE=mvstore \ + -e SENDIUM_DLR_POSTGRESQL_ACTIVE=false \ -e QUARKUS_LOG_FILE_ENABLE=true \ -e QUARKUS_LOG_CONSOLE_ENABLE=false \ -e QUARKUS_LOG_FILE_PATH=/work/logs/smsg.log \ diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 5754eef..ed9dcbc 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -47,17 +47,17 @@ In the Docker image, the working directory is `/work`, so the default configurat | Variable | Default | Description | | :--- | :--- | :--- | -| `SENDIUM_DLR_STORAGE` | `mvstore` | Selects `mvstore` or `postgresql`. Generated Quick Start runtimes explicitly select PostgreSQL. | +| `SENDIUM_DLR_STORAGE` | `postgresql` | Selects `postgresql` or the explicit `mvstore` compatibility backend. | | `SENDIUM_DLR_MVSTORE_PATH` | `data/dlr-mvstore.db` | MVStore compatibility file path. | -| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `false` | Activates the named PostgreSQL datasource and Flyway migration. Must be `true` when PostgreSQL is selected. | +| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` | Activates the named PostgreSQL datasource and Flyway migration. Must be `false` when MVStore is selected. | | `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | Empty | PostgreSQL JDBC URL. | -| `SENDIUM_DLR_POSTGRESQL_USERNAME` | Empty | PostgreSQL role name. | -| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Empty | PostgreSQL password; provide through an access-controlled environment or secret. | +| `SENDIUM_DLR_POSTGRESQL_USERNAME` | Empty | PostgreSQL role name when required by the database authentication method. | +| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Empty | PostgreSQL password when required; provide through an access-controlled environment or secret. | | `SENDIUM_DLR_POSTGRESQL_POOL_MIN_SIZE` | `0` | Minimum datasource pool size. | | `SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE` | `10` | Maximum datasource pool size. | | `SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT` | `5S` | Maximum wait for a pooled connection. | -PostgreSQL selection is fail-closed and requires the URL, username, password, active datasource, and Flyway migration to agree. See [DLR Persistence](13-dlr-persistence.md) before switching an existing deployment; Sendium does not transfer pending state between MVStore and PostgreSQL. +PostgreSQL selection is fail-closed. A default startup requires a valid datasource URL and any username, password, certificates, or tokens required by the database authentication method; a bare launch fails rather than falling back. Explicit MVStore compatibility requires both `SENDIUM_DLR_STORAGE=mvstore` and `SENDIUM_DLR_POSTGRESQL_ACTIVE=false`. See [DLR Persistence](13-dlr-persistence.md) before switching an existing deployment; Sendium does not transfer pending state between MVStore and PostgreSQL. ## Logs diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 779ae04..20cefa4 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -1,6 +1,6 @@ # DLR Persistence -Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client. PostgreSQL is the recommended backend for new deployments. MVStore remains available for compatibility with existing installations. +Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client. PostgreSQL is the default backend for new deployments. MVStore remains available for compatibility with existing installations. This storage boundary does not make Sendium's message queues or all delivery processing durable. Review [Durability Boundaries](#durability-boundaries) before using restart recovery as a delivery guarantee. @@ -38,15 +38,15 @@ unset SENDIUM_DLR_POSTGRESQL_PASSWORD The three values are an all-or-nothing override. Partial configuration is rejected instead of mixing local and external settings. -For a manual deployment, configure: +For a manual deployment, PostgreSQL selection and datasource activation default to the values below. A valid connection URL and the settings required by the database authentication method must still be supplied: | Variable | Required value or example | Purpose | | :--- | :--- | :--- | -| `SENDIUM_DLR_STORAGE` | `postgresql` | Selects the PostgreSQL storage adapter. | -| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` | Activates the named datasource and its Flyway migrations. | +| `SENDIUM_DLR_STORAGE` | `postgresql` (default) | Selects the PostgreSQL storage adapter. | +| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` (default) | Activates the named datasource and its Flyway migrations. | | `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | `jdbc:postgresql://db.example.com:5432/sendium` | JDBC connection URL. Add PostgreSQL JDBC TLS parameters for external networks. | -| `SENDIUM_DLR_POSTGRESQL_USERNAME` | `sendium` | Database role used by Sendium and Flyway. | -| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Secret value | Database password. Supply it through an access-controlled environment or secret mechanism. | +| `SENDIUM_DLR_POSTGRESQL_USERNAME` | `sendium` | Database role used by Sendium and Flyway when required by the authentication method. | +| `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Secret value | Database password when required. Supply it through an access-controlled environment or secret mechanism. | The database role must be able to connect to the database and create and manage the `sendium_dlr` schema. Flyway creates or validates the schema during startup. Keep migration privileges available for future application upgrades. diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index 08f4535..e9e5e20 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -2,12 +2,15 @@ smsg.routing.file.path=conf/routingTable.conf smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml -# DLR persistence. PostgreSQL also requires the named datasource URL and credentials. -sendium.dlr.storage=${SENDIUM_DLR_STORAGE:mvstore} +%test.sendium.dlr.storage=mvstore +%test.quarkus.datasource.dlr.active=false + +# DLR persistence. PostgreSQL requires valid named datasource connection settings. +sendium.dlr.storage=${SENDIUM_DLR_STORAGE:postgresql} sendium.dlr.db.path=${SENDIUM_DLR_MVSTORE_PATH:data/dlr-mvstore.db} quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=${SENDIUM_DLR_POSTGRESQL_ACTIVE:false} +quarkus.datasource.dlr.active=${SENDIUM_DLR_POSTGRESQL_ACTIVE:true} quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.url=${SENDIUM_DLR_POSTGRESQL_JDBC_URL:} quarkus.datasource.dlr.username=${SENDIUM_DLR_POSTGRESQL_USERNAME:} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java index 120d8d6..90c6532 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java @@ -33,7 +33,7 @@ public class ConfiguredDlrStorage implements DlrStorage { """; @Inject - @ConfigProperty(name = "sendium.dlr.storage", defaultValue = "mvstore") + @ConfigProperty(name = "sendium.dlr.storage", defaultValue = "postgresql") String configuredBackend; @Inject diff --git a/sendium-core/src/main/resources/application.properties b/sendium-core/src/main/resources/application.properties index aa00fff..b753c86 100644 --- a/sendium-core/src/main/resources/application.properties +++ b/sendium-core/src/main/resources/application.properties @@ -1,13 +1,15 @@ %test.smsg.routing.file.path=src/test/resources/routingTable.conf %test.smsg.properties.file.path=src/test/resources/smsg.properties %test.smsg.credentials.file.path=src/test/resources/credentials.yml +%test.sendium.dlr.storage=mvstore %test.sendium.dlr.db.path=${java.io.tmpdir}/sendium-dlr-${quarkus.uuid}.db +%test.quarkus.datasource.dlr.active=false -# The PostgreSQL DLR datasource stays inactive unless explicitly selected and activated. -sendium.dlr.storage=mvstore +# PostgreSQL is the application default. Tests explicitly retain Docker-free MVStore. +sendium.dlr.storage=postgresql quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=false +quarkus.datasource.dlr.active=true quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.min-size=0 quarkus.datasource.dlr.jdbc.max-size=10 diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java index cd6fa70..1bcf6b1 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java @@ -25,7 +25,7 @@ class DlrStorageRuntimeTest { MeterRegistry meterRegistry; @Test - void selectsExactlyOneMvStoreBackendByDefault() { + void selectsExactlyOneMvStoreBackendInTestProfile() { assertThat(storageInstance.isResolvable()).isTrue(); assertThat(storageInstance.stream()).hasSize(1); assertThat(storageInstance.get()).isSameAs(configuredStorage); diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index e4ad9a3..470de99 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -83,6 +83,10 @@ public static void main(String[] args) throws Exception { container = verifyUnpushedDlrSurvivesRestart(containerName, workDir, upstream); container = verifyHttpCorrelationSurvivesRestart(containerName, workDir, upstream, callbackServer, 2); verifySmppSubmitGetsDeliverSm(upstream, 3); + + stopContainer(containerName); + container = startMvStoreContainer(containerName, workDir); + waitForMvStoreReadiness(); } catch (Throwable t) { printDockerLogs(containerName); throw t; @@ -218,7 +222,15 @@ private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Pa } private static Process startSendiumContainer(String containerName, Path workDir) throws Exception { - List command = List.of( + return startSendiumContainer(containerName, workDir, false); + } + + private static Process startMvStoreContainer(String containerName, Path workDir) throws Exception { + return startSendiumContainer(containerName, workDir, true); + } + + private static Process startSendiumContainer(String containerName, Path workDir, boolean mvStore) throws Exception { + List command = new ArrayList<>(List.of( "docker", "run", "--rm", "-d", "--name", containerName, "--add-host", "host.docker.internal:host-gateway", @@ -228,14 +240,21 @@ private static Process startSendiumContainer(String containerName, Path workDir) "-v", workDir.resolve("data").toAbsolutePath() + ":/work/data", "-v", workDir.resolve("logs").toAbsolutePath() + ":/work/logs", "-e", "QUARKUS_LOG_LEVEL=INFO", - "-e", "LOG_LEVEL=INFO", - "-e", "SENDIUM_DLR_STORAGE=postgresql", - "-e", "SENDIUM_DLR_POSTGRESQL_ACTIVE=true", - "-e", "SENDIUM_DLR_POSTGRESQL_JDBC_URL=" + POSTGRESQL_JDBC_URL, - "-e", "SENDIUM_DLR_POSTGRESQL_USERNAME=" + POSTGRESQL_USERNAME, - "-e", "SENDIUM_DLR_POSTGRESQL_PASSWORD=" + POSTGRESQL_PASSWORD, - IMAGE - ); + "-e", "LOG_LEVEL=INFO" + )); + if (mvStore) { + command.addAll(List.of( + "-e", "SENDIUM_DLR_STORAGE=mvstore", + "-e", "SENDIUM_DLR_POSTGRESQL_ACTIVE=false" + )); + } else { + command.addAll(List.of( + "-e", "SENDIUM_DLR_POSTGRESQL_JDBC_URL=" + POSTGRESQL_JDBC_URL, + "-e", "SENDIUM_DLR_POSTGRESQL_USERNAME=" + POSTGRESQL_USERNAME, + "-e", "SENDIUM_DLR_POSTGRESQL_PASSWORD=" + POSTGRESQL_PASSWORD + )); + } + command.add(IMAGE); Process process = run(command, true); require(process.waitFor(30, TimeUnit.SECONDS), "Timed out starting Sendium container"); require(process.exitValue() == 0, "Failed to start Sendium container"); @@ -315,6 +334,14 @@ private static void waitForPort(String host, int port, Duration timeout) throws } private static void waitForPostgresqlReadiness() throws Exception { + waitForStorageReadiness("postgresql", null); + } + + private static void waitForMvStoreReadiness() throws Exception { + waitForStorageReadiness("mvstore", "persistent"); + } + + private static void waitForStorageReadiness(String backend, String mode) throws Exception { long deadline = System.nanoTime() + TIMEOUT.toNanos(); while (System.nanoTime() < deadline) { try { @@ -322,14 +349,15 @@ private static void waitForPostgresqlReadiness() throws Exception { String body = response.body().replaceAll("\\s", ""); if (response.statusCode() == HttpURLConnection.HTTP_OK && body.contains("\"name\":\"sendium-dlr-storage\"") - && body.contains("\"backend\":\"postgresql\"")) { + && body.contains("\"backend\":\"" + backend + "\"") + && (mode == null || body.contains("\"mode\":\"" + mode + "\""))) { return; } } catch (IOException ignored) { } Thread.sleep(500); } - throw new IllegalStateException("Timed out waiting for PostgreSQL-backed Sendium readiness"); + throw new IllegalStateException("Timed out waiting for " + backend + "-backed Sendium readiness"); } private static void awaitSuccessfulStorageOperation(String operation) throws Exception { From 01c245da4f1f4d9dd14eb90eaece0b6d0ef52091 Mon Sep 17 00:00:00 2001 From: pavlos Date: Wed, 19 Aug 2026 12:13:35 +0300 Subject: [PATCH 13/20] refactor(dlr): remove MVStore backend --- .github/CONTRIBUTING.md | 35 +- .github/workflows/run_tests.yml | 2 +- docs/01-architecture.md | 6 +- docs/02-docker-deployment.md | 22 +- docs/09-configuration-reference.md | 7 +- docs/13-dlr-persistence.md | 65 +- docs/DocumentationMap.md | 6 +- pom.xml | 15 - quick-start.sh | 5 +- sendium-app/src/main/docker/Dockerfile.jvm | 2 +- sendium-app/src/main/docker/Dockerfile.native | 3 +- .../src/main/resources/application.properties | 7 +- sendium-core/pom.xml | 6 - .../core/worker/DlrStorageReadinessCheck.java | 8 +- ...DlrStorage.java => ManagedDlrStorage.java} | 53 +- .../core/worker/MvStoreDlrStorage.java | 575 ------------------ .../src/main/resources/application.properties | 6 +- ...onTest.java => PostgresqlMigrationIT.java} | 4 +- .../sendium/core/http/KannelResourceIT.java | 3 + ...ava => StandardMessageJsonResourceIT.java} | 5 +- .../core/worker/ConfiguredDlrStorageTest.java | 143 ----- .../worker/DlrStorageReadinessCheckTest.java | 19 +- .../core/worker/DlrStorageRuntimeTest.java | 56 -- .../core/worker/ManagedDlrStorageTest.java | 60 ++ .../core/worker/MvStoreDlrStorageTest.java | 288 --------- .../PostgresqlDlrQuarkusTestResource.java | 5 - ...eTest.java => PostgresqlDlrRuntimeIT.java} | 14 +- ...eTest.java => PostgresqlDlrStorageIT.java} | 4 +- .../src/test/java/utils/NativeE2eSmoke.java | 50 +- tests/quick-start-test.sh | 2 - 30 files changed, 165 insertions(+), 1311 deletions(-) rename sendium-core/src/main/java/gr/cytech/sendium/core/worker/{ConfiguredDlrStorage.java => ManagedDlrStorage.java} (73%) delete mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java rename sendium-core/src/test/java/gr/cytech/sendium/core/dlr/{PostgresqlMigrationTest.java => PostgresqlMigrationIT.java} (98%) rename sendium-core/src/test/java/gr/cytech/sendium/core/message/{StandardMessageJsonResourceTest.java => StandardMessageJsonResourceIT.java} (78%) delete mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java delete mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/ManagedDlrStorageTest.java delete mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java rename sendium-core/src/test/java/gr/cytech/sendium/core/worker/{PostgresqlDlrRuntimeTest.java => PostgresqlDlrRuntimeIT.java} (96%) rename sendium-core/src/test/java/gr/cytech/sendium/core/worker/{PostgresqlDlrStorageTest.java => PostgresqlDlrStorageIT.java} (99%) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4ce6c92..7bf13b5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -27,10 +27,19 @@ cd sendium ``` **2. Start the application in development mode:** -PostgreSQL is the runtime default. To run locally without a database, explicitly select MVStore compatibility mode: +Start a local PostgreSQL instance: + ```bash -SENDIUM_DLR_STORAGE=mvstore \ -SENDIUM_DLR_POSTGRESQL_ACTIVE=false \ +docker run --rm -d --name sendium-postgres-dev \ + -e POSTGRES_DB=sendium \ + -e POSTGRES_USER=sendium \ + -e POSTGRES_PASSWORD=sendium-dev \ + -p 5432:5432 \ + postgres:17-alpine + +SENDIUM_DLR_POSTGRESQL_JDBC_URL=jdbc:postgresql://localhost:5432/sendium \ +SENDIUM_DLR_POSTGRESQL_USERNAME=sendium \ +SENDIUM_DLR_POSTGRESQL_PASSWORD=sendium-dev \ ./mvnw -pl sendium-app -am quarkus:dev ``` Note: This will start the server with live reload enabled. Any changes you make to the Java code will automatically trigger a compilation and reload. @@ -38,8 +47,16 @@ Note: This will start the server with live reload enabled. Any changes you make On Windows PowerShell: ```powershell -$env:SENDIUM_DLR_STORAGE = "mvstore" -$env:SENDIUM_DLR_POSTGRESQL_ACTIVE = "false" +docker run --rm -d --name sendium-postgres-dev ` + -e POSTGRES_DB=sendium ` + -e POSTGRES_USER=sendium ` + -e POSTGRES_PASSWORD=sendium-dev ` + -p 5432:5432 ` + postgres:17-alpine + +$env:SENDIUM_DLR_POSTGRESQL_JDBC_URL = "jdbc:postgresql://localhost:5432/sendium" +$env:SENDIUM_DLR_POSTGRESQL_USERNAME = "sendium" +$env:SENDIUM_DLR_POSTGRESQL_PASSWORD = "sendium-dev" .\mvnw.cmd -pl sendium-app -am quarkus:dev ``` @@ -47,15 +64,15 @@ $env:SENDIUM_DLR_POSTGRESQL_ACTIVE = "false" We value reliability. Before submitting any changes, please ensure all tests pass. -You do not need Docker running locally to execute the default unit and integration suite. PostgreSQL-specific tests are skipped: +Unit tests do not require Docker: ```bash -./mvnw verify +./mvnw test ``` -To include the PostgreSQL migration, adapter, and outage tests, start Docker and run: +The complete verification suite includes PostgreSQL migration, adapter, Quarkus, outage, and protocol tests. Start Docker and run: ```bash -./mvnw verify -Ppostgresql-tests +./mvnw verify ``` **4. 💅 Code Style & Linting** diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index fdec117..b73ce1d 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -54,7 +54,7 @@ jobs: run: | export QUARKUS_LOG_LEVEL=ERROR chmod +x ./mvnw - ./mvnw verify -Ppostgresql-tests + ./mvnw verify - name: Build quick-start test image run: docker build -t sendium:quick-start-test -f sendium-app/src/main/docker/Dockerfile.jvm sendium-app - name: Test generated container startup diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 6f413c8..03cc1b1 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -23,7 +23,7 @@ flowchart LR workerQueues[Worker queues] smppClients["SMPP client workers
smppclient instances"] carriers["Upstream SMSCs
carriers or SMPP providers"] - dlrStore["DLR storage
PostgreSQL or MVStore"] + dlrStore["DLR storage
PostgreSQL"] webhooks["HTTP webhooks
DLR and MO callbacks"] config["Runtime config files
credentials.yml
smsg.properties
routingTable.conf"] @@ -216,9 +216,9 @@ Sendium expects runtime files in the configured `conf` directory. ## Persistence Boundaries -Most runtime queues are in memory. DLR tracking, provider correlations, and unpushed downstream SMPP receipts can use PostgreSQL or the compatibility MVStore backend. Sendium completes the selected storage operation before HTTP routing or successful downstream SMPP acknowledgement; PostgreSQL makes that state durable, while MVStore can fall back to memory if its file cannot be opened. Queued and in-flight messages remain process-local. +Most runtime queues are in memory. DLR tracking, provider correlations, and unpushed downstream SMPP receipts use PostgreSQL. Sendium completes the required storage operation before HTTP routing or successful downstream SMPP acknowledgement. Queued and in-flight messages remain process-local. -PostgreSQL does not make multipart assembly, replay claims, callback retries, or router and worker queues durable. See [DLR Persistence](13-dlr-persistence.md) for retention, restart guarantees, cutover, rollback, and the remaining crash windows. +PostgreSQL does not make multipart assembly, replay claims, callback retries, or router and worker queues durable. See [DLR Persistence](13-dlr-persistence.md) for retention, restart guarantees, and the remaining crash windows. ## Related Documentation diff --git a/docs/02-docker-deployment.md b/docs/02-docker-deployment.md index 165c9e8..1c04e2c 100644 --- a/docs/02-docker-deployment.md +++ b/docs/02-docker-deployment.md @@ -39,7 +39,6 @@ sendium/ credentials.yml smsg.properties routingTable.conf - data/ logs/ ``` @@ -47,7 +46,7 @@ sendium/ Using `--force` regenerates the HTTP/SMPP credentials and configuration while preserving the generated local database password required by the existing PostgreSQL volume. When startup is enabled, Quick Start recreates the containers so the new credentials and worker configuration take effect together. With `--no-start`, it prints the required `docker compose up -d --force-recreate --remove-orphans` command instead. -To use an operator-managed PostgreSQL database, set `SENDIUM_DLR_POSTGRESQL_JDBC_URL`, `SENDIUM_DLR_POSTGRESQL_USERNAME`, and `SENDIUM_DLR_POSTGRESQL_PASSWORD` together before running Quick Start. The generated Compose file then omits the local PostgreSQL service. See [DLR Persistence](13-dlr-persistence.md) for TLS, permissions, retention, cutover, and rollback guidance. +To use an operator-managed PostgreSQL database, set `SENDIUM_DLR_POSTGRESQL_JDBC_URL`, `SENDIUM_DLR_POSTGRESQL_USERNAME`, and `SENDIUM_DLR_POSTGRESQL_PASSWORD` together before running Quick Start. The generated Compose file then omits the local PostgreSQL service. See [DLR Persistence](13-dlr-persistence.md) for TLS, permissions, retention, and durability guidance. To generate a separate runtime using the native image, first stop any generated runtime using the same local ports: @@ -64,7 +63,7 @@ sh quick-start.sh \ ### Prerequisites - Docker installed on the host machine. -- A working directory with `conf`, `data`, and `logs` subdirectories. +- A working directory with `conf` and `logs` subdirectories. - The required configuration files inside `conf`: `credentials.yml`, `smsg.properties`, and `routingTable.conf`. ### Directory Layout @@ -75,7 +74,6 @@ sendium-runtime/ credentials.yml smsg.properties routingTable.conf - data/ logs/ ``` @@ -94,7 +92,6 @@ Set `outSms.instance..srv.host = 0.0.0.0` inside the container for the Doc | Host path | Container path | Purpose | | :--- | :--- | :--- | | `./conf` | `/work/conf` | Runtime configuration files. | -| `./data` | `/work/data` | Local runtime data. | | `./logs` | `/work/logs` | Application, SMPP, and HTTP access logs. | ### Docker Images @@ -108,12 +105,16 @@ Sendium publishes two Docker image variants: ### Run Command -This standalone example explicitly uses MVStore compatibility mode. For the default PostgreSQL backend, use Generated Quick Start or configure an external database as described in [DLR Persistence](13-dlr-persistence.md). +This example expects PostgreSQL to be reachable on port `5432` of the Docker host. Export the database password from an access-controlled secret source before starting Sendium: ```bash +export SENDIUM_DLR_POSTGRESQL_PASSWORD='replace-with-a-secret' + docker run -d --name sendium \ - -e SENDIUM_DLR_STORAGE=mvstore \ - -e SENDIUM_DLR_POSTGRESQL_ACTIVE=false \ + --add-host host.docker.internal:host-gateway \ + -e SENDIUM_DLR_POSTGRESQL_JDBC_URL=jdbc:postgresql://host.docker.internal:5432/sendium \ + -e SENDIUM_DLR_POSTGRESQL_USERNAME=sendium \ + -e SENDIUM_DLR_POSTGRESQL_PASSWORD \ -e QUARKUS_LOG_FILE_ENABLE=true \ -e QUARKUS_LOG_CONSOLE_ENABLE=false \ -e QUARKUS_LOG_FILE_PATH=/work/logs/smsg.log \ @@ -123,9 +124,10 @@ docker run -d --name sendium \ -p 127.0.0.1:8080:8080 \ -p 127.0.0.1:27777:27777 \ -v ./conf:/work/conf \ - -v ./data:/work/data \ -v ./logs:/work/logs \ cytechmobile/sendium:latest + +unset SENDIUM_DLR_POSTGRESQL_PASSWORD ``` To run the native image instead, use `cytechmobile/sendium:latest-native`. @@ -175,4 +177,4 @@ docker stop sendium docker rm sendium ``` -See [DLR Persistence](13-dlr-persistence.md) before changing storage backends or changing how the database volume is managed. +See [DLR Persistence](13-dlr-persistence.md) before changing how the database volume is managed. diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index ed9dcbc..d683eb3 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -47,9 +47,6 @@ In the Docker image, the working directory is `/work`, so the default configurat | Variable | Default | Description | | :--- | :--- | :--- | -| `SENDIUM_DLR_STORAGE` | `postgresql` | Selects `postgresql` or the explicit `mvstore` compatibility backend. | -| `SENDIUM_DLR_MVSTORE_PATH` | `data/dlr-mvstore.db` | MVStore compatibility file path. | -| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` | Activates the named PostgreSQL datasource and Flyway migration. Must be `false` when MVStore is selected. | | `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | Empty | PostgreSQL JDBC URL. | | `SENDIUM_DLR_POSTGRESQL_USERNAME` | Empty | PostgreSQL role name when required by the database authentication method. | | `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Empty | PostgreSQL password when required; provide through an access-controlled environment or secret. | @@ -57,7 +54,7 @@ In the Docker image, the working directory is `/work`, so the default configurat | `SENDIUM_DLR_POSTGRESQL_POOL_MAX_SIZE` | `10` | Maximum datasource pool size. | | `SENDIUM_DLR_POSTGRESQL_ACQUISITION_TIMEOUT` | `5S` | Maximum wait for a pooled connection. | -PostgreSQL selection is fail-closed. A default startup requires a valid datasource URL and any username, password, certificates, or tokens required by the database authentication method; a bare launch fails rather than falling back. Explicit MVStore compatibility requires both `SENDIUM_DLR_STORAGE=mvstore` and `SENDIUM_DLR_POSTGRESQL_ACTIVE=false`. See [DLR Persistence](13-dlr-persistence.md) before switching an existing deployment; Sendium does not transfer pending state between MVStore and PostgreSQL. +PostgreSQL is the only DLR persistence backend and is fail-closed. Startup requires a valid datasource URL and any username, password, certificates, or tokens required by the database authentication method; a bare launch fails rather than falling back to local or in-memory storage. See [DLR Persistence](13-dlr-persistence.md) for the complete durability contract. ## Logs @@ -84,7 +81,7 @@ When the HTTP server is running, Sendium exposes: | :--- | :--- | | `/swagger-ui` | Interactive Swagger UI. | | `/openapi.json` | OpenAPI JSON document. | -| `/q/health/ready` | Readiness status and selected DLR backend. | +| `/q/health/ready` | Readiness status and PostgreSQL DLR availability. | | `/q/metrics` | Prometheus metrics, including DLR storage and datasource metrics. | ## Related Documentation diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 20cefa4..7186204 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -1,18 +1,18 @@ # DLR Persistence -Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client. PostgreSQL is the default backend for new deployments. MVStore remains available for compatibility with existing installations. +Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client in PostgreSQL. This storage boundary does not make Sendium's message queues or all delivery processing durable. Review [Durability Boundaries](#durability-boundaries) before using restart recovery as a delivery guarantee. ## Quick Start PostgreSQL -The generated Quick Start runtime selects PostgreSQL and creates: +The generated Quick Start runtime creates: - A private `postgres:17-alpine` service with no published database port. - A named Docker volume for `/var/lib/postgresql/data`. - A generated 256-bit database password in `.sendium.env`, with mode `600` where the filesystem can enforce Unix permissions. - A PostgreSQL health check that gates Sendium startup. -- A readiness check that verifies the selected DLR schema is available. +- A readiness check that verifies the DLR schema is available. Run Quick Start normally: @@ -24,6 +24,12 @@ sh quick-start.sh Quick Start preserves the local database password during `--force` regeneration. PostgreSQL initialization variables cannot rotate the password of a role that already exists in a persistent data volume. +## Upgrade From MVStore Builds + +Older Sendium builds could store DLR state in `data/dlr-mvstore.db`. Current builds do not read or import that file. Before upgrading an MVStore-configured runtime, stop accepting submissions and allow pending provider correlations and unpushed downstream receipts to drain, or explicitly accept that the remaining state will be unavailable after the upgrade. Stop Sendium and preserve the old file before starting the PostgreSQL-only build. + +Provision PostgreSQL and require readiness to report `UP` with `backend=postgresql` before reopening traffic. State written to PostgreSQL is not available to an older MVStore build if the application is later downgraded. + ## External PostgreSQL To omit the local PostgreSQL service and connect Sendium to an operator-managed database, first export `SENDIUM_DLR_POSTGRESQL_PASSWORD` from an access-controlled secret source without placing its value in shell history. Then provide the URL and username when generating the runtime: @@ -38,12 +44,10 @@ unset SENDIUM_DLR_POSTGRESQL_PASSWORD The three values are an all-or-nothing override. Partial configuration is rejected instead of mixing local and external settings. -For a manual deployment, PostgreSQL selection and datasource activation default to the values below. A valid connection URL and the settings required by the database authentication method must still be supplied: +For a manual deployment, supply a valid connection URL and the settings required by the database authentication method: | Variable | Required value or example | Purpose | | :--- | :--- | :--- | -| `SENDIUM_DLR_STORAGE` | `postgresql` (default) | Selects the PostgreSQL storage adapter. | -| `SENDIUM_DLR_POSTGRESQL_ACTIVE` | `true` (default) | Activates the named datasource and its Flyway migrations. | | `SENDIUM_DLR_POSTGRESQL_JDBC_URL` | `jdbc:postgresql://db.example.com:5432/sendium` | JDBC connection URL. Add PostgreSQL JDBC TLS parameters for external networks. | | `SENDIUM_DLR_POSTGRESQL_USERNAME` | `sendium` | Database role used by Sendium and Flyway when required by the authentication method. | | `SENDIUM_DLR_POSTGRESQL_PASSWORD` | Secret value | Database password when required. Supply it through an access-controlled environment or secret mechanism. | @@ -72,7 +76,7 @@ Check readiness rather than only checking whether the HTTP listener is open: curl -fsS http://127.0.0.1:8080/q/health/ready ``` -The `sendium-dlr-storage` readiness check reports the selected backend. It returns `DOWN` with a sanitized `unavailable` reason when the selected PostgreSQL schema cannot be queried. +The `sendium-dlr-storage` readiness check reports `backend=postgresql`. It returns `DOWN` with a sanitized `unavailable` reason when the PostgreSQL schema cannot be queried. Inspect storage and datasource metrics with: @@ -80,9 +84,9 @@ Inspect storage and datasource metrics with: curl -fsS http://127.0.0.1:8080/q/metrics | grep -E 'sendium_dlr_storage|agroal' ``` -Relevant metrics include the selected backend and storage-operation latency/counts tagged by operation and success or error outcome. PostgreSQL pool metrics use the Agroal metric prefix. +Relevant metrics include storage-operation latency/counts tagged by backend, operation, and success or error outcome. PostgreSQL pool metrics use the Agroal metric prefix. -PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP submissions return the retryable `503` response and new SMPP submissions return `ESME_RSYSERR`; Sendium does not fall back to MVStore or memory. +PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP submissions return the retryable `503` response and new SMPP submissions return `ESME_RSYSERR`; Sendium does not fall back to local or in-memory storage. ## Retention @@ -108,48 +112,7 @@ Cleanup is triggered by storage activity and runs no more than once per hour. Th | HTTP DLR callback retry | The resolved callback is attempted up to 10 times while the process remains running. | The retry schedule is in memory and is lost on restart. There is no durable callback outbox. | | Database files | The Quick Start named volume survives normal container replacement and `docker compose down`. | Volume deletion, host-disk loss, and disaster recovery require backups or external PostgreSQL replication managed by the operator. | -These limits are intentional V1 boundaries. PostgreSQL replaces the existing DLR persistence store; it is not a durable queue, distributed claim coordinator, or delivery outbox. - -## MVStore Compatibility - -For a manual deployment that must remain on MVStore, use: - -```text -SENDIUM_DLR_STORAGE=mvstore -SENDIUM_DLR_POSTGRESQL_ACTIVE=false -SENDIUM_DLR_MVSTORE_PATH=/work/data/dlr-mvstore.db -``` - -The MVStore path must be on persistent storage. If the file cannot be opened, MVStore compatibility behavior can fall back to in-memory storage; check readiness data for `mode=persistent` rather than assuming the mount is working. - -## Cut Over From MVStore - -There is no MVStore-to-PostgreSQL importer, dual-read period, or live migration. Existing correlations and unpushed receipts do not move when the backend changes. - -1. Confirm whether pending DLR state can be allowed to expire or be abandoned. The safest compatibility choice is to remain on MVStore until a deliberate maintenance window is acceptable. -2. Stop accepting new HTTP and SMPP submissions. -3. Allow in-flight provider receipts and downstream replay to drain. The longest cleanup threshold is seven days, and cleanup is opportunistic rather than an exact deadline; continuous-traffic installations cannot obtain a lossless cutover without an importer. -4. Stop Sendium and back up the complete runtime, including `data/dlr-mvstore.db`. -5. Provision PostgreSQL, backups, access controls, and TLS where required. -6. Configure the five PostgreSQL variables described above and start exactly one Sendium instance. -7. Require `/q/health/ready` to report `UP` with `backend=postgresql` before reopening traffic. -8. Submit controlled HTTP and SMPP messages and verify provider correlation, callbacks, and downstream receipts. -9. Retain the MVStore backup and PostgreSQL database until the rollback decision window has closed. - -Provider receipts for messages that existed only in MVStore will be unknown after the switch. Do not run MVStore-backed and PostgreSQL-backed Sendium instances concurrently against the same traffic as a migration strategy. - -## Roll Back To MVStore - -Rollback is also non-seamless. State written to PostgreSQL is not copied back to MVStore. - -1. Stop accepting traffic and stop every Sendium instance. -2. Preserve the PostgreSQL database; do not drop its schema or volume. -3. Restore the previous MVStore file and runtime configuration. -4. Set `SENDIUM_DLR_STORAGE=mvstore` and `SENDIUM_DLR_POSTGRESQL_ACTIVE=false`. Remove the PostgreSQL URL and credentials from the Sendium container environment when they are no longer needed. -5. Start one Sendium instance and require readiness to report `backend=mvstore` and `mode=persistent`. -6. Reopen traffic only after controlled HTTP/SMPP checks pass. - -Messages accepted while PostgreSQL was active remain only in PostgreSQL. A later switch back to PostgreSQL can see still-retained PostgreSQL rows, so preserve both stores and record the exact cutover times during any rollback. +These limits are intentional V1 boundaries. PostgreSQL provides DLR persistence; it is not a durable queue, distributed claim coordinator, or delivery outbox. ## Related Documentation diff --git a/docs/DocumentationMap.md b/docs/DocumentationMap.md index a18750b..bb24aad 100644 --- a/docs/DocumentationMap.md +++ b/docs/DocumentationMap.md @@ -21,7 +21,7 @@ Sendium is an open-source, headless SMS gateway for high-throughput messaging. I | Start migrating from Kannel config | [Kannel migration converter](https://cytechmobile.github.io/sendium/) | | Understand releases and publishing | [11. Release Process](11-release-process.md) | | Review current features and roadmap | [12. Features And Roadmap](12-features-roadmap.md) | -| Configure PostgreSQL DLR persistence or plan a cutover | [13. DLR Persistence](13-dlr-persistence.md) | +| Configure PostgreSQL DLR persistence and review durability | [13. DLR Persistence](13-dlr-persistence.md) | | Contribute code or docs | [Contributing](../.github/CONTRIBUTING.md) | ## Core Concepts @@ -62,7 +62,7 @@ Sendium expects these files in the configured `conf` directory. The Docker quick | [10. Troubleshooting](10-troubleshooting.md) | Common startup, authentication, routing, SMPP, webhook, and logging issues. | | [11. Release Process](11-release-process.md) | Release Please flow, Conventional Commit rules, release PR handling, GitHub Packages, and Docker publishing. | | [12. Features And Roadmap](12-features-roadmap.md) | Current product capabilities, planned roadmap phases, and related feature documentation. | -| [13. DLR Persistence](13-dlr-persistence.md) | PostgreSQL and MVStore setup, retention, restart guarantees, cutover, rollback, and durability limits. | +| [13. DLR Persistence](13-dlr-persistence.md) | PostgreSQL setup, retention, restart guarantees, and durability limits. | | [Kannel migration converter](https://cytechmobile.github.io/sendium/) | Browser-only helper for turning a legacy `kannel.conf` into Sendium starter files. | ## API Discovery @@ -75,7 +75,7 @@ When Sendium is running, the HTTP API can be inspected through: | `/swagger-ui` | Interactive Swagger UI. | | `/openapi.json` | OpenAPI specification. | | `/q/metrics` | Prometheus-compatible Micrometer metrics endpoint. | -| `/q/health/ready` | Readiness status, including the selected DLR storage backend. | +| `/q/health/ready` | Readiness status, including PostgreSQL DLR storage availability. | ## Community And Project Files diff --git a/pom.xml b/pom.xml index 5d5244a..03fe3b4 100644 --- a/pom.xml +++ b/pom.xml @@ -37,12 +37,10 @@ false ${skipTests} ${skipTests} - false 7.2.2 3.27.7 5.23.0 - 2.4.240 @@ -73,11 +71,6 @@ ${mockito.version} test - - com.h2database - h2-mvstore - ${h2.version} - org.assertj assertj-core @@ -100,12 +93,4 @@ - - - postgresql-tests - - true - - - diff --git a/quick-start.sh b/quick-start.sh index ce788f6..12a2d60 100644 --- a/quick-start.sh +++ b/quick-start.sh @@ -420,7 +420,7 @@ else fi fi -mkdir -p "$target_dir/conf" "$target_dir/data" "$target_dir/logs" +mkdir -p "$target_dir/conf" "$target_dir/logs" staging_dir=$(mktemp -d "$target_dir/.quick-start.XXXXXX") || fail "could not create a staging directory" mkdir -p "$staging_dir/conf" install_started=false @@ -454,8 +454,6 @@ SENDIUM_HTTP_USER='$http_user' SENDIUM_HTTP_PASSWORD='$http_password' SENDIUM_SMPP_USER='$smpp_user' SENDIUM_SMPP_PASSWORD='$smpp_password' -SENDIUM_DLR_STORAGE='postgresql' -SENDIUM_DLR_POSTGRESQL_ACTIVE='true' SENDIUM_DLR_POSTGRESQL_JDBC_URL='$database_jdbc_url' SENDIUM_DLR_POSTGRESQL_USERNAME='$database_username' SENDIUM_DLR_POSTGRESQL_PASSWORD='$database_password' @@ -587,7 +585,6 @@ cat >> "$staging_dir/compose.yml" <<'EOF' - "127.0.0.1:27777:27777" volumes: - ./conf:/work/conf - - ./data:/work/data - ./logs:/work/logs EOF diff --git a/sendium-app/src/main/docker/Dockerfile.jvm b/sendium-app/src/main/docker/Dockerfile.jvm index 175a9bd..753aaf7 100644 --- a/sendium-app/src/main/docker/Dockerfile.jvm +++ b/sendium-app/src/main/docker/Dockerfile.jvm @@ -6,7 +6,7 @@ COPY target/quarkus-app/ ./ EXPOSE 8080 27777 -VOLUME ["/work/conf", "/work/logs", "/work/data"] +VOLUME ["/work/conf", "/work/logs"] # The "-Dquarkus.http.host=0.0.0.0" argument is important to make the application accessible from outside the container. CMD ["java", "-Dquarkus.http.host=0.0.0.0", "-jar", "quarkus-run.jar"] diff --git a/sendium-app/src/main/docker/Dockerfile.native b/sendium-app/src/main/docker/Dockerfile.native index d1c56be..dd736b8 100644 --- a/sendium-app/src/main/docker/Dockerfile.native +++ b/sendium-app/src/main/docker/Dockerfile.native @@ -11,8 +11,7 @@ RUN chmod 775 ./application EXPOSE 8080 27777 -VOLUME ["/work/conf", "/work/logs", "/work/data"] +VOLUME ["/work/conf", "/work/logs"] # The "-Dquarkus.http.host=0.0.0.0" argument is important to make the application accessible from outside the container. CMD ["./application", "-Dquarkus.http.host=0.0.0.0"] - diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index e9e5e20..b0fb63d 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -2,15 +2,10 @@ smsg.routing.file.path=conf/routingTable.conf smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml -%test.sendium.dlr.storage=mvstore -%test.quarkus.datasource.dlr.active=false - # DLR persistence. PostgreSQL requires valid named datasource connection settings. -sendium.dlr.storage=${SENDIUM_DLR_STORAGE:postgresql} -sendium.dlr.db.path=${SENDIUM_DLR_MVSTORE_PATH:data/dlr-mvstore.db} quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=${SENDIUM_DLR_POSTGRESQL_ACTIVE:true} +quarkus.datasource.dlr.active=true quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.url=${SENDIUM_DLR_POSTGRESQL_JDBC_URL:} quarkus.datasource.dlr.username=${SENDIUM_DLR_POSTGRESQL_USERNAME:} diff --git a/sendium-core/pom.xml b/sendium-core/pom.xml index f8fe1fe..fe16063 100644 --- a/sendium-core/pom.xml +++ b/sendium-core/pom.xml @@ -66,11 +66,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - com.h2database - h2-mvstore - - io.quarkus quarkus-junit @@ -157,7 +152,6 @@ org.jboss.logmanager.LogManager ${maven.home} - ${sendium.postgresql.tests} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java index 0a46533..5b3ed24 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java @@ -15,18 +15,12 @@ public class DlrStorageReadinessCheck implements HealthCheck { private static final String CHECK_NAME = "sendium-dlr-storage"; @Inject - ConfiguredDlrStorage storage; + ManagedDlrStorage storage; @Override public HealthCheckResponse call() { HealthCheckResponseBuilder response = HealthCheckResponse.named(CHECK_NAME) .withData("backend", storage.backend()); - if (!"postgresql".equals(storage.backend())) { - return response.up() - .withData("mode", storage.mode()) - .build(); - } - try { storage.verifyPostgresqlSchema(); return response.up().build(); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java similarity index 73% rename from sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java rename to sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java index 90c6532..dab74fb 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java @@ -10,7 +10,6 @@ import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -18,24 +17,20 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; -import java.util.Locale; import java.util.Optional; import java.util.function.Supplier; @Startup @ApplicationScoped -public class ConfiguredDlrStorage implements DlrStorage { +public class ManagedDlrStorage implements DlrStorage { private static final String METRIC_NAME = "sendium.dlr.storage.operation"; + private static final String BACKEND = "postgresql"; private static final String POSTGRESQL_PROBE_SQL = """ SELECT 1 FROM sendium_dlr.tracked_message WHERE FALSE """; - @Inject - @ConfigProperty(name = "sendium.dlr.storage", defaultValue = "postgresql") - String configuredBackend; - @Inject @ConfigProperty(name = "quarkus.flyway.dlr.active", defaultValue = "false") boolean flywayActive; @@ -44,9 +39,6 @@ public class ConfiguredDlrStorage implements DlrStorage { @ConfigProperty(name = "quarkus.flyway.dlr.migrate-at-start", defaultValue = "false") boolean flywayMigrateAtStart; - @Inject - Instance mvStoreStorage; - @Inject @DataSource("dlr") InjectableInstance postgresqlDataSource; @@ -56,47 +48,26 @@ public class ConfiguredDlrStorage implements DlrStorage { private DlrStorage delegate; private AgroalDataSource selectedPostgresqlDataSource; - private String backend; @PostConstruct void initialize() { - backend = configuredBackend.strip().toLowerCase(Locale.ROOT); boolean postgresqlActive = postgresqlDataSource.getHandle().getBean().isActive(); - delegate = switch (backend) { - case "mvstore" -> { - if (postgresqlActive || flywayActive || flywayMigrateAtStart) { - throw new IllegalStateException( - "The DLR PostgreSQL datasource and Flyway must be inactive when MVStore is selected"); - } - yield mvStoreStorage.get(); - } - case "postgresql" -> { - if (!postgresqlActive || !flywayActive || !flywayMigrateAtStart) { - throw new IllegalStateException( - "PostgreSQL DLR storage requires the active 'dlr' datasource and Flyway migration"); - } - selectedPostgresqlDataSource = postgresqlDataSource.get(); - yield new PostgresqlDlrStorage(selectedPostgresqlDataSource); - } - default -> throw new IllegalStateException("Unsupported DLR storage backend: " + backend); - }; + if (!postgresqlActive || !flywayActive || !flywayMigrateAtStart) { + throw new IllegalStateException( + "PostgreSQL DLR storage requires the active 'dlr' datasource and Flyway migration"); + } + selectedPostgresqlDataSource = postgresqlDataSource.get(); + delegate = new PostgresqlDlrStorage(selectedPostgresqlDataSource); Gauge.builder("sendium.dlr.storage.selected", this, ignored -> 1.0) - .description("Selected Sendium DLR storage backend") - .tag("backend", backend) + .description("Active Sendium DLR storage backend") + .tag("backend", BACKEND) .strongReference(true) .register(meterRegistry); } String backend() { - return backend; - } - - String mode() { - if (delegate instanceof MvStoreDlrStorage mvStore) { - return mvStore.isPersistent() ? "persistent" : "memory"; - } - return "persistent"; + return BACKEND; } void verifyPostgresqlSchema() throws SQLException { @@ -186,7 +157,7 @@ private void timed(String operation, Runnable action) { private Timer timer(String operation, String outcome) { return Timer.builder(METRIC_NAME) .description("Sendium DLR storage operation latency") - .tags("backend", backend, "operation", operation, "outcome", outcome) + .tags("backend", BACKEND, "operation", operation, "outcome", outcome) .register(meterRegistry); } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java deleted file mode 100644 index f247f9b..0000000 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MvStoreDlrStorage.java +++ /dev/null @@ -1,575 +0,0 @@ -package gr.cytech.sendium.core.worker; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import gr.cytech.sendium.core.message.StandardMessage; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Typed; -import jakarta.inject.Inject; -import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.h2.mvstore.MVStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - * Stores DLR correlation state and unpushed SMPP DLRs. - * - *

- * The service uses H2 MVStore when available and falls back to in-memory maps if the store cannot be opened. - * The primary/correlation maps track submitted messages until operator DLRs arrive. The unpushed-DLR maps - * persist DLRs that could not be delivered to a disconnected SMPP client, then replay them when the matching - * systemId reconnects. - * - *

- * This is an application-scoped singleton. The primary/correlation state follows the existing model of map-level - * concurrency: each operation is safe to call from worker threads, but multi-step updates are not globally serialized. - * Unpushed DLRs have stronger consistency requirements because each entry is split across payload, timestamp, and - * systemId index maps. Those compound operations are guarded by {@code unpushedDlrLock}. Replay also claims keys - * before returning them so concurrent reconnect callbacks for the same systemId cannot enqueue the same DLR twice. - */ -@ApplicationScoped -@Typed(MvStoreDlrStorage.class) -public class MvStoreDlrStorage implements DlrStorage { - private static final Logger logger = LoggerFactory.getLogger(MvStoreDlrStorage.class); - private static final long SEVEN_DAYS_MILLIS = TimeUnit.DAYS.toMillis(7); - private static final long THREE_DAYS_MILLIS = TimeUnit.DAYS.toMillis(3); - private static final long EXPIRY_CHECK_INTERVAL = TimeUnit.HOURS.toMillis(1); - - private static final String DB_PATH_PROPERTY = "sendium.dlr.db.path"; - private static final String DEFAULT_DB_PATH = "data/dlr-mvstore.db"; - private static final ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private static final TypeReference> STRING_LIST_TYPE = new TypeReference<>() { - }; - - private final Object unpushedDlrStateLock = new Object(); - private final Set claimedUnpushedDlrKeys = ConcurrentHashMap.newKeySet(); - - @Inject - @ConfigProperty(name = DB_PATH_PROPERTY, defaultValue = DEFAULT_DB_PATH) - private String configuredDbPath; - - private MVStore store; - - private Map primaryStore; - private Map correlationIndex; - private Map primaryTimestamps; - private Map correlationTimestamps; - private Map unpushedDlrStore; - private Map unpushedDlrTimestamps; - private Map unpushedDlrIndex; - - private volatile long lastExpiryCheck = 0; - @SuppressWarnings("unused") - private volatile boolean initialized = false; - - @PostConstruct - void init() { - String dbPath = configuredDbPath != null ? - configuredDbPath - : System.getProperty(DB_PATH_PROPERTY, DEFAULT_DB_PATH); - File dbFile = new File(dbPath); - File dbDir = dbFile.getParentFile(); - - if (dbDir != null && !dbDir.exists()) { - boolean created = dbDir.mkdirs(); - if (created) { - logger.info("Created DLR database directory: {}", dbDir.getAbsolutePath()); - } - } - - try { - if (dbFile.exists() && dbFile.length() > 0) { - store = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) - .autoCommitBufferSize(1024) - .open(); - logger.info("Opened existing DLR database: " + dbPath); - } else { - store = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) - .open(); - logger.info("Created new DLR database: " + dbPath); - } - - primaryStore = store.openMap("primaryStore"); - correlationIndex = store.openMap("correlationIndex"); - primaryTimestamps = store.openMap("primaryTimestamps"); - correlationTimestamps = store.openMap("correlationTimestamps"); - unpushedDlrStore = store.openMap("unpushedDlrStore"); - unpushedDlrTimestamps = store.openMap("unpushedDlrTimestamps"); - unpushedDlrIndex = store.openMap("unpushedDlrIndex"); - - if (primaryStore == null || correlationIndex == null || unpushedDlrStore == null || unpushedDlrIndex == null) { - logger.error("Failed to load maps from DB, falling back to in-memory"); - fallbackToInMemory(); - } else { - logger.info("Loaded from DB - primaryStore: {}, correlationIndex: {}, unpushedDlrStore: {}, unpushedDlrIndex: {}", - primaryStore.size(), correlationIndex.size(), unpushedDlrStore.size(), unpushedDlrIndex.size()); - initialized = true; - } - } catch (Exception e) { - logger.warn("Failed to initialize MVStore, falling back to in-memory: ", e); - fallbackToInMemory(); - } - - if (!initialized) { - fallbackToInMemory(); - } - } - - private void fallbackToInMemory() { - store = null; - primaryStore = new ConcurrentHashMap<>(); - correlationIndex = new ConcurrentHashMap<>(); - primaryTimestamps = new ConcurrentHashMap<>(); - correlationTimestamps = new ConcurrentHashMap<>(); - unpushedDlrStore = new ConcurrentHashMap<>(); - unpushedDlrTimestamps = new ConcurrentHashMap<>(); - unpushedDlrIndex = new ConcurrentHashMap<>(); - initialized = true; - logger.info("Using in-memory mode (no persistence)"); - } - - @PreDestroy - void onStop() { - logger.info("MvStoreDlrStorage shutting down"); - saveAndClose(); - } - - private synchronized void saveAndClose() { - if (store != null && !store.isClosed()) { - try { - store.commit(); - logger.info("Saved DLR database"); - } catch (Exception e) { - logger.warn("Failed to commit DB: {}", e.getMessage()); - } - try { - store.close(); - logger.info("Closed DLR database"); - } catch (Exception e) { - logger.warn("Failed to close DB: {}", e.getMessage()); - } - } - } - - @Override - public void saveInitialState(MessageState context) { - if (primaryStore != null) { - checkExpiry(); - try { - String json = mapper.writeValueAsString(context); - primaryStore.put(context.getGatewayMsgId(), json); - primaryTimestamps.put(context.getGatewayMsgId(), System.currentTimeMillis()); - } catch (JsonProcessingException e) { - logger.error("Failed to serialize MessageState for gatewayMsgId: {}", context.getGatewayMsgId(), e); - } - } - } - - @Override - public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { - checkExpiry(); - if (primaryStore == null || correlationIndex == null) { - return; - } - - int maxRetries = 20; - long retryIntervalMs = 200; - MessageState state = null; - - for (int i = 0; i < maxRetries; i++) { - String stateJson = primaryStore.get(gatewayMsgId); - if (stateJson != null) { - try { - // Deserialize back to object - state = mapper.readValue(stateJson, MessageState.class); - break; // State found and parsed, exit the retry loop - } catch (JsonProcessingException e) { - logger.error("Failed to deserialize MessageState for gatewayMsgId: {}", gatewayMsgId, e); - break; - } - } - try { - Thread.sleep(retryIntervalMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Thread interrupted while retrying for gatewayMsgId: {}", gatewayMsgId); - break; - } - } - - // Check if we successfully found the state after the retries - if (state != null) { - state.setOperatorMsgId(operatorMsgId); - state.setStatus(MessageState.MessageStatus.SENT); - state.setTimestamp(System.currentTimeMillis()); - try { - primaryStore.put(gatewayMsgId, mapper.writeValueAsString(state)); - correlationIndex.put(operatorMsgId, gatewayMsgId); - correlationTimestamps.put(operatorMsgId, System.currentTimeMillis()); - } catch (JsonProcessingException e) { - logger.error("Failed to serialize updated MessageState", e); - } - } else { - logger.warn("GatewayMsgId not found for linking after {} retries: {}", maxRetries, gatewayMsgId); - } - } - - @Override - public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { - checkExpiry(); - if (correlationIndex == null || primaryStore == null) { - return Optional.empty(); - } - String gatewayMsgId = correlationIndex.get(operatorMsgId); - if (gatewayMsgId == null) { - logger.warn("No gatewayMsgId found for operatorMsgId: {} (expired or unknown)", operatorMsgId); - return Optional.empty(); - } - - String stateJson = primaryStore.get(gatewayMsgId); - if (stateJson != null) { - try { - MessageState state = mapper.readValue(stateJson, MessageState.class); - state.setTimestamp(System.currentTimeMillis()); - state.setStatus(status); - - primaryStore.remove(gatewayMsgId); - primaryTimestamps.remove(gatewayMsgId); - correlationIndex.remove(operatorMsgId); - correlationTimestamps.remove(operatorMsgId); - logger.debug("Resolved and removed DLR for gatewayMsgId: {}", gatewayMsgId); - - return Optional.of(state); - } catch (JsonProcessingException e) { - logger.error("Failed to deserialize MessageState during resolve", e); - } - } - - logger.warn("MessageState not found for gatewayMsgId: {}", gatewayMsgId); - return Optional.empty(); - } - - @Override - public Optional getState(String gatewayMsgId) { - checkExpiry(); - if (primaryStore == null) { - return Optional.empty(); - } - - String stateJson = primaryStore.get(gatewayMsgId); - if (stateJson != null) { - try { - return Optional.of(mapper.readValue(stateJson, MessageState.class)); - } catch (JsonProcessingException e) { - logger.error("Failed to deserialize MessageState in getState", e); - } - } - return Optional.empty(); - } - - @Override - public boolean markAsFailed(String gatewayMsgId) { - checkExpiry(); - if (primaryStore == null) { - return false; - } - - String stateJson = primaryStore.get(gatewayMsgId); - if (stateJson != null) { - try { - MessageState state = mapper.readValue(stateJson, MessageState.class); - state.setStatus(MessageState.MessageStatus.FAILED); - state.setTimestamp(System.currentTimeMillis()); - primaryStore.put(gatewayMsgId, mapper.writeValueAsString(state)); - return true; - } catch (JsonProcessingException e) { - logger.error("Failed to process MessageState in markAsFailed", e); - } - } - return false; - } - - /** - * Persist a DLR that could not be pushed to the SMPP client. - */ - @Override - public boolean saveUnpushedDlr(StandardMessage msg) { - checkExpiry(); - if (unpushedDlrStore == null || unpushedDlrIndex == null || msg == null || msg.type != StandardMessage.MSG_DLR || - msg.systemId == null || msg.systemId.isBlank()) { - return false; - } - - String key = getUnpushedDlrKey(msg); - synchronized (unpushedDlrStateLock) { - try { - unpushedDlrStore.put(key, mapper.writeValueAsString(UnpushedDlr.fromMessage(msg))); - unpushedDlrTimestamps.put(key, System.currentTimeMillis()); - addKeyToUnpushedDlrIndex(msg.systemId, key); - commitStore(); - logger.info("Saved unpushed DLR key: {}", key); - return true; - } catch (JsonProcessingException e) { - logger.error("Failed to serialize unpushed DLR key: {}", key, e); - return false; - } - } - } - - /** - * Load unpushed DLRs for one SMPP systemId without marking them for replay. - */ - @Override - public List getUnpushedDlrs(String systemId) { - return loadUnpushedDlrs(systemId, false); - } - - /** - * Load and claim unpushed DLRs for replay. Claimed entries are hidden from later claims until removed or released. - */ - @Override - public List claimUnpushedDlrs(String systemId) { - return loadUnpushedDlrs(systemId, true); - } - - private List loadUnpushedDlrs(String systemId, boolean claimForReplay) { - checkExpiry(); - List messages = new ArrayList<>(); - if (unpushedDlrStore == null || unpushedDlrIndex == null || systemId == null || systemId.isBlank()) { - return messages; - } - - boolean changed = false; - synchronized (unpushedDlrStateLock) { - for (String key : getUnpushedDlrKeys(systemId)) { - if (claimForReplay && claimedUnpushedDlrKeys.contains(key)) { - continue; - } - - String msgJson = unpushedDlrStore.get(key); - if (msgJson == null) { - removeKeyFromUnpushedDlrIndex(systemId, key); - changed = true; - continue; - } - try { - UnpushedDlr dlr = mapper.readValue(msgJson, UnpushedDlr.class); - if (isUnpushedDlrForConnection(dlr, systemId)) { - if (!claimForReplay || claimedUnpushedDlrKeys.add(key)) { - messages.add(dlr.toMessage()); - } - } else { - removeKeyFromUnpushedDlrIndex(systemId, key); - changed = true; - } - } catch (JsonProcessingException e) { - logger.error("Failed to deserialize unpushed DLR key: {}. Removing corrupt entry", key, e); - unpushedDlrStore.remove(key); - unpushedDlrTimestamps.remove(key); - claimedUnpushedDlrKeys.remove(key); - removeKeyFromUnpushedDlrIndex(systemId, key); - changed = true; - } - } - if (changed) { - commitStore(); - } - } - - return messages; - } - - /** - * Remove a replayed DLR from all unpushed-DLR maps. - */ - @Override - public boolean removeUnpushedDlr(StandardMessage msg) { - if (unpushedDlrStore == null || unpushedDlrIndex == null || msg == null || msg.systemId == null || msg.systemId.isBlank()) { - return false; - } - - String key = getUnpushedDlrKey(msg); - synchronized (unpushedDlrStateLock) { - final boolean removed = unpushedDlrStore.remove(key) != null; - unpushedDlrTimestamps.remove(key); - claimedUnpushedDlrKeys.remove(key); - removeKeyFromUnpushedDlrIndex(msg.systemId, key); - if (removed) { - commitStore(); - } - return removed; - } - } - - /** - * Make a claimed but not yet removed DLR eligible for a later replay attempt. - */ - @Override - public void releaseUnpushedDlrClaim(StandardMessage msg) { - if (msg == null || msg.systemId == null || msg.systemId.isBlank()) { - return; - } - - synchronized (unpushedDlrStateLock) { - claimedUnpushedDlrKeys.remove(getUnpushedDlrKey(msg)); - } - } - - private boolean isUnpushedDlrForConnection(UnpushedDlr dlr, String systemId) { - return dlr != null && dlr.systemId != null && dlr.systemId.equals(systemId); - } - - private String getUnpushedDlrKey(StandardMessage msg) { - return String.join("|", - nullToEmpty(msg.systemId), - nullToEmpty(msg.serial), - String.valueOf(msg.state), - nullToEmpty(msg.errcode), - String.valueOf(msg.msgId)); - } - - private void addKeyToUnpushedDlrIndex(String systemId, String key) throws JsonProcessingException { - List keys = getUnpushedDlrKeys(systemId); - if (!keys.contains(key)) { - keys.add(key); - unpushedDlrIndex.put(systemId, mapper.writeValueAsString(keys)); - } - } - - private List getUnpushedDlrKeys(String systemId) { - String keysJson = unpushedDlrIndex.get(systemId); - if (keysJson == null || keysJson.isBlank()) { - return new ArrayList<>(); - } - try { - return new ArrayList<>(mapper.readValue(keysJson, STRING_LIST_TYPE)); - } catch (JsonProcessingException e) { - logger.error("Failed to deserialize unpushed DLR index for systemId: {}. Clearing corrupt index", systemId, e); - unpushedDlrIndex.remove(systemId); - return new ArrayList<>(); - } - } - - private void removeKeyFromUnpushedDlrIndex(String systemId, String key) { - if (systemId == null || unpushedDlrIndex == null) { - return; - } - List keys = getUnpushedDlrKeys(systemId); - if (!keys.remove(key)) { - return; - } - if (keys.isEmpty()) { - unpushedDlrIndex.remove(systemId); - return; - } - try { - unpushedDlrIndex.put(systemId, mapper.writeValueAsString(keys)); - } catch (JsonProcessingException e) { - logger.error("Failed to serialize unpushed DLR index for systemId: {}. Clearing index", systemId, e); - unpushedDlrIndex.remove(systemId); - } - } - - private String getSystemIdFromUnpushedDlrKey(String key) { - int separator = key.indexOf('|'); - return separator >= 0 ? key.substring(0, separator) : key; - } - - private String nullToEmpty(String value) { - return value == null ? "" : value; - } - - private void commitStore() { - if (store != null && !store.isClosed()) { - store.commit(); - } - } - - private synchronized void checkExpiry() { - long now = System.currentTimeMillis(); - if (now - lastExpiryCheck < EXPIRY_CHECK_INTERVAL) { - return; - } - lastExpiryCheck = now; - - if (primaryStore == null || primaryTimestamps == null) { - return; - } - - for (String key : primaryTimestamps.keySet()) { - Long ts = primaryTimestamps.get(key); - if (ts != null && (now - ts) > SEVEN_DAYS_MILLIS) { - primaryStore.remove(key); - primaryTimestamps.remove(key); - logger.debug("Expired primary entry: {}", key); - } - } - - if (correlationIndex != null && correlationTimestamps != null) { - for (String key : correlationTimestamps.keySet()) { - Long ts = correlationTimestamps.get(key); - if (ts != null && (now - ts) > THREE_DAYS_MILLIS) { - correlationIndex.remove(key); - correlationTimestamps.remove(key); - logger.debug("Expired correlation entry: {}", key); - } - } - } - - boolean removedExpired = false; - if (unpushedDlrStore != null && unpushedDlrTimestamps != null) { - synchronized (unpushedDlrStateLock) { - for (String key : unpushedDlrTimestamps.keySet()) { - Long ts = unpushedDlrTimestamps.get(key); - if (ts != null && (now - ts) > SEVEN_DAYS_MILLIS) { - unpushedDlrStore.remove(key); - unpushedDlrTimestamps.remove(key); - claimedUnpushedDlrKeys.remove(key); - removeKeyFromUnpushedDlrIndex(getSystemIdFromUnpushedDlrKey(key), key); - removedExpired = true; - logger.debug("Expired unpushed DLR entry: {}", key); - } - } - } - } - if (removedExpired) { - commitStore(); - } - } - - public int getPrimaryStoreSize() { - return primaryStore != null ? primaryStore.size() : 0; - } - - public int getCorrelationIndexSize() { - return correlationIndex != null ? correlationIndex.size() : 0; - } - - public int getUnpushedDlrStoreSize() { - return unpushedDlrStore != null ? unpushedDlrStore.size() : 0; - } - - public int getUnpushedDlrIndexSize() { - return unpushedDlrIndex != null ? unpushedDlrIndex.size() : 0; - } - - public boolean isPersistent() { - return store != null && !store.isClosed(); - } -} diff --git a/sendium-core/src/main/resources/application.properties b/sendium-core/src/main/resources/application.properties index b753c86..2739297 100644 --- a/sendium-core/src/main/resources/application.properties +++ b/sendium-core/src/main/resources/application.properties @@ -1,12 +1,8 @@ %test.smsg.routing.file.path=src/test/resources/routingTable.conf %test.smsg.properties.file.path=src/test/resources/smsg.properties %test.smsg.credentials.file.path=src/test/resources/credentials.yml -%test.sendium.dlr.storage=mvstore -%test.sendium.dlr.db.path=${java.io.tmpdir}/sendium-dlr-${quarkus.uuid}.db -%test.quarkus.datasource.dlr.active=false -# PostgreSQL is the application default. Tests explicitly retain Docker-free MVStore. -sendium.dlr.storage=postgresql +# PostgreSQL is the only DLR persistence backend. quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql quarkus.datasource.dlr.active=true diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java similarity index 98% rename from sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java index 7081184..a7591e8 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java @@ -5,7 +5,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.testcontainers.postgresql.PostgreSQLContainer; import java.sql.Connection; @@ -20,8 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") -class PostgresqlMigrationTest { +class PostgresqlMigrationIT { private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; private static final UUID INVALID_STATUS_GATEWAY_ID = UUID.fromString("00000000-0000-0000-0000-000000000001"); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java index 62a781e..5c03026 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceIT.java @@ -4,8 +4,10 @@ import gr.cytech.sendium.core.queue.Queue; import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.MessageState; +import gr.cytech.sendium.core.worker.PostgresqlDlrQuarkusTestResource; import gr.cytech.sendium.routing.OutgoingWorkerManager; import gr.cytech.sendium.routing.StandardOutgoingWorkerHandler; +import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusTest; import jakarta.enterprise.inject.spi.CDI; import org.junit.jupiter.api.BeforeAll; @@ -22,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; @QuarkusTest +@QuarkusTestResource(value = PostgresqlDlrQuarkusTestResource.class, restrictToAnnotatedClass = true) class KannelResourceIT { static StandardOutgoingWorkerHandler outgoingWorkerHandler; static DlrService dlrService; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceIT.java similarity index 78% rename from sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceIT.java index 270f284..166369b 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/message/StandardMessageJsonResourceIT.java @@ -1,5 +1,7 @@ package gr.cytech.sendium.core.message; +import gr.cytech.sendium.core.worker.PostgresqlDlrQuarkusTestResource; +import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusTest; import org.junit.jupiter.api.Test; @@ -7,7 +9,8 @@ import static org.hamcrest.Matchers.equalTo; @QuarkusTest -class StandardMessageJsonResourceTest { +@QuarkusTestResource(value = PostgresqlDlrQuarkusTestResource.class, restrictToAnnotatedClass = true) +class StandardMessageJsonResourceIT { @Test void shouldDeserializeStandardMessageWithPrimitiveByteFields() { given() diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java deleted file mode 100644 index 44cd364..0000000 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ConfiguredDlrStorageTest.java +++ /dev/null @@ -1,143 +0,0 @@ -package gr.cytech.sendium.core.worker; - -import io.agroal.api.AgroalDataSource; -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; -import io.quarkus.arc.InjectableInstance; -import jakarta.enterprise.inject.Instance; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class ConfiguredDlrStorageTest { - private Instance mvStoreInstance; - private MvStoreDlrStorage mvStore; - private InjectableInstance postgresqlDataSource; - private SimpleMeterRegistry meterRegistry; - private ConfiguredDlrStorage storage; - - @BeforeEach - @SuppressWarnings("unchecked") - void setUp() { - mvStoreInstance = mock(Instance.class); - mvStore = mock(MvStoreDlrStorage.class); - postgresqlDataSource = mock(InjectableInstance.class, RETURNS_DEEP_STUBS); - meterRegistry = new SimpleMeterRegistry(); - when(mvStoreInstance.get()).thenReturn(mvStore); - when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(false); - - storage = new ConfiguredDlrStorage(); - storage.configuredBackend = "mvstore"; - storage.mvStoreStorage = mvStoreInstance; - storage.postgresqlDataSource = postgresqlDataSource; - storage.meterRegistry = meterRegistry; - } - - @Test - void selectsMvStoreAndRecordsLowCardinalityMetrics() { - when(mvStore.getState("sensitive-gateway-id")).thenReturn(Optional.empty()); - - storage.initialize(); - storage.getState("sensitive-gateway-id"); - - verify(mvStore).getState("sensitive-gateway-id"); - assertThat(storage.backend()).isEqualTo("mvstore"); - assertThat(meterRegistry.find("sendium.dlr.storage.selected") - .tag("backend", "mvstore").gauge().value()).isEqualTo(1.0); - assertThat(meterRegistry.find("sendium.dlr.storage.operation") - .tags("backend", "mvstore", "operation", "get_state", "outcome", "success") - .timer().count()).isOne(); - assertThat(meterRegistry.getMeters()) - .flatExtracting(meter -> meter.getId().getTags()) - .noneMatch(tag -> tag.getValue().contains("sensitive")); - } - - @Test - void recordsThrownStorageFailureAsError() { - when(mvStore.markAsFailed("gateway-id")).thenThrow(new DlrStorageException("failure")); - storage.initialize(); - - assertThatThrownBy(() -> storage.markAsFailed("gateway-id")) - .isInstanceOf(DlrStorageException.class); - - assertThat(meterRegistry.find("sendium.dlr.storage.operation") - .tags("backend", "mvstore", "operation", "mark_failed", "outcome", "error") - .timer().count()).isOne(); - } - - @Test - void delegatesBatchSavesAndRecordsMetrics() { - List states = List.of( - new MessageState("gateway-id", "system", "source", "destination", null)); - storage.initialize(); - - storage.saveInitialStates(states); - - verify(mvStore).saveInitialStates(states); - assertThat(meterRegistry.find("sendium.dlr.storage.operation") - .tags("backend", "mvstore", "operation", "save_initial_batch", "outcome", "success") - .timer().count()).isOne(); - } - - @Test - void rejectsUnknownBackend() { - storage.configuredBackend = "unknown"; - - assertThatThrownBy(storage::initialize) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Unsupported DLR storage backend: unknown"); - } - - @Test - void rejectsActivePostgresqlDatasourceForMvStore() { - when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); - - assertThatThrownBy(storage::initialize) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("must be inactive"); - } - - @Test - void rejectsInactivePostgresqlDatasourceWhenSelected() { - storage.configuredBackend = "postgresql"; - - assertThatThrownBy(storage::initialize) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); - } - - @Test - void rejectsPostgresqlDatasourceWithoutFlyway() { - when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); - storage.configuredBackend = "postgresql"; - - assertThatThrownBy(storage::initialize) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); - } - - @Test - void selectsActivePostgresqlDatasourceWithoutMvStoreFallback() { - AgroalDataSource dataSource = mock(AgroalDataSource.class); - when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); - when(postgresqlDataSource.get()).thenReturn(dataSource); - storage.configuredBackend = " POSTGRESQL "; - storage.flywayActive = true; - storage.flywayMigrateAtStart = true; - - storage.initialize(); - - assertThat(storage.backend()).isEqualTo("postgresql"); - assertThat(storage.mode()).isEqualTo("persistent"); - verify(postgresqlDataSource).get(); - verify(mvStoreInstance, org.mockito.Mockito.never()).get(); - } -} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java index 4065b1b..11311f2 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheckTest.java @@ -12,24 +12,9 @@ import static org.mockito.Mockito.when; class DlrStorageReadinessCheckTest { - @Test - void reportsMvStoreMode() { - ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); - when(storage.backend()).thenReturn("mvstore"); - when(storage.mode()).thenReturn("memory"); - DlrStorageReadinessCheck check = new DlrStorageReadinessCheck(); - check.storage = storage; - - HealthCheckResponse response = check.call(); - Map data = response.getData().orElseThrow(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); - assertThat(data).containsEntry("backend", "mvstore").containsEntry("mode", "memory"); - } - @Test void reportsPostgresqlSchemaAsReady() throws SQLException { - ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); + ManagedDlrStorage storage = mock(ManagedDlrStorage.class); when(storage.backend()).thenReturn("postgresql"); DlrStorageReadinessCheck check = new DlrStorageReadinessCheck(); check.storage = storage; @@ -43,7 +28,7 @@ void reportsPostgresqlSchemaAsReady() throws SQLException { @Test void reportsSanitizedPostgresqlFailure() throws SQLException { - ConfiguredDlrStorage storage = mock(ConfiguredDlrStorage.class); + ManagedDlrStorage storage = mock(ManagedDlrStorage.class); when(storage.backend()).thenReturn("postgresql"); doThrow(new SQLException("jdbc:postgresql://secret-host/database")) .when(storage).verifyPostgresqlSchema(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java deleted file mode 100644 index 1bcf6b1..0000000 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrStorageRuntimeTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package gr.cytech.sendium.core.worker; - -import io.micrometer.core.instrument.MeterRegistry; -import io.quarkus.test.junit.QuarkusTest; -import jakarta.enterprise.inject.Instance; -import jakarta.inject.Inject; -import org.junit.jupiter.api.Test; - -import java.util.UUID; - -import static io.restassured.RestAssured.given; -import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -@QuarkusTest -class DlrStorageRuntimeTest { - @Inject - Instance storageInstance; - - @Inject - ConfiguredDlrStorage configuredStorage; - - @Inject - MeterRegistry meterRegistry; - - @Test - void selectsExactlyOneMvStoreBackendInTestProfile() { - assertThat(storageInstance.isResolvable()).isTrue(); - assertThat(storageInstance.stream()).hasSize(1); - assertThat(storageInstance.get()).isSameAs(configuredStorage); - assertThat(configuredStorage.backend()).isEqualTo("mvstore"); - } - - @Test - void exposesReadinessAndStorageMetrics() { - configuredStorage.getState(UUID.randomUUID().toString()); - - given() - .when().get("/q/health/ready") - .then() - .statusCode(200) - .body("status", equalTo("UP")) - .body("checks.find { it.name == 'sendium-dlr-storage' }.data.backend", equalTo("mvstore")); - - given() - .when().get("/q/metrics") - .then() - .statusCode(200) - .body(containsString("sendium_dlr_storage_selected")) - .body(containsString("sendium_dlr_storage_operation_seconds_count")); - - assertThat(meterRegistry.find("sendium.dlr.storage.operation") - .tag("operation", "get_state").timer().count()).isGreaterThanOrEqualTo(1); - } -} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ManagedDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ManagedDlrStorageTest.java new file mode 100644 index 0000000..76a956d --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ManagedDlrStorageTest.java @@ -0,0 +1,60 @@ +package gr.cytech.sendium.core.worker; + +import io.agroal.api.AgroalDataSource; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import io.quarkus.arc.InjectableInstance; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ManagedDlrStorageTest { + private InjectableInstance postgresqlDataSource; + private ManagedDlrStorage storage; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + postgresqlDataSource = mock(InjectableInstance.class, RETURNS_DEEP_STUBS); + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(false); + + storage = new ManagedDlrStorage(); + storage.postgresqlDataSource = postgresqlDataSource; + storage.meterRegistry = new SimpleMeterRegistry(); + } + + @Test + void rejectsInactivePostgresqlDatasource() { + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); + } + + @Test + void rejectsPostgresqlDatasourceWithoutFlyway() { + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); + + assertThatThrownBy(storage::initialize) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requires the active 'dlr' datasource and Flyway"); + } + + @Test + void initializesActivePostgresqlDatasource() { + AgroalDataSource dataSource = mock(AgroalDataSource.class); + when(postgresqlDataSource.getHandle().getBean().isActive()).thenReturn(true); + when(postgresqlDataSource.get()).thenReturn(dataSource); + storage.flywayActive = true; + storage.flywayMigrateAtStart = true; + + storage.initialize(); + + assertThat(storage.backend()).isEqualTo("postgresql"); + verify(postgresqlDataSource).get(); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java deleted file mode 100644 index f0b9385..0000000 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MvStoreDlrStorageTest.java +++ /dev/null @@ -1,288 +0,0 @@ -package gr.cytech.sendium.core.worker; - -import gr.cytech.sendium.core.message.StandardMessage; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.*; - -class MvStoreDlrStorageTest { - - private MvStoreDlrStorage storage; - private Path dbPath; - private String oldDbPath; - - @BeforeEach - void setUp() throws Exception { - oldDbPath = System.getProperty("sendium.dlr.db.path"); - dbPath = Files.createTempFile("dlr-service-test", ".db"); - Files.deleteIfExists(dbPath); - System.setProperty("sendium.dlr.db.path", dbPath.toString()); - storage = new MvStoreDlrStorage(); - storage.init(); - } - - @AfterEach - void tearDown() throws Exception { - if (storage != null) { - storage.onStop(); - } - if (oldDbPath == null) { - System.clearProperty("sendium.dlr.db.path"); - } else { - System.setProperty("sendium.dlr.db.path", oldDbPath); - } - if (dbPath != null) { - Files.deleteIfExists(dbPath); - } - } - - @Test - void saveInitialState_StoresInPrimaryStore() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - - storage.saveInitialState(state); - - assertEquals(1, storage.getPrimaryStoreSize()); - } - - @Test - void saveInitialState_SetsTimestamp() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - - storage.saveInitialState(state); - - Optional retrieved = storage.getState("gw-123"); - assertTrue(retrieved.isPresent()); - assertTrue(retrieved.get().getTimestamp() > 0); - } - - @Test - void linkOperatorId_LinksCorrelation() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - storage.saveInitialState(state); - - storage.linkOperatorId("gw-123", "op-456"); - - assertEquals(1, storage.getCorrelationIndexSize()); - } - - @Test - void linkOperatorId_UpdatesStatusToSent() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - storage.saveInitialState(state); - - storage.linkOperatorId("gw-123", "op-456"); - - Optional retrieved = storage.getState("gw-123"); - assertTrue(retrieved.isPresent()); - assertEquals(MessageState.MessageStatus.SENT, retrieved.get().getStatus()); - } - - @Test - void resolveAndRemoveDlr_ReturnsAndRemoves() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - storage.saveInitialState(state); - storage.linkOperatorId("gw-123", "op-456"); - - long beforeResolve = System.currentTimeMillis(); - Optional result = storage.resolveAndRemoveDlr( - "op-456", MessageState.MessageStatus.DELIVERED); - - assertTrue(result.isPresent()); - assertEquals(MessageState.MessageStatus.DELIVERED, result.get().getStatus()); - assertEquals("op-456", result.get().getOperatorMsgId()); - assertTrue(result.get().getTimestamp() >= beforeResolve); - assertEquals(0, storage.getPrimaryStoreSize()); - assertEquals(0, storage.getCorrelationIndexSize()); - } - - @Test - void resolveAndRemoveDlr_MissingId_ReturnsEmpty() { - Optional result = storage.resolveAndRemoveDlr( - "unknown", MessageState.MessageStatus.DELIVERED); - - assertTrue(result.isEmpty()); - } - - @Test - void getState_ReturnsWrappedState() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - storage.saveInitialState(state); - - Optional result = storage.getState("gw-123"); - - assertTrue(result.isPresent()); - assertEquals("gw-123", result.get().getGatewayMsgId()); - } - - @Test - void getState_MissingId_ReturnsEmpty() { - Optional result = storage.getState("unknown"); - - assertTrue(result.isEmpty()); - } - - @Test - void markAsFailed_UpdatesStatusToFailed() { - MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - storage.saveInitialState(state); - - boolean result = storage.markAsFailed("gw-123"); - - assertTrue(result); - Optional updated = storage.getState("gw-123"); - assertTrue(updated.isPresent()); - assertEquals(MessageState.MessageStatus.FAILED, updated.get().getStatus()); - } - - @Test - void markAsFailed_MissingId_ReturnsFalse() { - boolean result = storage.markAsFailed("unknown"); - - assertFalse(result); - } - - @Test - void saveUnpushedDlr_StoresAndReturnsMatchingDlr() { - StandardMessage dlr = createDlr("account1", "sys1"); - - boolean result = storage.saveUnpushedDlr(dlr); - List dlrs = storage.getUnpushedDlrs("sys1"); - - assertTrue(result); - assertTrue(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); - StandardMessage stored = dlrs.getFirst(); - assertEquals(dlr.state, stored.state); - assertEquals(dlr.errcode, stored.errcode); - assertEquals(dlr.acked, stored.acked); - assertEquals(dlr.priority, stored.priority); - assertEquals(dlr.reassembledParts, stored.reassembledParts); - assertEquals(1, storage.getUnpushedDlrIndexSize()); - } - - @Test - void saveUnpushedDlr_BlankSystemIdReturnsFalse() { - StandardMessage dlr = createDlr("account1", null); - - boolean result = storage.saveUnpushedDlr(dlr); - - assertFalse(result); - } - - @Test - void getUnpushedDlrs_DifferentSystemIdDoesNotMatch() { - StandardMessage dlr = createDlr("account1", "sys1"); - storage.saveUnpushedDlr(dlr); - - List dlrs = storage.getUnpushedDlrs("sys2"); - - assertFalse(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); - } - - @Test - void getUnpushedDlrs_UsesSystemIdIndex() { - StandardMessage sys1Dlr = createDlr("account1", "sys1"); - StandardMessage sys2Dlr = createDlr("account2", "sys2"); - storage.saveUnpushedDlr(sys1Dlr); - storage.saveUnpushedDlr(sys2Dlr); - - List dlrs = storage.getUnpushedDlrs("sys1"); - - assertEquals(2, storage.getUnpushedDlrIndexSize()); - assertTrue(dlrs.stream().anyMatch(msg -> sys1Dlr.serial.equals(msg.serial))); - assertFalse(dlrs.stream().anyMatch(msg -> sys2Dlr.serial.equals(msg.serial))); - } - - @Test - void removeUnpushedDlr_RemovesStoredDlr() { - StandardMessage dlr = createDlr("account1", "sys1"); - storage.saveUnpushedDlr(dlr); - - boolean result = storage.removeUnpushedDlr(dlr); - List dlrs = storage.getUnpushedDlrs("sys1"); - - assertTrue(result); - assertFalse(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); - assertEquals(0, storage.getUnpushedDlrIndexSize()); - } - - @Test - void claimUnpushedDlrs_HidesClaimedDlrUntilReleased() { - StandardMessage dlr = createDlr("account1", "sys1"); - storage.saveUnpushedDlr(dlr); - - List firstClaim = storage.claimUnpushedDlrs("sys1"); - List secondClaim = storage.claimUnpushedDlrs("sys1"); - storage.releaseUnpushedDlrClaim(firstClaim.getFirst()); - List afterRelease = storage.claimUnpushedDlrs("sys1"); - - assertEquals(1, firstClaim.size()); - assertTrue(secondClaim.isEmpty()); - assertEquals(1, afterRelease.size()); - assertEquals(dlr.serial, afterRelease.getFirst().serial); - } - - @Test - void unpushedDlrs_SurviveRestart() throws Exception { - StandardMessage dlr = createDlr("account-restart", "sys-restart"); - - assertTrue(storage.saveUnpushedDlr(dlr)); - storage.onStop(); - - storage = new MvStoreDlrStorage(); - storage.init(); - List dlrs = storage.getUnpushedDlrs("sys-restart"); - - assertTrue(dlrs.stream().anyMatch(msg -> dlr.serial.equals(msg.serial))); - } - - @Test - void getPrimaryStoreSize_ReturnsCount() { - storage.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); - storage.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); - storage.saveInitialState(new MessageState("gw-3", "systemId", "from", "to", null)); - - assertEquals(3, storage.getPrimaryStoreSize()); - } - - @Test - void getCorrelationIndexSize_ReturnsCount() { - storage.saveInitialState(new MessageState("gw-1", "systemId", "from", "to", null)); - storage.saveInitialState(new MessageState("gw-2", "systemId", "from", "to", null)); - storage.linkOperatorId("gw-1", "op-1"); - storage.linkOperatorId("gw-2", "op-2"); - - assertEquals(2, storage.getCorrelationIndexSize()); - } - - @Test - void isPersistent_TrueWhenDbAvailable() { - assertTrue(storage.isPersistent()); - } - - private StandardMessage createDlr(String accountId, String systemId) { - StandardMessage dlr = new StandardMessage(); - dlr.type = StandardMessage.MSG_DLR; - dlr.owner_id = accountId; - dlr.systemId = systemId; - dlr.serial = UUID.randomUUID().toString(); - dlr.from = "from"; - dlr.to = "to"; - dlr.state = StandardMessage.DLR_STAT_DELIVRD; - dlr.errcode = "0"; - dlr.acked = true; - dlr.priority = StandardMessage.HIGH_PRIORITY; - dlr.msgId = 123; - dlr.reassembledParts = new ArrayList<>(List.of("part-1", "part-2")); - return dlr; - } -} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java index 19b72fb..e064a12 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrQuarkusTestResource.java @@ -16,10 +16,6 @@ public class PostgresqlDlrQuarkusTestResource implements QuarkusTestResourceLife @Override public Map start() { - if (!Boolean.getBoolean("sendium.postgresql.tests")) { - return Map.of(); - } - postgresql = new PostgreSQLContainer("postgres:17-alpine") .withDatabaseName("sendium") .withUsername("sendium") @@ -29,7 +25,6 @@ public Map start() { smppConfiguration = createSmppConfiguration(smppPort); String jdbcUrl = postgresql.getJdbcUrl() + "&connectTimeout=2&socketTimeout=2"; return Map.ofEntries( - Map.entry("sendium.dlr.storage", "postgresql"), Map.entry("quarkus.datasource.dlr.active", "true"), Map.entry("quarkus.flyway.dlr.active", "true"), Map.entry("quarkus.flyway.dlr.migrate-at-start", "true"), diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeIT.java similarity index 96% rename from sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeIT.java index 252fd40..e7067d2 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrRuntimeIT.java @@ -22,7 +22,6 @@ import jakarta.inject.Inject; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import utils.CaptorWorker; import io.netty.channel.nio.NioEventLoopGroup; @@ -41,13 +40,12 @@ @QuarkusTest @QuarkusTestResource(value = PostgresqlDlrQuarkusTestResource.class, restrictToAnnotatedClass = true) -@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") -class PostgresqlDlrRuntimeTest { +class PostgresqlDlrRuntimeIT { @Inject DlrStorage storage; @Inject - ConfiguredDlrStorage configuredStorage; + ManagedDlrStorage managedStorage; @Inject @DataSource("dlr") @@ -66,8 +64,8 @@ class PostgresqlDlrRuntimeTest { @Test void wiresPoolMigrationStorageHealthAndMetrics() throws SQLException { long successfulSavesBefore = metricCount("save_initial", "success"); - assertThat(storage).isSameAs(configuredStorage); - assertThat(configuredStorage.backend()).isEqualTo("postgresql"); + assertThat(storage).isSameAs(managedStorage); + assertThat(managedStorage.backend()).isEqualTo("postgresql"); assertThat(dataSource.getHandle().getBean().isActive()).isTrue(); assertThat(flyway.getHandle().getBean().isActive()).isTrue(); assertThat(flyway.get().info().current().getVersion().getVersion()).isEqualTo("1"); @@ -86,9 +84,9 @@ void wiresPoolMigrationStorageHealthAndMetrics() throws SQLException { .body("checks.find { it.name == 'sendium-dlr-storage' }.data.backend", equalTo("postgresql")); + assertThat(metricCount("save_initial", "success")).isEqualTo(successfulSavesBefore + 1); assertThat(meterRegistry.find("sendium.dlr.storage.selected") .tag("backend", "postgresql").gauge().value()).isEqualTo(1.0); - assertThat(metricCount("save_initial", "success")).isEqualTo(successfulSavesBefore + 1); assertThat(meterRegistry.getMeters()) .extracting(meter -> meter.getId().getName()) .anyMatch(name -> name.startsWith("agroal")); @@ -120,7 +118,7 @@ void databaseOutageRejectsHttpAndSmppWithoutRoutingOrFallback() throws Exception assertThat(failedSmpp.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR); assertThat(failedSmpp.getMessageId()).isBlank(); assertThat(captorWorker.captures).isEmpty(); - assertThat(configuredStorage.backend()).isEqualTo("postgresql"); + assertThat(managedStorage.backend()).isEqualTo("postgresql"); given() .when().get("/q/health/ready") diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java similarity index 99% rename from sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java index f8677bb..23f8ffd 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java @@ -6,7 +6,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.postgresql.PostgreSQLContainer; @@ -28,8 +27,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@EnabledIfSystemProperty(named = "sendium.postgresql.tests", matches = "true") -class PostgresqlDlrStorageTest { +class PostgresqlDlrStorageIT { private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") .withDatabaseName("sendium") diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index 470de99..6c57c63 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -83,10 +83,6 @@ public static void main(String[] args) throws Exception { container = verifyUnpushedDlrSurvivesRestart(containerName, workDir, upstream); container = verifyHttpCorrelationSurvivesRestart(containerName, workDir, upstream, callbackServer, 2); verifySmppSubmitGetsDeliverSm(upstream, 3); - - stopContainer(containerName); - container = startMvStoreContainer(containerName, workDir); - waitForMvStoreReadiness(); } catch (Throwable t) { printDockerLogs(containerName); throw t; @@ -222,39 +218,21 @@ private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Pa } private static Process startSendiumContainer(String containerName, Path workDir) throws Exception { - return startSendiumContainer(containerName, workDir, false); - } - - private static Process startMvStoreContainer(String containerName, Path workDir) throws Exception { - return startSendiumContainer(containerName, workDir, true); - } - - private static Process startSendiumContainer(String containerName, Path workDir, boolean mvStore) throws Exception { - List command = new ArrayList<>(List.of( + List command = List.of( "docker", "run", "--rm", "-d", "--name", containerName, "--add-host", "host.docker.internal:host-gateway", "-p", SENDIUM_HTTP_PORT + ":8080", "-p", SENDIUM_SMPP_PORT + ":27777", "-v", workDir.resolve("conf").toAbsolutePath() + ":/work/conf", - "-v", workDir.resolve("data").toAbsolutePath() + ":/work/data", "-v", workDir.resolve("logs").toAbsolutePath() + ":/work/logs", "-e", "QUARKUS_LOG_LEVEL=INFO", - "-e", "LOG_LEVEL=INFO" - )); - if (mvStore) { - command.addAll(List.of( - "-e", "SENDIUM_DLR_STORAGE=mvstore", - "-e", "SENDIUM_DLR_POSTGRESQL_ACTIVE=false" - )); - } else { - command.addAll(List.of( - "-e", "SENDIUM_DLR_POSTGRESQL_JDBC_URL=" + POSTGRESQL_JDBC_URL, - "-e", "SENDIUM_DLR_POSTGRESQL_USERNAME=" + POSTGRESQL_USERNAME, - "-e", "SENDIUM_DLR_POSTGRESQL_PASSWORD=" + POSTGRESQL_PASSWORD - )); - } - command.add(IMAGE); + "-e", "LOG_LEVEL=INFO", + "-e", "SENDIUM_DLR_POSTGRESQL_JDBC_URL=" + POSTGRESQL_JDBC_URL, + "-e", "SENDIUM_DLR_POSTGRESQL_USERNAME=" + POSTGRESQL_USERNAME, + "-e", "SENDIUM_DLR_POSTGRESQL_PASSWORD=" + POSTGRESQL_PASSWORD, + IMAGE + ); Process process = run(command, true); require(process.waitFor(30, TimeUnit.SECONDS), "Timed out starting Sendium container"); require(process.exitValue() == 0, "Failed to start Sendium container"); @@ -264,7 +242,6 @@ private static Process startSendiumContainer(String containerName, Path workDir, private static void writeRuntimeConfig(Path workDir) throws IOException { Path conf = workDir.resolve("conf"); Files.createDirectories(conf); - Files.createDirectories(workDir.resolve("data")); Files.createDirectories(workDir.resolve("logs")); Files.writeString(conf.resolve("credentials.yml"), """ @@ -334,14 +311,6 @@ private static void waitForPort(String host, int port, Duration timeout) throws } private static void waitForPostgresqlReadiness() throws Exception { - waitForStorageReadiness("postgresql", null); - } - - private static void waitForMvStoreReadiness() throws Exception { - waitForStorageReadiness("mvstore", "persistent"); - } - - private static void waitForStorageReadiness(String backend, String mode) throws Exception { long deadline = System.nanoTime() + TIMEOUT.toNanos(); while (System.nanoTime() < deadline) { try { @@ -349,15 +318,14 @@ private static void waitForStorageReadiness(String backend, String mode) throws String body = response.body().replaceAll("\\s", ""); if (response.statusCode() == HttpURLConnection.HTTP_OK && body.contains("\"name\":\"sendium-dlr-storage\"") - && body.contains("\"backend\":\"" + backend + "\"") - && (mode == null || body.contains("\"mode\":\"" + mode + "\""))) { + && body.contains("\"backend\":\"postgresql\"")) { return; } } catch (IOException ignored) { } Thread.sleep(500); } - throw new IllegalStateException("Timed out waiting for " + backend + "-backed Sendium readiness"); + throw new IllegalStateException("Timed out waiting for PostgreSQL-backed Sendium readiness"); } private static void awaitSuccessfulStorageOperation(String operation) throws Exception { diff --git a/tests/quick-start-test.sh b/tests/quick-start-test.sh index 2ce214a..79e6eda 100644 --- a/tests/quick-start-test.sh +++ b/tests/quick-start-test.sh @@ -121,8 +121,6 @@ assert_equals 48 "${#http_password}" "HTTP password length" assert_equals 8 "${#smpp_password}" "SMPP password length" assert_equals 64 "${#database_password}" "PostgreSQL password length" assert_equals "$database_password" "$postgres_password" "PostgreSQL container password" -assert_contains "SENDIUM_DLR_STORAGE='postgresql'" "$local_dir/.sendium.env" -assert_contains "SENDIUM_DLR_POSTGRESQL_ACTIVE='true'" "$local_dir/.sendium.env" assert_contains "SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://postgres:5432/sendium'" "$local_dir/.sendium.env" assert_not_contains "$database_password" "$local_dir/compose.yml" assert_equals "$http_user" "$(credential_value HTTP systemId "$local_dir/conf/credentials.yml")" "HTTP credential username" From 91d1f54539b46ac5f9d64aad953bcab9248ff3bb Mon Sep 17 00:00:00 2001 From: pavlos Date: Wed, 19 Aug 2026 16:55:07 +0300 Subject: [PATCH 14/20] feat(core): make DLR persistence optional --- docs/09-configuration-reference.md | 6 ++++++ docs/13-dlr-persistence.md | 6 ++++++ .../src/main/resources/application.properties | 1 + .../sendium/core/http/KannelResource.java | 9 ++++++--- .../cytech/sendium/core/worker/DlrService.java | 2 ++ .../core/worker/DlrStorageReadinessCheck.java | 2 ++ .../sendium/core/worker/ForwardDlrService.java | 4 +++- .../sendium/core/worker/ManagedDlrStorage.java | 2 ++ .../external/WorkerResourceProvider.java | 8 ++++++-- .../src/main/resources/application.properties | 5 +++-- .../sendium/core/http/KannelResourceTest.java | 17 ++++++++++++++++- .../src/test/resources/application.properties | 1 + 12 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 sendium-core/src/test/resources/application.properties diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index d683eb3..d53438a 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -56,6 +56,12 @@ In the Docker image, the working directory is `/work`, so the default configurat PostgreSQL is the only DLR persistence backend and is fail-closed. Startup requires a valid datasource URL and any username, password, certificates, or tokens required by the database authentication method; a bare launch fails rather than falling back to local or in-memory storage. See [DLR Persistence](13-dlr-persistence.md) for the complete durability contract. +### Core Embedding + +`sendium.dlr.persistence.enabled` is a build-time setting. The `sendium-core` module defaults it to `false`, while the standalone `sendium-app` sets it to `true`. + +When disabled, Sendium does not create its DLR services, PostgreSQL datasource, Flyway migration, or storage readiness check. An application that embeds `sendium-core` must set the property to `true` before Quarkus augmentation to opt into the complete DLR subsystem, then provide the PostgreSQL settings above. Enabled persistence remains fail-closed. + ## Logs | Log | Description | diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 7186204..0749dcd 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -4,6 +4,12 @@ Sendium stores the state needed to correlate upstream delivery receipts (DLRs) a This storage boundary does not make Sendium's message queues or all delivery processing durable. Review [Durability Boundaries](#durability-boundaries) before using restart recovery as a delivery guarantee. +## Application Boundary + +The standalone `sendium-app` enables PostgreSQL DLR persistence and requires it to be available. Applications that embed `sendium-core` default to no Sendium-owned DLR subsystem and can run without a DLR database. + +The `sendium.dlr.persistence.enabled` build-time property controls this boundary. When it is `false`, the DLR services, PostgreSQL datasource, Flyway migration, and storage readiness check are absent. Set the property to `true` before Quarkus augmentation to opt into the complete DLR subsystem; partial or no-op persistence is not provided. + ## Quick Start PostgreSQL The generated Quick Start runtime creates: diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index b0fb63d..3c96411 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -3,6 +3,7 @@ smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml # DLR persistence. PostgreSQL requires valid named datasource connection settings. +sendium.dlr.persistence.enabled=true quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql quarkus.datasource.dlr.active=true diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index 8e1306f..dc7690f 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -10,6 +10,7 @@ import gr.cytech.sendium.core.worker.MessageState; import gr.cytech.sendium.util.MessageTrace; import jakarta.annotation.security.PermitAll; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -45,7 +46,7 @@ public class KannelResource { CredentialFileWatcher credentialFileWatcher; @Inject - DlrService dlrService; + Instance dlrServices; @Inject SendiumConfigurationHandler configurationHandler; @@ -211,8 +212,10 @@ public Response receiveSms( } msg.acked = true; msg.serial = UUID.randomUUID().toString(); - MessageState state = new MessageState(msg.serial, usr, msg.from, msg.to, dlrUrl); - dlrService.saveInitialState(state); + if (!dlrServices.isUnsatisfied()) { + MessageState state = new MessageState(msg.serial, usr, msg.from, msg.to, dlrUrl); + dlrServices.get().saveInitialState(state); + } queueProvider.getRouterQueue().enqueue(msg); if (MessageTrace.shouldLog(configurationHandler, MessageTrace.EVENT_ACCEPTED)) { logger.info("message.accepted ingress=http {}", MessageTrace.identifiers(msg)); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java index 6877a43..dcb9063 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java @@ -1,6 +1,7 @@ package gr.cytech.sendium.core.worker; import gr.cytech.sendium.core.message.StandardMessage; +import io.quarkus.arc.properties.IfBuildProperty; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -8,6 +9,7 @@ import java.util.Optional; @ApplicationScoped +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") public class DlrService { @Inject DlrStorage storage; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java index 5b3ed24..7836b6e 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java @@ -1,5 +1,6 @@ package gr.cytech.sendium.core.worker; +import io.quarkus.arc.properties.IfBuildProperty; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.eclipse.microprofile.health.HealthCheck; @@ -11,6 +12,7 @@ @Readiness @ApplicationScoped +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") public class DlrStorageReadinessCheck implements HealthCheck { private static final String CHECK_NAME = "sendium-dlr-storage"; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java index dfcaf3c..e3aea84 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java @@ -1,5 +1,6 @@ package gr.cytech.sendium.core.worker; +import io.quarkus.arc.properties.IfBuildProperty; import jakarta.enterprise.context.ApplicationScoped; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,6 +13,7 @@ import java.time.Duration; @ApplicationScoped +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") public class ForwardDlrService { private static final Logger logger = LoggerFactory.getLogger(ForwardDlrService.class); @@ -116,4 +118,4 @@ private void scheduleRetry(String url, String gatewayMsgId, int attempt) { logger.error("Retry sleep interrupted for gatewayMsgId: {}", gatewayMsgId); } } -} \ No newline at end of file +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java index dab74fb..3fac701 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java @@ -7,6 +7,7 @@ import io.micrometer.core.instrument.Timer; import io.quarkus.agroal.DataSource; import io.quarkus.arc.InjectableInstance; +import io.quarkus.arc.properties.IfBuildProperty; import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; import jakarta.enterprise.context.ApplicationScoped; @@ -22,6 +23,7 @@ @Startup @ApplicationScoped +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") public class ManagedDlrStorage implements DlrStorage { private static final String METRIC_NAME = "sendium.dlr.storage.operation"; private static final String BACKEND = "postgresql"; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java index 0aa715b..5f8ea0c 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java @@ -8,6 +8,7 @@ import gr.cytech.sendium.core.worker.ForwardMoService; import io.quarkus.arc.DefaultBean; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,7 @@ public enum Visibility { INTERNAL, EXTERNAL } @Inject InMemoryQueueProvider queueProvider; @Inject CredentialFileWatcher credentialFileWatcher; - @Inject DlrService dlrService; + @Inject Instance dlrServices; @Inject ForwardMoService forwardMoService; @Inject SmppClientHolder smppClientHolder; @@ -41,7 +42,10 @@ public CredentialFileWatcher getCredentialFileWatcher() { } public DlrService getDlrService() { - return dlrService; + if (dlrServices.isUnsatisfied()) { + throw new IllegalStateException("Sendium DLR persistence is disabled"); + } + return dlrServices.get(); } public ForwardMoService getForwardMoService() { diff --git a/sendium-core/src/main/resources/application.properties b/sendium-core/src/main/resources/application.properties index 2739297..5337799 100644 --- a/sendium-core/src/main/resources/application.properties +++ b/sendium-core/src/main/resources/application.properties @@ -2,10 +2,11 @@ %test.smsg.properties.file.path=src/test/resources/smsg.properties %test.smsg.credentials.file.path=src/test/resources/credentials.yml -# PostgreSQL is the only DLR persistence backend. +# Sendium-owned DLR persistence is enabled by the standalone application. +sendium.dlr.persistence.enabled=false quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=true +quarkus.datasource.dlr.active=${sendium.dlr.persistence.enabled} quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.min-size=0 quarkus.datasource.dlr.jdbc.max-size=10 diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java index b5d6762..4a4f0f4 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java @@ -8,6 +8,7 @@ import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.MessageState; +import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.Response; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -32,6 +33,7 @@ class KannelResourceTest { private Queue routerQueue; private DlrService dlrService; + private Instance dlrServices; private KannelResource resource; @BeforeEach @@ -51,7 +53,9 @@ void setUp() { resource.credentialFileWatcher = credentials; resource.configurationHandler = mock(SendiumConfigurationHandler.class); dlrService = mock(DlrService.class); - resource.dlrService = dlrService; + dlrServices = mock(Instance.class); + when(dlrServices.get()).thenReturn(dlrService); + resource.dlrServices = dlrServices; } @Test @@ -84,6 +88,17 @@ void rejectsBeforeQueueAdmissionWhenPersistenceFails() throws InterruptedExcepti verify(routerQueue, never()).enqueue(any(StandardMessage.class)); } + @Test + void acceptsSubmissionWithoutDlrTrackingWhenPersistenceIsDisabled() throws InterruptedException { + when(dlrServices.isUnsatisfied()).thenReturn(true); + + Response response = submit("https://callback.test/dlr"); + + assertThat(response.getStatus()).isEqualTo(Response.Status.ACCEPTED.getStatusCode()); + verify(dlrService, never()).saveInitialState(any(MessageState.class)); + verify(routerQueue).enqueue(any(StandardMessage.class)); + } + @Test void returnsRetryableFailureWhenQueueAdmissionIsInterruptedAfterPersistence() throws InterruptedException { doThrow(new InterruptedException("interrupted")) diff --git a/sendium-core/src/test/resources/application.properties b/sendium-core/src/test/resources/application.properties new file mode 100644 index 0000000..b3d4863 --- /dev/null +++ b/sendium-core/src/test/resources/application.properties @@ -0,0 +1 @@ +sendium.dlr.persistence.enabled=true From 81f58af9efb7d5dfc9c77aafb0f7888bce0b9a33 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 20 Aug 2026 16:57:47 +0300 Subject: [PATCH 15/20] fix(dlr): harden PostgreSQL persistence --- docs/01-architecture.md | 6 +- docs/02-docker-deployment.md | 2 +- docs/04-smpp-configuration.md | 3 +- docs/09-configuration-reference.md | 4 +- docs/13-dlr-persistence.md | 12 +- pom.xml | 1 - quick-start.sh | 42 +- .../src/main/resources/application.properties | 2 +- .../sendium/core/AbstractOutWorker.java | 12 +- .../core/smpp/client/SmppClientWorker.java | 80 ++- .../InMemorySmppServerMessageStore.java | 21 + .../core/smpp/server/SmppServerWorker.java | 2 +- .../core/worker/DlrMessageStorage.java | 8 +- .../sendium/core/worker/DlrService.java | 11 +- .../core/worker/DlrStorageReadinessCheck.java | 27 +- .../core/worker/ForwardDlrService.java | 2 +- .../core/worker/InMemoryMessageTracker.java | 45 +- .../core/worker/ManagedDlrStorage.java | 20 +- .../sendium/core/worker/MessageState.java | 22 +- .../core/worker/PostgresqlDlrStorage.java | 474 +++++++++++++----- .../cytech/sendium/core/worker/Tracker.java | 7 +- .../external/WorkerResourceProvider.java | 9 + .../gr/cytech/sendium/util/MessageTrace.java | 2 +- .../src/main/resources/application.properties | 7 +- .../V1__create_sendium_dlr_schema.sql | 34 +- .../core/dlr/PostgresqlMigrationIT.java | 73 ++- .../smpp/client/SmppClientWorkerTest.java | 120 ++++- .../InMemorySmppServerMessageStoreTest.java | 34 ++ .../sendium/core/worker/DlrServiceTest.java | 16 +- .../worker/InMemoryMessageTrackerTest.java | 82 ++- .../sendium/core/worker/MessageStateTest.java | 19 +- .../core/worker/PostgresqlDlrStorageIT.java | 356 ++++++++++--- .../PostgresqlDlrStorageRetentionTest.java | 74 +++ .../cytech/sendium/util/MessageTraceTest.java | 4 +- .../src/test/java/utils/NativeE2eSmoke.java | 4 +- tests/quick-start-test.sh | 35 ++ 36 files changed, 1316 insertions(+), 356 deletions(-) create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageRetentionTest.java diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 03cc1b1..c8f0761 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -103,7 +103,7 @@ sequenceDiagram Router->>Worker: Enqueue to selected worker Worker->>SMSC: submit_sm SMSC-->>Worker: submit_sm_resp - Worker->>DLR: Link gateway UUID to operator message ID + Worker->>DLR: Link gateway UUID to provider message ID ``` ## SMPP Server Flow @@ -151,7 +151,7 @@ sequenceDiagram ## DLR Handling -Outbound HTTP messages can include a Kannel-style `dlr-url`. Sendium stores the gateway message ID and later links it to the operator/SMSC message ID returned by the SMPP provider. When a DLR arrives, the DLR service resolves the correlation and forwards the callback. +Outbound HTTP messages can include a Kannel-style `dlr-url`. Sendium stores the gateway message ID and later links it to the message ID returned by the SMPP provider. Provider message IDs are scoped by the outbound provider name, so different providers may reuse the same ID without overwriting each other's correlation. When a DLR arrives, the DLR service resolves the provider and message-ID pair and forwards the callback. ```mermaid sequenceDiagram @@ -165,7 +165,7 @@ sequenceDiagram SMSC->>Worker: deliver_sm delivery receipt Worker->>Tracker: createAndEnqueueDLR - Tracker->>Store: Resolve operator message ID + Tracker->>Store: Resolve provider name and provider message ID Store->>DLRHook: Forward DLR callback if URL exists DLRHook->>App: HTTP GET callback Tracker->>Router: Enqueue internal MSG_DLR diff --git a/docs/02-docker-deployment.md b/docs/02-docker-deployment.md index 1c04e2c..2981200 100644 --- a/docs/02-docker-deployment.md +++ b/docs/02-docker-deployment.md @@ -44,7 +44,7 @@ sendium/ `.sendium.env`, `credentials.yml`, and `smsg.properties` contain secrets. The generated `.gitignore` excludes them, but they still require access-controlled storage and backups. The local PostgreSQL service is private to the Compose network and does not publish a database port. -Using `--force` regenerates the HTTP/SMPP credentials and configuration while preserving the generated local database password required by the existing PostgreSQL volume. When startup is enabled, Quick Start recreates the containers so the new credentials and worker configuration take effect together. With `--no-start`, it prints the required `docker compose up -d --force-recreate --remove-orphans` command instead. +Using `--force` regenerates the HTTP/SMPP credentials and configuration while preserving the generated local database password required by the existing PostgreSQL volume. The local password is retained separately from an external database password, so switching to an external database and later returning to the bundled database does not break authentication. When startup is enabled, Quick Start recreates the containers so the new credentials and worker configuration take effect together. With `--no-start`, it prints the required `docker compose up -d --force-recreate --remove-orphans` command instead. To use an operator-managed PostgreSQL database, set `SENDIUM_DLR_POSTGRESQL_JDBC_URL`, `SENDIUM_DLR_POSTGRESQL_USERNAME`, and `SENDIUM_DLR_POSTGRESQL_PASSWORD` together before running Quick Start. The generated Compose file then omits the local PostgreSQL service. See [DLR Persistence](13-dlr-persistence.md) for TLS, permissions, retention, and durability guidance. diff --git a/docs/04-smpp-configuration.md b/docs/04-smpp-configuration.md index 3bf39a7..8a5f684 100644 --- a/docs/04-smpp-configuration.md +++ b/docs/04-smpp-configuration.md @@ -247,6 +247,7 @@ The SMPP Client worker (`WorkerType: smppclient`) allows the application to conn | `dcs.charset.ext` | `""` | Override or add extra DCS-to-Charset mappings (Format: `DCS_CHARSET`). | | `dlr.charset.fixed` | `""` | Forces a specific character set for decoding Delivery Receipts. | | `msg.id.type` | `0` | Determines how the SMSC Message ID is parsed (0=StringLiteral, 1=SubmitRespHexDlrDec, 2=SubmitRespDecDlrHex). | +| `msg.hash.prefix` | Worker instance name | Stable provider namespace used for DLR correlation. Configure the same value on multiple workers only when one SMSC account may deliver their receipts interchangeably. Changing it strands outstanding correlations under the previous value. | ## 🚦 Error Handling & Routing Policies @@ -272,7 +273,7 @@ The SMPP Client worker (`WorkerType: smppclient`) allows the application to conn ## 📊 Logging & Diagnostics -SMPP client PDU, response, and MO diagnostics are disabled by default. These logs can include bind passwords, phone numbers, provider message IDs, callback data, and message bodies, so enable them only when the log destination is access-controlled and retention is appropriate. Default `message.trace.mode = necessary` preserves submit and DLR milestones without logging payloads; use `all` for submit-response and operator-link details. +SMPP client PDU, response, and MO diagnostics are disabled by default. These logs can include bind passwords, phone numbers, provider message IDs, callback data, and message bodies, so enable them only when the log destination is access-controlled and retention is appropriate. Default `message.trace.mode = necessary` preserves submit and DLR milestones without logging payloads; use `all` for submit-response and provider-link details. | Property | Default Value | Description | | :--- | :--- | :--- | diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index d53438a..17f0b3e 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -58,9 +58,9 @@ PostgreSQL is the only DLR persistence backend and is fail-closed. Startup requi ### Core Embedding -`sendium.dlr.persistence.enabled` is a build-time setting. The `sendium-core` module defaults it to `false`, while the standalone `sendium-app` sets it to `true`. +`sendium.dlr.persistence.enabled` is a build-time setting. The `sendium-core` module leaves it undefined, which means disabled; the standalone `sendium-app` sets it to `true`. -When disabled, Sendium does not create its DLR services, PostgreSQL datasource, Flyway migration, or storage readiness check. An application that embeds `sendium-core` must set the property to `true` before Quarkus augmentation to opt into the complete DLR subsystem, then provide the PostgreSQL settings above. Enabled persistence remains fail-closed. +When disabled, Sendium does not create its DLR services, PostgreSQL datasource, Flyway migration, or storage readiness check. Submissions are still accepted and routed, but no gateway DLR state is stored and Sendium emits no delivery receipts of its own. An application that embeds `sendium-core` must declare the property as `true` before Quarkus augmentation to opt into the complete DLR subsystem, then provide the PostgreSQL settings above. Enabled persistence remains fail-closed. ## Logs diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 0749dcd..8b0d532 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -8,7 +8,9 @@ This storage boundary does not make Sendium's message queues or all delivery pro The standalone `sendium-app` enables PostgreSQL DLR persistence and requires it to be available. Applications that embed `sendium-core` default to no Sendium-owned DLR subsystem and can run without a DLR database. -The `sendium.dlr.persistence.enabled` build-time property controls this boundary. When it is `false`, the DLR services, PostgreSQL datasource, Flyway migration, and storage readiness check are absent. Set the property to `true` before Quarkus augmentation to opt into the complete DLR subsystem; partial or no-op persistence is not provided. +The `sendium.dlr.persistence.enabled` build-time property controls this boundary. `sendium-core` leaves it undefined and an undefined property means disabled, so an embedding application opts in by declaring it as `true` itself before Quarkus augmentation. When it is not enabled, the DLR services, PostgreSQL datasource, Flyway migration, and storage readiness check are absent; partial or no-op persistence is not provided. + +Message paths degrade rather than fail when the subsystem is absent. HTTP and downstream SMPP submissions are accepted and routed without gateway DLR state, undelivered downstream receipts fall back to the worker's in-memory retry, and provider receipts are not correlated, so Sendium emits no delivery receipts of its own. An application that embeds `sendium-core` without this subsystem is expected to supply its own `Tracker` and message store if it needs delivery receipts. ## Quick Start PostgreSQL @@ -28,7 +30,7 @@ sh quick-start.sh `docker compose down` removes the containers and network but retains the PostgreSQL volume. Do not use `docker compose down --volumes` or manually delete the volume unless permanent database deletion is intended. -Quick Start preserves the local database password during `--force` regeneration. PostgreSQL initialization variables cannot rotate the password of a role that already exists in a persistent data volume. +Quick Start preserves the local database password during `--force` regeneration, including while an external database is temporarily selected. PostgreSQL initialization variables cannot rotate the password of a role that already exists in a persistent data volume. If the saved local password is lost, Quick Start fails with instructions to delete both the volume and its generated Compose marker before a new password is generated. ## Upgrade From MVStore Builds @@ -94,18 +96,22 @@ Relevant metrics include storage-operation latency/counts tagged by backend, ope PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP submissions return the retryable `503` response and new SMPP submissions return `ESME_RSYSERR`; Sendium does not fall back to local or in-memory storage. +Provider message IDs are correlated within the outbound provider namespace rather than globally. The worker instance name is the default namespace; workers connected to the same SMSC account can share `msg.hash.prefix` when that SMSC may deliver their receipts interchangeably. Different providers may therefore return the same message ID without overwriting each other's state. The namespace must remain stable while correlations are outstanding: changing `msg.hash.prefix` or renaming a worker using the default makes earlier receipts unresolvable. + ## Retention The V1 retention thresholds are fixed application behavior, not environment settings: | State | Eligible for cleanup after | | :--- | :--- | -| Provider/operator correlation | 3 days | +| Provider message correlation | 3 days | | Tracked gateway message | 7 days | | Unpushed downstream SMPP receipt | 7 days | Cleanup is triggered by storage activity and runs no more than once per hour. These values are therefore eligibility thresholds, not exact physical deletion deadlines: idle records can remain in the database longer, and an active deployment can retain newly eligible state until the next cleanup pass. A provider receipt cannot be matched after its correlation has been removed. Making the thresholds or cleanup schedule configurable is outside the V1 storage replacement. +Cleanup is best-effort maintenance and is isolated from message handling. One caller at a time runs a pass while every other caller proceeds immediately, and a failed pass is logged and left until the next interval rather than rejecting the submission that triggered it. + ## Durability Boundaries | State or transition | PostgreSQL guarantee | Remaining limit | diff --git a/pom.xml b/pom.xml index 03fe3b4..89a936c 100644 --- a/pom.xml +++ b/pom.xml @@ -92,5 +92,4 @@ - diff --git a/quick-start.sh b/quick-start.sh index 12a2d60..379df92 100644 --- a/quick-start.sh +++ b/quick-start.sh @@ -20,6 +20,7 @@ database_username=${SENDIUM_DLR_POSTGRESQL_USERNAME-} database_password=${SENDIUM_DLR_POSTGRESQL_PASSWORD-} unset SENDIUM_UPSTREAM_PASSWORD unset SENDIUM_DLR_POSTGRESQL_PASSWORD +unset SENDIUM_LOCAL_POSTGRESQL_PASSWORD usage() { cat <<'EOF' @@ -394,6 +395,32 @@ if [ "$upstream_enabled" = true ]; then upstream_password=$(escape_property_value "$upstream_password") fi +existing_bundled_database=false +local_database_password='' +if [ "$force" = true ]; then + if [ -f "$target_dir/compose.yml" ] && \ + grep -q 'postgres-data:/var/lib/postgresql/data' "$target_dir/compose.yml"; then + existing_bundled_database=true + fi + if [ -f "$target_dir/.sendium.env" ]; then + local_database_password=$(sed -n "s/^SENDIUM_LOCAL_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$target_dir/.sendium.env") + if [ "${#local_database_password}" -ne 64 ] && [ "$existing_bundled_database" = true ]; then + local_database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$target_dir/.sendium.env") + fi + if [ "${#local_database_password}" -ne 64 ] && [ "$existing_bundled_database" = true ]; then + local_database_password=$(sed -n "s/^POSTGRES_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$target_dir/.sendium.env") + fi + fi + if [ "$existing_bundled_database" = true ] && [ "${#local_database_password}" -ne 64 ]; then + fail \ +"the existing PostgreSQL volume requires the password recorded in $target_dir/.sendium.env, which could not be read. +Restore that file, or permanently delete the database and its generated Compose marker with: + docker compose -f \"$target_dir/compose.yml\" --project-directory \"$target_dir\" down --volumes + rm -f \"$target_dir/compose.yml\" +Then rerun Quick Start with --force." + fi +fi + external_database=false if [ -n "$database_jdbc_url" ] || [ -n "$database_username" ] || [ -n "$database_password" ]; then [ -n "$database_jdbc_url" ] && [ -n "$database_username" ] && [ -n "$database_password" ] || \ @@ -409,15 +436,13 @@ if [ -n "$database_jdbc_url" ] || [ -n "$database_username" ] || [ -n "$database else database_jdbc_url='jdbc:postgresql://postgres:5432/sendium' database_username='sendium' - if [ "$force" = true ] && [ -f "$target_dir/.sendium.env" ]; then - existing_database_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$target_dir/.sendium.env") - if [ "${#existing_database_password}" -eq 64 ]; then - database_password=$existing_database_password - fi + if [ "${#local_database_password}" -eq 64 ]; then + database_password=$local_database_password fi if [ -z "$database_password" ]; then database_password=$(generate_secret 32) fi + local_database_password=$database_password fi mkdir -p "$target_dir/conf" "$target_dir/logs" @@ -459,6 +484,12 @@ SENDIUM_DLR_POSTGRESQL_USERNAME='$database_username' SENDIUM_DLR_POSTGRESQL_PASSWORD='$database_password' EOF +if [ -n "$local_database_password" ]; then + cat >> "$staging_dir/.sendium.env" <> "$staging_dir/.sendium.env" <> "$staging_dir/compose.yml" <<'EOF' environment: + SENDIUM_LOCAL_POSTGRESQL_PASSWORD: '' QUARKUS_LOG_FILE_ENABLE: "true" QUARKUS_LOG_CONSOLE_ENABLE: "true" QUARKUS_LOG_FILE_PATH: /work/logs/smsg.log diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index 3c96411..95e3da4 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -6,7 +6,7 @@ smsg.credentials.file.path=conf/credentials.yml sendium.dlr.persistence.enabled=true quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=true +quarkus.datasource.dlr.active=${sendium.dlr.persistence.enabled} quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.url=${SENDIUM_DLR_POSTGRESQL_JDBC_URL:} quarkus.datasource.dlr.username=${SENDIUM_DLR_POSTGRESQL_USERNAME:} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java index 2608daf..a5b39b9 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java @@ -347,6 +347,10 @@ public final String getFullName() { return fullName; } + public String getDlrProviderName() { + return getFullName(); + } + public long getAlertMaxPendingQueue() { return alertMaxPendingQueue; } @@ -566,10 +570,10 @@ public Tracker getMessageTracker() { return messageTracker; } - public int updateSendStatusAndExtID(String smsid, M pMsg, String smscid) { - pMsg.extrid = smscid; - pMsg.field1 = smsid; - return messageTracker.updateSendStatusAndExtID(smsid, pMsg, smscid); + public int updateSendStatusAndExtID(String hashedProviderMessageId, M message, String providerMessageId) { + message.extrid = providerMessageId; + message.field1 = hashedProviderMessageId; + return messageTracker.updateSendStatusAndExtID(hashedProviderMessageId, message, providerMessageId); } public String getHashedMessageID(String messageId) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java index 75c639f..d4142aa 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java @@ -32,6 +32,7 @@ import gr.cytech.sendium.core.smpp.util.CustomCharset; import gr.cytech.sendium.core.smpp.util.SmppServerUtil; import gr.cytech.sendium.core.smpp.util.VFGRCharset; +import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.ForwardMoService; import gr.cytech.sendium.core.worker.Tracker; import gr.cytech.sendium.core.worker.WorkerType; @@ -421,6 +422,11 @@ public String getHashedMessageID(String messageId) { return SecurityUtils.generateMD5(messageHashPrefix.concat(messageId)); } + @Override + public String getDlrProviderName() { + return messageHashPrefix == null || messageHashPrefix.isBlank() ? getInstanceName() : messageHashPrefix; + } + @Override public boolean myPropertyChange(String key, String newValue, String oldValue) { if (key.equals(_srcAddrAutodetect[0]) || @@ -795,7 +801,7 @@ public void configMessageHashPrefix() { //this allows to have multiple clients (multiple instances of the worker, not multiple connections within the worker) //connecting to the same SMSC (host/port/user) and dealing with the received DLRs (which the host might send to either instance) String prefix = configurationProvider.getPrpt(_msgHashPrefix); - if (Strings.isNullOrEmpty(prefix)) { + if (prefix == null || prefix.isBlank()) { this.messageHashPrefix = getInstanceName(); logger.debug("No message hash prefix has been specified. Auto-created one using instance-name:{}", this.messageHashPrefix); @@ -988,13 +994,19 @@ public PduResponse parseDlrAndCreateResponse(DeliverSm deliverSm) { if (dlrBody.length() > 159) { dlrBody = dlrBody.substring(0, 159); } - String smscid = decodeMessageID(true, receipt.getMessageId()); - if (Strings.isNullOrEmpty(smscid)) { - logger.warn("Invalid smscid: null or empty, skipping unknown dlr {}", MessageTrace.pdu(deliverSm)); + String providerMessageId = decodeMessageID(true, receipt.getMessageId()); + if (Strings.isNullOrEmpty(providerMessageId)) { + logger.warn("Invalid provider message ID, skipping unknown DLR {}", MessageTrace.pdu(deliverSm)); return deliverSm.createGenericNack(SmppConstants.STATUS_SYSERR); } HashMap tlvs = extractTlvs(this.tlvsDlrs, deliverSm); - messageTracker.createAndEnqueueDLR(0, smscid, getHashedMessageID(smscid), from, to, dlrBody, state, errcode, tlvs); + messageTracker.createAndEnqueueDLR(0, providerMessageId, getHashedMessageID(providerMessageId), + from, to, dlrBody, state, errcode, tlvs); + } catch (DlrStorageException e) { + logger.warn("DLR storage unavailable while processing provider receipt {}", MessageTrace.pdu(deliverSm)); + PduResponse resp = deliverSm.createResponse(); + resp.setCommandStatus(SmppConstants.STATUS_SYSERR); + return resp; } catch (Exception e) { //our own extended delivery receipt parsing method will not throw exception for dlr field validation //so this means that something else went really wrong @@ -1119,7 +1131,7 @@ protected void routeMo(M msg) { public void handleResponse(SmppClientSessionHandler handler, int statusCode, String respMessageId, M msg) { if (printResps) { - logger.info("Received response:{}-{} with smscid:{} for msg:{}", + logger.info("Received response:{}-{} with providerMessageId:{} for msg:{}", statusCode, handler.lookupResultMessage(statusCode), respMessageId, msg); } @@ -1167,8 +1179,8 @@ public NackHandlePolicy findPolicyForStatusCode(int statusCode) { } protected void successMessage(String respMessageId, M msg) { - String smscid = updateSendStatusAndSmscId(respMessageId, msg); - logSubmitResponse(SmppConstants.STATUS_OK, smscid, msg); + String providerMessageId = updateSendStatusAndProviderMessageId(respMessageId, msg); + logSubmitResponse(SmppConstants.STATUS_OK, providerMessageId, msg); try { onMessageSuccess(msg); } catch (Exception e) { @@ -1182,9 +1194,9 @@ public void failMessage(int commandStatus, String respMessageId, M msg) { return; } - String smscid = updateSendStatusAndSmscId(respMessageId, msg); - logSubmitResponse(commandStatus, smscid, msg); - String smsid = getHashedMessageID(smscid); + String providerMessageId = updateSendStatusAndProviderMessageId(respMessageId, msg); + logSubmitResponse(commandStatus, providerMessageId, msg); + String hashedProviderMessageId = getHashedMessageID(providerMessageId); String errorCode; if (respErrCodeMap != null && !respErrCodeMap.isEmpty()) { @@ -1194,36 +1206,50 @@ public void failMessage(int commandStatus, String respMessageId, M msg) { errorCode = String.valueOf(StandardMessage.DLR_ERR_SMS_FAILED); } - messageTracker.createAndEnqueueDLR(msg.msgId, smscid, smsid, msg.from, msg.to, "" + commandStatus, - StandardMessage.DLR_STAT_FAILED, errorCode, null); + try { + messageTracker.createAndEnqueueDLR(msg.msgId, providerMessageId, hashedProviderMessageId, + msg.from, msg.to, "" + commandStatus, StandardMessage.DLR_STAT_FAILED, errorCode, null); + } catch (DlrStorageException e) { + // A submit_sm_resp cannot be rejected or retried by this client. Keep the session callback alive and + // leave the tracked state for retention rather than losing the upstream connection as well. + logger.error("Failed to create submission failure DLR providerMessageId={} {}: {}", + MessageTrace.value(providerMessageId), MessageTrace.identifiers(msg), e.getMessage()); + } } - protected void logSubmitResponse(int statusCode, String operatorMsgId, M msg) { + protected void logSubmitResponse(int statusCode, String providerMessageId, M msg) { if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_SUBMIT_RESPONSE)) { - logger.info("message.submit.response worker={} status={} operatorMsgId={} {}", getFullName(), statusCode, - MessageTrace.value(operatorMsgId), MessageTrace.identifiers(msg)); + logger.info("message.submit.response worker={} status={} providerMessageId={} {}", getFullName(), + statusCode, MessageTrace.value(providerMessageId), MessageTrace.identifiers(msg)); } } - public String updateSendStatusAndSmscId(String respMessageId, M msg) { + public String updateSendStatusAndProviderMessageId(String respMessageId, M msg) { if (msg.msgId < 0) { return null; } - //message was sent, we need to record the mapping between smsid and mqid - final String smscid; - if (Strings.isNullOrEmpty(respMessageId)) { - smscid = getInternalSmscId(msg.msgId); + // The message was sent, so record its provider ID against the gateway message ID. + final String providerMessageId; + if (respMessageId == null || respMessageId.isBlank()) { + providerMessageId = getInternalProviderMessageId(msg.msgId); } else { - smscid = decodeMessageID(false, respMessageId); + providerMessageId = decodeMessageID(false, respMessageId); } - final String hashedMessageID = getHashedMessageID(smscid); + final String hashedProviderMessageId = getHashedMessageID(providerMessageId); int size = getThreadCount(); - updateSendStatusAndExtID(hashedMessageID, msg, smscid); - return smscid; + try { + updateSendStatusAndExtID(hashedProviderMessageId, msg, providerMessageId); + } catch (DlrStorageException e) { + // The SMSC has already produced its response, so there is no protocol acknowledgement available to + // request a retry. Isolate the storage failure from Cloudhopper's response callback. + logger.error("Failed to link provider message ID {} {}: {}", + MessageTrace.value(providerMessageId), MessageTrace.identifiers(msg), e.getMessage()); + } + return providerMessageId; } - public String getInternalSmscId(int msgId) { + public String getInternalProviderMessageId(int msgId) { return getFullName() + "_internal_" + msgId; } @@ -1274,7 +1300,7 @@ public String decodeMessageID(boolean dlr, String messageId) { decoded = messageId; } } - logger.debug("Decoded SMSC ID from:{} to:{}", messageId, decoded); + logger.debug("Decoded provider message ID from:{} to:{}", messageId, decoded); return decoded; } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java index f3fe714..2d7433d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java @@ -59,6 +59,10 @@ private boolean persistBatch(List> eventsQueue) { if (eventsQueue.isEmpty()) { return true; } + if (!isDlrPersistenceEnabled()) { + worker.handlePersistedMessages(eventsQueue); + return true; + } List states = new ArrayList<>(eventsQueue.size()); for (InEvent event : eventsQueue) { if (event == null) { @@ -87,6 +91,11 @@ private boolean persistBatch(List> eventsQueue) { return true; } + /** + * Acknowledgement and router admission are always deferred to + * {@link SmppServerWorker#handlePersistedMessages(List)}, so the worker must drain the ingress queue on shutdown + * even when Sendium-owned DLR persistence is disabled and the persist step itself is a no-op. + */ @Override public boolean persistsBeforeAcknowledgement() { return true; @@ -97,6 +106,10 @@ public boolean markAsUnpushed(StandardMessage msg) { if (msg == null || msg.type != StandardMessage.MSG_DLR) { return false; } + if (!isDlrPersistenceEnabled()) { + //let the worker retry in memory, as documented on SmppServerMessageStore#markAsUnpushed + return false; + } try { boolean saved = getDlrService().saveUnpushedDlr(msg); @@ -112,6 +125,10 @@ public boolean markAsUnpushed(StandardMessage msg) { @Override public void onClientConnected(String systemId) { + if (!isDlrPersistenceEnabled()) { + return; + } + DlrService dlrService = getDlrService(); List unpushedDlrs = dlrService.claimUnpushedDlrs(systemId); if (unpushedDlrs.isEmpty()) { @@ -134,6 +151,10 @@ public void onClientConnected(String systemId) { } } + private boolean isDlrPersistenceEnabled() { + return worker.getWorkerResources().isDlrPersistenceEnabled(); + } + private DlrService getDlrService() { return worker.getWorkerResources().getDlrService(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java index 720e87a..3cde255 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java @@ -241,7 +241,7 @@ public boolean stop() { keepOnRunning = false; messagePartsHandler.stop(); boolean drainPersistedIngress = messageStore != null && messageStore.persistsBeforeAcknowledgement(); - if (!drainPersistedIngress) { + if (!drainPersistedIngress && messageStore != null) { messageStore.stop(); } if (inactivityTimeFuture != null) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java index 763624d..6a7116c 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java @@ -10,13 +10,15 @@ default void saveInitialStates(List states) { states.forEach(this::saveInitialState); } - void linkOperatorId(String gatewayMsgId, String operatorMsgId); + void linkProviderMessageId(String gatewayMessageId, String providerName, String providerMessageId); /** * Resolves and removes one provider correlation and its tracked message. - * The returned state must contain the supplied status, the linked operator ID, and an updated timestamp. + * The returned state must contain the supplied status, provider name, linked provider message ID, and an updated + * timestamp. */ - Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status); + Optional resolveAndRemoveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status); Optional getState(String gatewayMsgId); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java index dcb9063..4c9fd68 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java @@ -9,7 +9,7 @@ import java.util.Optional; @ApplicationScoped -@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true", enableIfMissing = false) public class DlrService { @Inject DlrStorage storage; @@ -25,12 +25,13 @@ public void saveInitialStates(List states) { storage.saveInitialStates(states); } - public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { - storage.linkOperatorId(gatewayMsgId, operatorMsgId); + public void linkProviderMessageId(String gatewayMessageId, String providerName, String providerMessageId) { + storage.linkProviderMessageId(gatewayMessageId, providerName, providerMessageId); } - public Optional resolveAndRemoveDlr(String operatorMsgId, int dlrState) { - Optional state = storage.resolveAndRemoveDlr(operatorMsgId, mapDlrState(dlrState)); + public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, int dlrState) { + Optional state = storage.resolveAndRemoveDlr( + providerName, providerMessageId, mapDlrState(dlrState)); state.filter(messageState -> messageState.getForwardDlrUrl() != null) .filter(messageState -> !messageState.getForwardDlrUrl().isEmpty()) .ifPresent(forwardDlrService::forwardDlr); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java index 7836b6e..88049de 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorageReadinessCheck.java @@ -7,13 +7,16 @@ import org.eclipse.microprofile.health.HealthCheckResponse; import org.eclipse.microprofile.health.HealthCheckResponseBuilder; import org.eclipse.microprofile.health.Readiness; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.sql.SQLException; @Readiness @ApplicationScoped -@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true", enableIfMissing = false) public class DlrStorageReadinessCheck implements HealthCheck { + private static final Logger logger = LoggerFactory.getLogger(DlrStorageReadinessCheck.class); private static final String CHECK_NAME = "sendium-dlr-storage"; @Inject @@ -21,15 +24,27 @@ public class DlrStorageReadinessCheck implements HealthCheck { @Override public HealthCheckResponse call() { - HealthCheckResponseBuilder response = HealthCheckResponse.named(CHECK_NAME) - .withData("backend", storage.backend()); + HealthCheckResponseBuilder response = HealthCheckResponse.named(CHECK_NAME); try { + response.withData("backend", storage.backend()); storage.verifyPostgresqlSchema(); return response.up().build(); } catch (SQLException e) { - return response.down() - .withData("reason", "unavailable") - .build(); + // The probe result stays sanitized; the cause is only logged so an outage remains diagnosable. + logger.warn("DLR storage readiness probe failed: sqlState={} errorCode={} reason={}", + e.getSQLState(), e.getErrorCode(), e.getMessage(), e); + return down(response); + } catch (RuntimeException e) { + // An unchecked escape would otherwise reach SmallRye, which replaces the whole payload with the raw + // exception message and drops both the check name and the sanitized reason. + logger.warn("DLR storage readiness probe failed unexpectedly", e); + return down(response); } } + + private HealthCheckResponse down(HealthCheckResponseBuilder response) { + return response.down() + .withData("reason", "unavailable") + .build(); + } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java index e3aea84..29bb5fb 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java @@ -13,7 +13,7 @@ import java.time.Duration; @ApplicationScoped -@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true", enableIfMissing = false) public class ForwardDlrService { private static final Logger logger = LoggerFactory.getLogger(ForwardDlrService.class); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java index 18d5eee..e4ddcc2 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java @@ -43,17 +43,27 @@ public void configure(String key, String newValue, String oldValue) { } @Override - public int updateSendStatusAndExtID(String smsid, StandardMessage pMsg, String smscid) { - smsid = pMsg.serial; //in our case no hash needed - if (smsid != null && !smsid.isEmpty() && smscid != null && !smscid.isEmpty()) { - outWorker.getWorkerResources().getDlrService().linkOperatorId(smsid, smscid); - if (MessageTrace.shouldLog(outWorker.getConfigurationProvider(), MessageTrace.EVENT_OPERATOR_LINKED)) { - logger.info("message.operator.linked operatorMsgId={} {}", MessageTrace.value(smscid), MessageTrace.identifiers(pMsg)); - } - return 1; + public int updateSendStatusAndExtID(String hashedProviderMessageId, StandardMessage message, + String providerMessageId) { + String gatewayMessageId = message.serial; + String providerName = outWorker.getDlrProviderName(); + if (gatewayMessageId == null || gatewayMessageId.isBlank() || + providerName == null || providerName.isBlank() || + providerMessageId == null || providerMessageId.isBlank()) { + logger.warn("Invalid DLR correlation identifiers"); + return 0; } - logger.warn("Invalid parameters: smsid={}, smscid={}", smsid, smscid); - return 0; + if (!outWorker.getWorkerResources().isDlrPersistenceEnabled()) { + return 0; + } + + outWorker.getWorkerResources().getDlrService() + .linkProviderMessageId(gatewayMessageId, providerName, providerMessageId); + if (MessageTrace.shouldLog(outWorker.getConfigurationProvider(), MessageTrace.EVENT_PROVIDER_LINKED)) { + logger.info("message.provider.linked providerMessageId={} {}", MessageTrace.value(providerMessageId), + MessageTrace.identifiers(message)); + } + return 1; } @Override @@ -71,9 +81,15 @@ public String getVendorPriceGateway() { } @Override - public void createAndEnqueueDLR(int mqid, String smscid, String smsid, String from, String to, + public void createAndEnqueueDLR(int mqid, String providerMessageId, String hashedProviderMessageId, + String from, String to, String body, int state, String errorCode, HashMap tlvs) { - Optional optState = outWorker.getWorkerResources().getDlrService().resolveAndRemoveDlr(smscid, state); + if (!outWorker.getWorkerResources().isDlrPersistenceEnabled()) { + return; + } + + Optional optState = outWorker.getWorkerResources().getDlrService() + .resolveAndRemoveDlr(outWorker.getDlrProviderName(), providerMessageId, state); if (optState.isPresent()) { MessageState msgState = optState.get(); @@ -96,10 +112,11 @@ public void createAndEnqueueDLR(int mqid, String smscid, String smsid, String fr outWorker.handleException(ie); } if (MessageTrace.shouldLog(outWorker.getConfigurationProvider(), MessageTrace.EVENT_DLR)) { - logger.info("message.dlr status={} operatorMsgId={} {}", state, MessageTrace.value(smscid), MessageTrace.identifiers(dlrMsg)); + logger.info("message.dlr status={} providerMessageId={} {}", state, + MessageTrace.value(providerMessageId), MessageTrace.identifiers(dlrMsg)); } } else { - logger.warn("DLR received for unknown/expired message: smsid={}", smsid); + logger.warn("DLR received for unknown/expired provider message ID"); } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java index 3fac701..f1dfc75 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java @@ -18,12 +18,14 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; @Startup @ApplicationScoped -@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true") +@IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true", enableIfMissing = false) public class ManagedDlrStorage implements DlrStorage { private static final String METRIC_NAME = "sendium.dlr.storage.operation"; private static final String BACKEND = "postgresql"; @@ -48,6 +50,8 @@ public class ManagedDlrStorage implements DlrStorage { @Inject MeterRegistry meterRegistry; + private final Map timers = new ConcurrentHashMap<>(); + private DlrStorage delegate; private AgroalDataSource selectedPostgresqlDataSource; @@ -93,13 +97,15 @@ public void saveInitialStates(List states) { } @Override - public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { - timed("link_operator", () -> delegate.linkOperatorId(gatewayMsgId, operatorMsgId)); + public void linkProviderMessageId(String gatewayMessageId, String providerName, String providerMessageId) { + timed("link_provider", () -> delegate.linkProviderMessageId( + gatewayMessageId, providerName, providerMessageId)); } @Override - public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { - return timed("resolve", () -> delegate.resolveAndRemoveDlr(operatorMsgId, status)); + public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status) { + return timed("resolve", () -> delegate.resolveAndRemoveDlr(providerName, providerMessageId, status)); } @Override @@ -157,9 +163,9 @@ private void timed(String operation, Runnable action) { } private Timer timer(String operation, String outcome) { - return Timer.builder(METRIC_NAME) + return timers.computeIfAbsent(operation + '/' + outcome, ignored -> Timer.builder(METRIC_NAME) .description("Sendium DLR storage operation latency") .tags("backend", BACKEND, "operation", operation, "outcome", outcome) - .register(meterRegistry); + .register(meterRegistry)); } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java index 25dcecd..180409a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java @@ -15,7 +15,8 @@ public class MessageState implements Serializable { private String systemId; private String sourceAddr; private String destAddr; - private String operatorMsgId; + private String providerName; + private String providerMessageId; private String forwardDlrUrl; private List reassembledParts; private MessageStatus status; @@ -34,7 +35,8 @@ public MessageState(String gatewayMsgId, String accountId, String systemId, Stri this.systemId = systemId; this.sourceAddr = sourceAddr; this.destAddr = destAddr; - this.operatorMsgId = null; + this.providerName = null; + this.providerMessageId = null; this.status = MessageStatus.ACCEPTED; this.timestamp = System.currentTimeMillis(); this.forwardDlrUrl = forwardDlrUrl; @@ -60,8 +62,12 @@ public String getDestAddr() { return destAddr; } - public String getOperatorMsgId() { - return operatorMsgId; + public String getProviderMessageId() { + return providerMessageId; + } + + public String getProviderName() { + return providerName; } public String getForwardDlrUrl() { @@ -80,8 +86,12 @@ public long getTimestamp() { return timestamp; } - public void setOperatorMsgId(String operatorMsgId) { - this.operatorMsgId = operatorMsgId; + public void setProviderMessageId(String providerMessageId) { + this.providerMessageId = providerMessageId; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; } public void setStatus(MessageStatus status) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java index 88edc67..befae5d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java @@ -1,6 +1,8 @@ package gr.cytech.sendium.core.worker; import gr.cytech.sendium.core.message.StandardMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.sql.DataSource; import java.sql.Array; @@ -8,19 +10,29 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; import java.sql.Types; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; public class PostgresqlDlrStorage implements DlrStorage { + private static final Logger logger = LoggerFactory.getLogger(PostgresqlDlrStorage.class); + private static final int DEFAULT_LINK_MAX_ATTEMPTS = 20; private static final long DEFAULT_LINK_RETRY_INTERVAL_MILLIS = 200; private static final long EXPIRY_CHECK_INTERVAL_MILLIS = TimeUnit.HOURS.toMillis(1); @@ -28,14 +40,15 @@ public class PostgresqlDlrStorage implements DlrStorage { private static final String SAVE_INITIAL_STATE_SQL = """ INSERT INTO sendium_dlr.tracked_message (gateway_message_id, account_id, system_id, source_address, destination_address, - operator_message_id, forward_dlr_url, reassembled_parts, status, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + provider_name, provider_message_id, forward_dlr_url, reassembled_parts, status, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (gateway_message_id) DO UPDATE SET account_id = EXCLUDED.account_id, system_id = EXCLUDED.system_id, source_address = EXCLUDED.source_address, destination_address = EXCLUDED.destination_address, - operator_message_id = EXCLUDED.operator_message_id, + provider_name = EXCLUDED.provider_name, + provider_message_id = EXCLUDED.provider_message_id, forward_dlr_url = EXCLUDED.forward_dlr_url, reassembled_parts = EXCLUDED.reassembled_parts, status = EXCLUDED.status, @@ -45,27 +58,52 @@ ON CONFLICT (gateway_message_id) DO UPDATE SET private static final String LINK_MESSAGE_SQL = """ UPDATE sendium_dlr.tracked_message - SET operator_message_id = ?, status = 'SENT', updated_at = CURRENT_TIMESTAMP + SET provider_name = ?, provider_message_id = ?, status = 'SENT', updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? + """; + + private static final String LOCK_MESSAGE_SQL = """ + SELECT 1 + FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? + FOR UPDATE + """; + + private static final String LOCK_CORRELATION_SQL = """ + SELECT pg_advisory_xact_lock(hashtextextended(?, 0) # hashtextextended(?, 1)) + """; + + private static final String GET_CORRELATION_OWNER_SQL = """ + SELECT gateway_message_id + FROM sendium_dlr.provider_correlation + WHERE provider_name = ? AND provider_message_id = ? """; private static final String SAVE_CORRELATION_SQL = """ - INSERT INTO sendium_dlr.operator_correlation - (operator_message_id, gateway_message_id) - VALUES (?, ?) - ON CONFLICT (operator_message_id) DO UPDATE SET - created_at = CURRENT_TIMESTAMP - WHERE operator_correlation.gateway_message_id = EXCLUDED.gateway_message_id + WITH saved_correlation AS ( + INSERT INTO sendium_dlr.provider_correlation + (provider_name, provider_message_id, gateway_message_id) + VALUES (?, ?, ?) + ON CONFLICT (provider_name, provider_message_id) DO UPDATE SET + gateway_message_id = EXCLUDED.gateway_message_id, + created_at = CURRENT_TIMESTAMP + RETURNING gateway_message_id + ) + UPDATE sendium_dlr.tracked_message + SET provider_name = NULL, provider_message_id = NULL, updated_at = CURRENT_TIMESTAMP + WHERE provider_name = ? AND provider_message_id = ? + AND gateway_message_id <> ? + AND EXISTS (SELECT 1 FROM saved_correlation) """; private static final String DELETE_CORRELATIONS_SQL = """ - DELETE FROM sendium_dlr.operator_correlation + DELETE FROM sendium_dlr.provider_correlation WHERE gateway_message_id = ? """; private static final String GET_STATE_SQL = """ SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, - tm.destination_address, tm.operator_message_id, tm.forward_dlr_url, + tm.destination_address, tm.provider_name, tm.provider_message_id, tm.forward_dlr_url, tm.reassembled_parts, tm.status, tm.updated_at FROM sendium_dlr.tracked_message tm WHERE tm.gateway_message_id = ? @@ -74,12 +112,14 @@ ON CONFLICT (operator_message_id) DO UPDATE SET private static final String RESOLVE_STATE_SQL = """ SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, tm.destination_address, tm.forward_dlr_url, tm.reassembled_parts, - correlation.operator_message_id, CURRENT_TIMESTAMP AS resolved_at - FROM sendium_dlr.operator_correlation correlation + correlation.provider_name, correlation.provider_message_id, + CURRENT_TIMESTAMP AS resolved_at + FROM sendium_dlr.provider_correlation correlation JOIN sendium_dlr.tracked_message tm ON tm.gateway_message_id = correlation.gateway_message_id - WHERE correlation.operator_message_id = ? - FOR UPDATE OF tm, correlation + WHERE correlation.provider_name = ? AND correlation.provider_message_id = ? + AND correlation.gateway_message_id = ? + FOR UPDATE OF correlation """; private static final String DELETE_STATE_SQL = """ @@ -94,20 +134,28 @@ ON CONFLICT (operator_message_id) DO UPDATE SET """; private static final String DELETE_EXPIRED_CORRELATIONS_SQL = """ - DELETE FROM sendium_dlr.operator_correlation + DELETE FROM sendium_dlr.provider_correlation WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '3 days' """; private static final String DELETE_EXPIRED_MESSAGES_SQL = """ - DELETE FROM sendium_dlr.tracked_message - WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' + WITH expired_messages AS ( + SELECT gateway_message_id + FROM sendium_dlr.tracked_message + WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' + ORDER BY gateway_message_id + FOR UPDATE + ) + DELETE FROM sendium_dlr.tracked_message message + USING expired_messages expired + WHERE message.gateway_message_id = expired.gateway_message_id """; private static final String SAVE_UNPUSHED_DLR_SQL = """ INSERT INTO sendium_dlr.unpushed_dlr (dlr_key, system_id, account_id, source_address, destination_address, serial, - message_id, dlr_state, error_code, acked, priority, reassembled_parts) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + message_id, dlr_state, error_code, acked, priority, reassembled_parts, generation_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (dlr_key) DO UPDATE SET system_id = EXCLUDED.system_id, account_id = EXCLUDED.account_id, @@ -120,12 +168,13 @@ ON CONFLICT (dlr_key) DO UPDATE SET acked = EXCLUDED.acked, priority = EXCLUDED.priority, reassembled_parts = EXCLUDED.reassembled_parts, + generation_id = EXCLUDED.generation_id, created_at = CURRENT_TIMESTAMP """; private static final String GET_UNPUSHED_DLRS_SQL = """ SELECT dlr_key, system_id, account_id, source_address, destination_address, serial, - message_id, dlr_state, error_code, acked, priority, reassembled_parts + message_id, dlr_state, error_code, acked, priority, reassembled_parts, generation_id FROM sendium_dlr.unpushed_dlr WHERE system_id = ? ORDER BY created_at, dlr_key @@ -133,13 +182,13 @@ ON CONFLICT (dlr_key) DO UPDATE SET private static final String DELETE_UNPUSHED_DLR_SQL = """ DELETE FROM sendium_dlr.unpushed_dlr - WHERE dlr_key = ? + WHERE dlr_key = ? AND generation_id = ? """; private static final String DELETE_EXPIRED_UNPUSHED_DLRS_SQL = """ DELETE FROM sendium_dlr.unpushed_dlr WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' - RETURNING dlr_key + RETURNING dlr_key, generation_id """; private final DataSource dataSource; @@ -147,7 +196,9 @@ ON CONFLICT (dlr_key) DO UPDATE SET private final long linkRetryIntervalMillis; private final long expiryCheckIntervalMillis; private final Object unpushedDlrStateLock = new Object(); - private final Set claimedUnpushedDlrKeys = ConcurrentHashMap.newKeySet(); + private final ConcurrentHashMap claimedUnpushedDlrKeys = new ConcurrentHashMap<>(); + private final IdentityHashMap claimedUnpushedDlrGenerations = new IdentityHashMap<>(); + private final AtomicBoolean expiryInProgress = new AtomicBoolean(); private volatile long lastExpiryCheck; public PostgresqlDlrStorage(DataSource dataSource) { @@ -183,12 +234,19 @@ public void saveInitialStates(List states) { return; } - List checkedStates = states.stream() + List suppliedStates = states.stream() .map(state -> Objects.requireNonNull(state, "state")) .toList(); - List gatewayMsgIds = checkedStates.stream() - .map(state -> parseGatewayId(state.getGatewayMsgId())) - .toList(); + suppliedStates.forEach(this::validateCorrelationFields); + Map finalStatesByGateway = new LinkedHashMap<>(); + for (MessageState state : suppliedStates) { + UUID gatewayMsgId = parseGatewayId(state.getGatewayMsgId()); + // Reinsert duplicate IDs so batch order reflects each gateway's final occurrence. + finalStatesByGateway.remove(gatewayMsgId); + finalStatesByGateway.put(gatewayMsgId, state); + } + List gatewayMsgIds = List.copyOf(finalStatesByGateway.keySet()); + List checkedStates = List.copyOf(finalStatesByGateway.values()); checkExpiry(); try (Connection connection = dataSource.getConnection()) { @@ -196,6 +254,32 @@ public void saveInitialStates(List states) { try (PreparedStatement saveStates = connection.prepareStatement(SAVE_INITIAL_STATE_SQL); PreparedStatement deleteCorrelations = connection.prepareStatement(DELETE_CORRELATIONS_SQL); PreparedStatement saveCorrelations = connection.prepareStatement(SAVE_CORRELATION_SQL)) { + List correlatedStates = checkedStates.stream() + .filter(state -> state.getProviderMessageId() != null) + .sorted(Comparator.comparing(MessageState::getProviderName) + .thenComparing(MessageState::getProviderMessageId)) + .toList(); + String previousProviderName = null; + String previousProviderMessageId = null; + for (MessageState state : correlatedStates) { + if (!state.getProviderName().equals(previousProviderName) || + !state.getProviderMessageId().equals(previousProviderMessageId)) { + lockCorrelation(connection, state.getProviderName(), state.getProviderMessageId()); + previousProviderName = state.getProviderName(); + previousProviderMessageId = state.getProviderMessageId(); + } + } + List messageIdsToLock = new ArrayList<>(gatewayMsgIds); + for (MessageState state : correlatedStates) { + findCorrelationOwner(connection, state.getProviderName(), state.getProviderMessageId()) + .ifPresent(messageIdsToLock::add); + } + for (UUID messageId : messageIdsToLock.stream() + .distinct() + .sorted(Comparator.comparing(UUID::toString)) + .toList()) { + lockMessage(connection, messageId); + } int correlationCount = 0; for (int index = 0; index < checkedStates.size(); index++) { MessageState state = checkedStates.get(index); @@ -204,9 +288,10 @@ public void saveInitialStates(List states) { saveStates.addBatch(); deleteCorrelations.setObject(1, gatewayMsgId); deleteCorrelations.addBatch(); - if (state.getOperatorMsgId() != null) { - saveCorrelations.setString(1, state.getOperatorMsgId()); - saveCorrelations.setObject(2, gatewayMsgId); + if (state.getProviderMessageId() != null && + isLastCorrelationOwner(checkedStates, index, state)) { + setCorrelationParameters(saveCorrelations, state.getProviderName(), + state.getProviderMessageId(), gatewayMsgId); saveCorrelations.addBatch(); correlationCount++; } @@ -215,11 +300,7 @@ public void saveInitialStates(List states) { saveStates.executeBatch(); deleteCorrelations.executeBatch(); if (correlationCount > 0) { - for (int result : saveCorrelations.executeBatch()) { - if (result == 0 || result == Statement.EXECUTE_FAILED) { - throw new SQLException("Operator message ID is already linked to another gateway message"); - } - } + saveCorrelations.executeBatch(); } connection.commit(); } catch (SQLException e) { @@ -232,30 +313,41 @@ public void saveInitialStates(List states) { } @Override - public void linkOperatorId(String gatewayMsgId, String operatorMsgId) { + public void linkProviderMessageId(String gatewayMessageId, String providerName, String providerMessageId) { checkExpiry(); - UUID gatewayId = parseGatewayId(gatewayMsgId); + requireCorrelation(providerName, providerMessageId); + UUID gatewayId = parseGatewayId(gatewayMessageId); for (int attempt = 0; attempt < linkMaxAttempts; attempt++) { - if (tryLinkOperatorId(gatewayId, operatorMsgId)) { + if (tryLinkProviderMessageId(gatewayId, providerName, providerMessageId)) { return; } if (attempt + 1 < linkMaxAttempts) { sleepBeforeLinkRetry(); } } - throw new DlrStorageException("Gateway message state not found while linking operator ID"); + throw new DlrStorageException("Gateway message state not found while linking provider message ID"); } @Override - public Optional resolveAndRemoveDlr(String operatorMsgId, MessageState.MessageStatus status) { + public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status) { Objects.requireNonNull(status, "status"); + requireCorrelation(providerName, providerMessageId); checkExpiry(); try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); try { - Optional state = lockResolvedState(connection, operatorMsgId, status); + lockCorrelation(connection, providerName, providerMessageId); + Optional gatewayMessageId = findCorrelationOwner( + connection, providerName, providerMessageId); + if (gatewayMessageId.isEmpty() || !lockMessage(connection, gatewayMessageId.get())) { + connection.rollback(); + return Optional.empty(); + } + Optional state = lockResolvedState( + connection, providerName, providerMessageId, gatewayMessageId.get(), status); if (state.isEmpty()) { connection.rollback(); return Optional.empty(); @@ -280,7 +372,7 @@ public Optional getState(String gatewayMsgId) { PreparedStatement statement = connection.prepareStatement(GET_STATE_SQL)) { statement.setObject(1, parseGatewayId(gatewayMsgId)); try (ResultSet resultSet = statement.executeQuery()) { - return resultSet.next() ? Optional.of(readState(resultSet)) : Optional.empty(); + return resultSet.next() ? Optional.of(readTrackedState(resultSet)) : Optional.empty(); } } catch (SQLException e) { throw failure("read DLR state", e); @@ -322,6 +414,7 @@ public boolean saveUnpushedDlr(StandardMessage message) { statement.setBoolean(10, dlr.acked); statement.setInt(11, dlr.priority); setStringArray(connection, statement, 12, dlr.reassembledParts); + statement.setObject(13, UUID.randomUUID()); return statement.executeUpdate() == 1; } catch (SQLException e) { throw failure("save unpushed DLR", e); @@ -346,11 +439,17 @@ public boolean removeUnpushedDlr(StandardMessage message) { String key = getUnpushedDlrKey(message); synchronized (unpushedDlrStateLock) { + UUID generationId = claimedUnpushedDlrGenerations.get(message); + if (generationId == null) { + return false; + } try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(DELETE_UNPUSHED_DLR_SQL)) { statement.setString(1, key); + statement.setObject(2, generationId); boolean removed = statement.executeUpdate() == 1; - claimedUnpushedDlrKeys.remove(key); + claimedUnpushedDlrGenerations.remove(message); + claimedUnpushedDlrKeys.remove(key, generationId); return removed; } catch (SQLException e) { throw failure("remove unpushed DLR", e); @@ -365,7 +464,10 @@ public void releaseUnpushedDlrClaim(StandardMessage message) { } synchronized (unpushedDlrStateLock) { - claimedUnpushedDlrKeys.remove(getUnpushedDlrKey(message)); + UUID generationId = claimedUnpushedDlrGenerations.remove(message); + if (generationId != null) { + claimedUnpushedDlrKeys.remove(getUnpushedDlrKey(message), generationId); + } } } @@ -383,8 +485,13 @@ private List loadUnpushedDlrs(String systemId, boolean claimFor List messages = new ArrayList<>(); while (resultSet.next()) { String key = resultSet.getString("dlr_key"); - if (!claimForReplay || claimedUnpushedDlrKeys.add(key)) { - messages.add(readUnpushedDlr(resultSet).toMessage()); + UUID generationId = resultSet.getObject("generation_id", UUID.class); + StandardMessage message = readUnpushedDlr(resultSet).toMessage(); + if (!claimForReplay) { + messages.add(message); + } else if (claimedUnpushedDlrKeys.putIfAbsent(key, generationId) == null) { + claimedUnpushedDlrGenerations.put(message, generationId); + messages.add(message); } } return messages; @@ -429,16 +536,32 @@ private String nullToEmpty(String value) { return value == null ? "" : value; } - private boolean tryLinkOperatorId(UUID gatewayMsgId, String operatorMsgId) { + private boolean tryLinkProviderMessageId(UUID gatewayMessageId, String providerName, String providerMessageId) { try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); try { - if (!markAsSent(connection, gatewayMsgId, operatorMsgId)) { + lockCorrelation(connection, providerName, providerMessageId); + Optional previousOwner = findCorrelationOwner(connection, providerName, providerMessageId); + List messageIdsToLock = new ArrayList<>(); + messageIdsToLock.add(gatewayMessageId); + previousOwner.ifPresent(messageIdsToLock::add); + boolean targetFound = false; + for (UUID messageId : messageIdsToLock.stream() + .distinct() + .sorted(Comparator.comparing(UUID::toString)) + .toList()) { + boolean found = lockMessage(connection, messageId); + if (messageId.equals(gatewayMessageId)) { + targetFound = found; + } + } + if (!targetFound) { connection.rollback(); return false; } - if (!saveCorrelation(connection, gatewayMsgId, operatorMsgId)) { - throw new SQLException("Operator message ID is already linked to another gateway message"); + saveCorrelation(connection, providerName, providerMessageId, gatewayMessageId); + if (!markAsSent(connection, gatewayMessageId, providerName, providerMessageId)) { + throw new SQLException("Gateway message state disappeared while linking provider message ID"); } connection.commit(); return true; @@ -447,15 +570,47 @@ private boolean tryLinkOperatorId(UUID gatewayMsgId, String operatorMsgId) { throw e; } } catch (SQLException e) { - throw failure("link operator DLR ID", e); + throw failure("link provider DLR ID", e); } } - private boolean markAsSent(Connection connection, UUID gatewayMsgId, - String operatorMsgId) throws SQLException { + private void lockCorrelation(Connection connection, String providerName, + String providerMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(LOCK_CORRELATION_SQL)) { + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.execute(); + } + } + + private Optional findCorrelationOwner(Connection connection, String providerName, + String providerMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(GET_CORRELATION_OWNER_SQL)) { + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + try (ResultSet resultSet = statement.executeQuery()) { + return resultSet.next() ? + Optional.of(resultSet.getObject("gateway_message_id", UUID.class)) + : Optional.empty(); + } + } + } + + private boolean lockMessage(Connection connection, UUID gatewayMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(LOCK_MESSAGE_SQL)) { + statement.setObject(1, gatewayMsgId); + try (ResultSet resultSet = statement.executeQuery()) { + return resultSet.next(); + } + } + } + + private boolean markAsSent(Connection connection, UUID gatewayMessageId, + String providerName, String providerMessageId) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(LINK_MESSAGE_SQL)) { - statement.setString(1, operatorMsgId); - statement.setObject(2, gatewayMsgId); + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setObject(3, gatewayMessageId); return statement.executeUpdate() == 1; } } @@ -467,35 +622,72 @@ private void setStateParameters(Connection connection, PreparedStatement stateme statement.setString(3, state.getSystemId()); statement.setString(4, state.getSourceAddr()); statement.setString(5, state.getDestAddr()); - statement.setString(6, state.getOperatorMsgId()); - statement.setString(7, state.getForwardDlrUrl()); - setStringArray(connection, statement, 8, state.getReassembledParts()); - statement.setString(9, state.getStatus().name()); - statement.setTimestamp(10, new Timestamp(state.getTimestamp())); + statement.setString(6, state.getProviderName()); + statement.setString(7, state.getProviderMessageId()); + statement.setString(8, state.getForwardDlrUrl()); + setStringArray(connection, statement, 9, state.getReassembledParts()); + statement.setString(10, state.getStatus().name()); + statement.setObject(11, OffsetDateTime.ofInstant( + Instant.ofEpochMilli(state.getTimestamp()), ZoneOffset.UTC)); } - private boolean saveCorrelation(Connection connection, UUID gatewayMsgId, - String operatorMsgId) throws SQLException { + private void saveCorrelation(Connection connection, String providerName, + String providerMessageId, UUID gatewayMessageId) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(SAVE_CORRELATION_SQL)) { - statement.setString(1, operatorMsgId); - statement.setObject(2, gatewayMsgId); - return statement.executeUpdate() == 1; + setCorrelationParameters(statement, providerName, providerMessageId, gatewayMessageId); + statement.executeUpdate(); } } - private Optional lockResolvedState(Connection connection, String operatorMsgId, - MessageState.MessageStatus status) throws SQLException { + private void setCorrelationParameters(PreparedStatement statement, String providerName, + String providerMessageId, UUID gatewayMessageId) throws SQLException { + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setObject(3, gatewayMessageId); + statement.setString(4, providerName); + statement.setString(5, providerMessageId); + statement.setObject(6, gatewayMessageId); + } + + private void validateCorrelationFields(MessageState state) { + if (state.getProviderName() == null && state.getProviderMessageId() == null) { + return; + } + if (state.getProviderName() == null || state.getProviderName().isBlank() || + state.getProviderMessageId() == null || state.getProviderMessageId().isBlank()) { + throw new IllegalArgumentException( + "Provider name and provider message ID must either both be set or both be absent"); + } + } + + private boolean isLastCorrelationOwner(List states, int index, MessageState candidate) { + for (int laterIndex = index + 1; laterIndex < states.size(); laterIndex++) { + MessageState later = states.get(laterIndex); + if (candidate.getProviderName().equals(later.getProviderName()) && + candidate.getProviderMessageId().equals(later.getProviderMessageId())) { + return false; + } + } + return true; + } + + private void requireCorrelation(String providerName, String providerMessageId) { + if (providerName == null || providerName.isBlank() || + providerMessageId == null || providerMessageId.isBlank()) { + throw new IllegalArgumentException("Provider name and provider message ID must not be blank"); + } + } + + private Optional lockResolvedState(Connection connection, String providerName, + String providerMessageId, + UUID gatewayMessageId, + MessageState.MessageStatus status) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(RESOLVE_STATE_SQL)) { - statement.setString(1, operatorMsgId); + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setObject(3, gatewayMessageId); try (ResultSet resultSet = statement.executeQuery()) { - if (!resultSet.next()) { - return Optional.empty(); - } - MessageState state = readState(resultSet); - state.setOperatorMsgId(resultSet.getString("operator_message_id")); - state.setStatus(status); - state.setTimestamp(resultSet.getTimestamp("resolved_at").getTime()); - return Optional.of(state); + return resultSet.next() ? Optional.of(readResolvedState(resultSet, status)) : Optional.empty(); } } } @@ -507,7 +699,7 @@ private void deleteState(Connection connection, UUID gatewayMsgId) throws SQLExc } } - private MessageState readState(ResultSet resultSet) throws SQLException { + private MessageState readBaseState(ResultSet resultSet) throws SQLException { MessageState state = new MessageState( resultSet.getObject("gateway_message_id", UUID.class).toString(), resultSet.getString("account_id"), @@ -515,24 +707,36 @@ private MessageState readState(ResultSet resultSet) throws SQLException { resultSet.getString("source_address"), resultSet.getString("destination_address"), resultSet.getString("forward_dlr_url")); - state.setOperatorMsgId(resultSet.getString("operator_message_id")); - if (hasColumn(resultSet, "status")) { - state.setStatus(MessageState.MessageStatus.valueOf(resultSet.getString("status"))); - } + state.setProviderName(resultSet.getString("provider_name")); + state.setProviderMessageId(resultSet.getString("provider_message_id")); state.setReassembledParts(readStringArray(resultSet, "reassembled_parts")); - if (hasColumn(resultSet, "updated_at")) { - state.setTimestamp(resultSet.getTimestamp("updated_at").getTime()); - } return state; } - private boolean hasColumn(ResultSet resultSet, String columnName) throws SQLException { - for (int index = 1; index <= resultSet.getMetaData().getColumnCount(); index++) { - if (columnName.equalsIgnoreCase(resultSet.getMetaData().getColumnLabel(index))) { - return true; - } - } - return false; + /** + * Reads a {@link #GET_STATE_SQL} row, which carries the stored status and update time. + */ + private MessageState readTrackedState(ResultSet resultSet) throws SQLException { + MessageState state = readBaseState(resultSet); + state.setStatus(MessageState.MessageStatus.valueOf(resultSet.getString("status"))); + state.setTimestamp(readEpochMillis(resultSet, "updated_at")); + return state; + } + + /** + * Reads a {@link #RESOLVE_STATE_SQL} row, which carries the provider message ID and resolution time. + */ + private MessageState readResolvedState(ResultSet resultSet, + MessageState.MessageStatus status) throws SQLException { + MessageState state = readBaseState(resultSet); + state.setStatus(status); + state.setTimestamp(readEpochMillis(resultSet, "resolved_at")); + return state; + } + + private long readEpochMillis(ResultSet resultSet, String columnName) throws SQLException { + OffsetDateTime value = resultSet.getObject(columnName, OffsetDateTime.class); + return value == null ? 0L : value.toInstant().toEpochMilli(); } private List readStringArray(ResultSet resultSet, String columnName) throws SQLException { @@ -540,7 +744,8 @@ private List readStringArray(ResultSet resultSet, String columnName) thr if (array == null) { return null; } - return new ArrayList<>(List.of((String[]) array.getArray())); + //Arrays.asList, not List.of: a text[] column can legally hold NULL elements + return new ArrayList<>(Arrays.asList((String[]) array.getArray())); } private void setStringArray(Connection connection, PreparedStatement statement, int index, @@ -557,48 +762,65 @@ private void sleepBeforeLinkRetry() { Thread.sleep(linkRetryIntervalMillis); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new DlrStorageException("Interrupted while linking operator ID", e); + throw new DlrStorageException("Interrupted while linking provider message ID", e); } } + /** + * Runs retention cleanup at most once per interval, on the thread that first observes the interval has elapsed. + * Cleanup is best-effort maintenance: at most one thread runs it, every other caller proceeds immediately, and a + * failed pass is logged and retried after the next interval instead of failing the operation that triggered it. + */ private void checkExpiry() { - long now = System.currentTimeMillis(); - if (now - lastExpiryCheck < expiryCheckIntervalMillis) { + if (System.currentTimeMillis() - lastExpiryCheck < expiryCheckIntervalMillis) { + return; + } + if (!expiryInProgress.compareAndSet(false, true)) { return; } - synchronized (this) { - if (now - lastExpiryCheck < expiryCheckIntervalMillis) { + try { + if (System.currentTimeMillis() - lastExpiryCheck < expiryCheckIntervalMillis) { return; } deleteExpiredState(); - lastExpiryCheck = now; + } catch (RuntimeException e) { + logger.warn("DLR retention cleanup failed; retrying after the next interval"); + } finally { + lastExpiryCheck = System.currentTimeMillis(); + expiryInProgress.set(false); } } private void deleteExpiredState() { - synchronized (unpushedDlrStateLock) { - List expiredUnpushedDlrKeys = new ArrayList<>(); - try (Connection connection = dataSource.getConnection()) { - connection.setAutoCommit(false); - try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); - PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL); - PreparedStatement unpushedDlrs = connection.prepareStatement(DELETE_EXPIRED_UNPUSHED_DLRS_SQL)) { - correlations.executeUpdate(); - messages.executeUpdate(); - try (ResultSet resultSet = unpushedDlrs.executeQuery()) { - while (resultSet.next()) { - expiredUnpushedDlrKeys.add(resultSet.getString("dlr_key")); - } + Map expiredUnpushedDlrKeys = new HashMap<>(); + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); + PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL); + PreparedStatement unpushedDlrs = connection.prepareStatement(DELETE_EXPIRED_UNPUSHED_DLRS_SQL)) { + // Rebinding also locks tracked messages before correlations; retain the same order to avoid deadlocks. + messages.executeUpdate(); + correlations.executeUpdate(); + try (ResultSet resultSet = unpushedDlrs.executeQuery()) { + while (resultSet.next()) { + expiredUnpushedDlrKeys.put( + resultSet.getString("dlr_key"), + resultSet.getObject("generation_id", UUID.class)); } - connection.commit(); - claimedUnpushedDlrKeys.removeAll(expiredUnpushedDlrKeys); - } catch (SQLException e) { - rollback(connection, e); - throw e; } + connection.commit(); } catch (SQLException e) { - throw failure("expire DLR state", e); + rollback(connection, e); + throw e; } + } catch (SQLException e) { + throw failure("expire DLR state", e); + } + synchronized (unpushedDlrStateLock) { + expiredUnpushedDlrKeys.forEach(claimedUnpushedDlrKeys::remove); + HashSet expiredGenerations = new HashSet<>(expiredUnpushedDlrKeys.values()); + claimedUnpushedDlrGenerations.entrySet() + .removeIf(entry -> expiredGenerations.contains(entry.getValue())); } } @@ -618,7 +840,15 @@ private void rollback(Connection connection, SQLException failure) { } } + /** + * Logs the database cause server-side and returns a caller-facing exception that carries no connection details. + * The one-line summary keeps an outage diagnosable without a stack trace per rejected message; the full cause is + * available at debug level. + */ private DlrStorageException failure(String operation, SQLException cause) { + logger.error("Failed to {}: sqlState={} errorCode={} reason={}", + operation, cause.getSQLState(), cause.getErrorCode(), cause.getMessage()); + logger.debug("DLR storage failure details while attempting to {}", operation, cause); return new DlrStorageException("Failed to " + operation, cause); } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java index bbf86d4..d5cbd50 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java @@ -12,14 +12,15 @@ public interface Tracker { void configure(String key, String newValue, String oldValue); - int updateSendStatusAndExtID(String smsid, M pMsg, String smscid); + int updateSendStatusAndExtID(String hashedProviderMessageId, M message, String providerMessageId); String getHashedMessageID(String messageId); String getVendorPriceGateway(); - void createAndEnqueueDLR(int mqid, String smscid, String smsid, String from, String to, String body, - int state, String errorCode, HashMap tlvs); + void createAndEnqueueDLR(int mqid, String providerMessageId, String hashedProviderMessageId, + String from, String to, String body, int state, String errorCode, + HashMap tlvs); int getConfiguredMccMnc(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java index 5f8ea0c..29805fa 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/external/WorkerResourceProvider.java @@ -41,6 +41,15 @@ public CredentialFileWatcher getCredentialFileWatcher() { return credentialFileWatcher; } + /** + * Whether Sendium owns the DLR persistence lifecycle in this build. Callers on message paths must check this + * before {@link #getDlrService()}: an application embedding {@code sendium-core} can leave + * {@code sendium.dlr.persistence.enabled} unset and take over DLR tracking itself. + */ + public boolean isDlrPersistenceEnabled() { + return !dlrServices.isUnsatisfied(); + } + public DlrService getDlrService() { if (dlrServices.isUnsatisfied()) { throw new IllegalStateException("Sendium DLR persistence is disabled"); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/util/MessageTrace.java b/sendium-core/src/main/java/gr/cytech/sendium/util/MessageTrace.java index f79353e..3eae6e6 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/util/MessageTrace.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/util/MessageTrace.java @@ -17,7 +17,7 @@ public final class MessageTrace { public static final String EVENT_DELIVERY_FAILED = "message.delivery.failed"; public static final String EVENT_DELIVERY_RETRY = "message.delivery.retry"; public static final String EVENT_ENQUEUED = "message.enqueued"; - public static final String EVENT_OPERATOR_LINKED = "message.operator.linked"; + public static final String EVENT_PROVIDER_LINKED = "message.provider.linked"; public static final String EVENT_ROUTED = "message.routed"; public static final String EVENT_ROUTING_MISS = "message.routing.miss"; public static final String EVENT_SUBMITTED = "message.submitted"; diff --git a/sendium-core/src/main/resources/application.properties b/sendium-core/src/main/resources/application.properties index 5337799..abddac9 100644 --- a/sendium-core/src/main/resources/application.properties +++ b/sendium-core/src/main/resources/application.properties @@ -2,11 +2,12 @@ %test.smsg.properties.file.path=src/test/resources/smsg.properties %test.smsg.credentials.file.path=src/test/resources/credentials.yml -# Sendium-owned DLR persistence is enabled by the standalone application. -sendium.dlr.persistence.enabled=false +# Sendium-owned DLR persistence is opt-in and is enabled by the standalone application. +# The flag is deliberately left undefined here: the DLR beans use enableIfMissing=false, so an embedding +# application enables the subsystem by declaring the property itself rather than by overriding this file. quarkus.datasource.devservices.enabled=false quarkus.datasource.dlr.db-kind=postgresql -quarkus.datasource.dlr.active=${sendium.dlr.persistence.enabled} +quarkus.datasource.dlr.active=${sendium.dlr.persistence.enabled:false} quarkus.datasource.dlr.devservices.enabled=false quarkus.datasource.dlr.jdbc.min-size=0 quarkus.datasource.dlr.jdbc.max-size=10 diff --git a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql index aac2c28..b41086b 100644 --- a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql +++ b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql @@ -6,12 +6,19 @@ CREATE TABLE sendium_dlr.tracked_message ( system_id TEXT, source_address TEXT, destination_address TEXT, - operator_message_id TEXT, + provider_name TEXT, + provider_message_id TEXT, forward_dlr_url TEXT, reassembled_parts TEXT[], status TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT tracked_message_provider_pair_check + CHECK ((provider_name IS NULL) = (provider_message_id IS NULL)), + CONSTRAINT tracked_message_provider_name_not_blank + CHECK (provider_name IS NULL OR provider_name !~ '^[[:space:]]*$'), + CONSTRAINT tracked_message_provider_message_id_not_blank + CHECK (provider_message_id IS NULL OR provider_message_id !~ '^[[:space:]]*$'), CONSTRAINT tracked_message_status_check CHECK (status IN ('ACCEPTED', 'SENT', 'DELIVERED', 'FAILED')) ); @@ -19,21 +26,31 @@ CREATE TABLE sendium_dlr.tracked_message ( CREATE INDEX tracked_message_created_at_idx ON sendium_dlr.tracked_message (created_at); -CREATE TABLE sendium_dlr.operator_correlation ( - operator_message_id TEXT PRIMARY KEY, +CREATE INDEX tracked_message_provider_message_id_idx + ON sendium_dlr.tracked_message (provider_name, provider_message_id) + WHERE provider_message_id IS NOT NULL; + +CREATE TABLE sendium_dlr.provider_correlation ( + provider_name TEXT NOT NULL, + provider_message_id TEXT NOT NULL, gateway_message_id UUID NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT operator_correlation_message_fk + PRIMARY KEY (provider_name, provider_message_id), + CONSTRAINT provider_correlation_provider_name_not_blank + CHECK (provider_name !~ '^[[:space:]]*$'), + CONSTRAINT provider_correlation_provider_message_id_not_blank + CHECK (provider_message_id !~ '^[[:space:]]*$'), + CONSTRAINT provider_correlation_message_fk FOREIGN KEY (gateway_message_id) REFERENCES sendium_dlr.tracked_message (gateway_message_id) ON DELETE CASCADE ); -CREATE INDEX operator_correlation_created_at_idx - ON sendium_dlr.operator_correlation (created_at); +CREATE INDEX provider_correlation_created_at_idx + ON sendium_dlr.provider_correlation (created_at); -CREATE INDEX operator_correlation_gateway_message_idx - ON sendium_dlr.operator_correlation (gateway_message_id); +CREATE INDEX provider_correlation_gateway_message_idx + ON sendium_dlr.provider_correlation (gateway_message_id); CREATE TABLE sendium_dlr.unpushed_dlr ( dlr_key TEXT PRIMARY KEY, @@ -48,6 +65,7 @@ CREATE TABLE sendium_dlr.unpushed_dlr ( acked BOOLEAN NOT NULL, priority INTEGER NOT NULL, reassembled_parts TEXT[], + generation_id UUID NOT NULL DEFAULT gen_random_uuid(), created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT unpushed_dlr_system_id_not_blank CHECK (system_id !~ '^[[:space:]]*$') diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java index a7591e8..a0ef084 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java @@ -13,6 +13,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -58,18 +59,21 @@ void migrationCreatesExpectedTablesAndIndexes() throws SQLException { try (Connection connection = connection()) { assertThat(loadNames(connection, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'sendium_dlr'")) - .containsExactlyInAnyOrder("tracked_message", "operator_correlation", "unpushed_dlr"); + .containsExactlyInAnyOrder("tracked_message", "provider_correlation", "unpushed_dlr"); assertThat(loadNames(connection, "SELECT indexname FROM pg_indexes WHERE schemaname = 'sendium_dlr'")) .contains("tracked_message_created_at_idx", - "operator_correlation_created_at_idx", - "operator_correlation_gateway_message_idx", + "tracked_message_provider_message_id_idx", + "provider_correlation_created_at_idx", + "provider_correlation_gateway_message_idx", "unpushed_dlr_system_created_at_idx", "unpushed_dlr_created_at_idx"); assertThat(loadColumnType(connection, "tracked_message", "gateway_message_id")) .isEqualTo("uuid"); - assertThat(loadColumnType(connection, "operator_correlation", "gateway_message_id")) + assertThat(loadColumnType(connection, "provider_correlation", "gateway_message_id")) .isEqualTo("uuid"); + assertThat(loadColumnType(connection, "provider_correlation", "provider_name")) + .isEqualTo("text"); } } @@ -94,12 +98,33 @@ void trackedMessageRejectsUnknownStatus() throws SQLException { } } + @Test + void providerCorrelationFieldsRejectBlankValues() throws SQLException { + try (Connection connection = connection()) { + for (String blank : List.of("", " ", "\t\n")) { + assertThatThrownBy(() -> insertTrackedMessageWithProvider( + connection, UUID.randomUUID(), blank, "provider-message")) + .isInstanceOf(SQLException.class); + assertThatThrownBy(() -> insertTrackedMessageWithProvider( + connection, UUID.randomUUID(), "provider", blank)) + .isInstanceOf(SQLException.class); + } + + UUID gatewayMessageId = UUID.randomUUID(); + insertTrackedMessage(connection, gatewayMessageId); + assertThatThrownBy(() -> insertCorrelation(connection, " ", "provider-message", gatewayMessageId)) + .isInstanceOf(SQLException.class); + assertThatThrownBy(() -> insertCorrelation(connection, "provider", "\t\n", gatewayMessageId)) + .isInstanceOf(SQLException.class); + } + } + @Test void correlationIsDeletedWithTrackedMessage() throws SQLException { try (Connection connection = connection()) { insertTrackedMessage(connection, CORRELATION_GATEWAY_ID); - insertCorrelation(connection, "operator-1", CORRELATION_GATEWAY_ID); - insertCorrelation(connection, "operator-2", CORRELATION_GATEWAY_ID); + insertCorrelation(connection, "provider-1", "provider-message-1", CORRELATION_GATEWAY_ID); + insertCorrelation(connection, "provider-1", "provider-message-2", CORRELATION_GATEWAY_ID); try (PreparedStatement statement = connection.prepareStatement(""" DELETE FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? @@ -131,7 +156,7 @@ void typedColumnsStoreCurrentDlrState() throws SQLException { insertCompleteUnpushedDlr(connection); try (PreparedStatement statement = connection.prepareStatement(""" - SELECT gateway_message_id, operator_message_id, reassembled_parts, created_at, updated_at + SELECT gateway_message_id, provider_message_id, reassembled_parts, created_at, updated_at FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? """)) { @@ -140,7 +165,7 @@ void typedColumnsStoreCurrentDlrState() throws SQLException { assertThat(resultSet.next()).isTrue(); assertThat(resultSet.getObject("gateway_message_id", UUID.class)) .isEqualTo(COMPLETE_GATEWAY_ID); - assertThat(resultSet.getString("operator_message_id")).isNull(); + assertThat(resultSet.getString("provider_message_id")).isNull(); assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) .containsExactly("part-1", "part-2"); assertThat(resultSet.getObject("created_at")).isNotNull(); @@ -193,15 +218,31 @@ private static void insertTrackedMessage(Connection connection, UUID gatewayMess } } - private static void insertCorrelation(Connection connection, String operatorMessageId, - UUID gatewayMessageId) throws SQLException { + private static void insertTrackedMessageWithProvider(Connection connection, UUID gatewayMessageId, + String providerName, + String providerMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.tracked_message + (gateway_message_id, provider_name, provider_message_id, status) + VALUES (?, ?, ?, 'ACCEPTED') + """)) { + statement.setObject(1, gatewayMessageId); + statement.setString(2, providerName); + statement.setString(3, providerMessageId); + statement.executeUpdate(); + } + } + + private static void insertCorrelation(Connection connection, String providerName, String providerMessageId, + UUID gatewayMessageId) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.operator_correlation - (operator_message_id, gateway_message_id) - VALUES (?, ?) + INSERT INTO sendium_dlr.provider_correlation + (provider_name, provider_message_id, gateway_message_id) + VALUES (?, ?, ?) """)) { - statement.setString(1, operatorMessageId); - statement.setObject(2, gatewayMessageId); + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setObject(3, gatewayMessageId); statement.executeUpdate(); } } @@ -264,7 +305,7 @@ private static void insertCompleteUnpushedDlr(Connection connection) throws SQLE private static int countCorrelations(Connection connection, UUID gatewayMessageId) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(""" SELECT COUNT(*) - FROM sendium_dlr.operator_correlation + FROM sendium_dlr.provider_correlation WHERE gateway_message_id = ? """)) { statement.setObject(1, gatewayMessageId); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index 3a60f63..4f2b540 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -11,6 +11,7 @@ import gr.cytech.sendium.conf.SendiumConfigurationProvider; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.ForwardMoService; import gr.cytech.sendium.core.worker.Tracker; import gr.cytech.sendium.external.WorkerResourceProvider; @@ -37,6 +38,32 @@ void defaultsSensitiveDiagnosticLoggingOff() { assertThat(config.getBlnPrpt(worker._printMos)).isFalse(); } + @Test + void dlrProviderNameDefaultsToWorkerFullName() { + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + + assertThat(worker.getDlrProviderName()).isEqualTo("test"); + } + + @Test + void dlrProviderNameUsesConfiguredSharedNamespace() { + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(Map.of("msg.hash.prefix", "provider-cluster")), + new Queue<>(), new CapturingTracker()); + + assertThat(worker.getDlrProviderName()).isEqualTo("provider-cluster"); + } + + @Test + void blankDlrProviderNameFallsBackToWorkerFullName() { + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(Map.of("msg.hash.prefix", " ")), + new Queue<>(), new CapturingTracker()); + + assertThat(worker.getDlrProviderName()).isEqualTo("test"); + } + @Test void parseDlrAndCreateResponse_whenReceiptIsValid_enqueuesDlrWithRegisteredTlvs() throws Exception { TestConfigurationProvider config = new TestConfigurationProvider(Map.of( @@ -56,13 +83,33 @@ void parseDlrAndCreateResponse_whenReceiptIsValid_enqueuesDlrWithRegisteredTlvs( PduResponse response = worker.parseDlrAndCreateResponse(deliverSm); assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK); - assertThat(tracker.dlrSmscId).isEqualTo("abc123"); + assertThat(tracker.dlrProviderMessageId).isEqualTo("abc123"); assertThat(tracker.dlrFrom).isEqualTo("smsc"); assertThat(tracker.dlrTo).isEqualTo("recipient"); assertThat(tracker.dlrState).isEqualTo(StandardMessage.DLR_STAT_DELIVRD); assertThat(tracker.dlrTlvs).containsEntry("carrier_1400", "network-a"); } + @Test + void parseDlrAndCreateResponse_whenStorageFails_returnsSystemErrorForProviderRetry() throws Exception { + CapturingTracker tracker = new CapturingTracker(); + tracker.failDlrCreation = true; + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), tracker); + DeliverSm deliverSm = new DeliverSm(); + deliverSm.setSourceAddress(new Address((byte) 1, (byte) 1, "smsc")); + deliverSm.setDestAddress(new Address((byte) 1, (byte) 1, "recipient")); + deliverSm.setDataCoding(SmppConstants.DATA_CODING_DEFAULT); + deliverSm.setShortMessage(CharsetUtil.encode( + "id:abc123 sub:001 dlvrd:001 submit date:2401010000 done date:2401010001 stat:DELIVRD err:000 text:ok", + CharsetUtil.NAME_GSM)); + + PduResponse response = worker.parseDlrAndCreateResponse(deliverSm); + + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SYSERR); + assertThat(tracker.dlrAttempts).isEqualTo(1); + } + @Test void parseDlrAndCreateResponse_whenReceiptHasNoMessageId_returnsSystemError() throws Exception { TestSmppClientWorker worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); @@ -178,11 +225,56 @@ void handleResponse_whenFailStatus_recordsFailureDlr() { worker.handleResponse(handler(worker), SmppConstants.STATUS_INVMSGLEN, "smsc-2", msg); assertThat(tracker.dlrMqId).isEqualTo(17); - assertThat(tracker.dlrSmscId).isEqualTo("smsc-2"); + assertThat(tracker.dlrProviderMessageId).isEqualTo("smsc-2"); assertThat(tracker.dlrState).isEqualTo(StandardMessage.DLR_STAT_FAILED); assertThat(tracker.dlrErrorCode).isEqualTo("7"); } + @Test + void updateSendStatusAndProviderMessageId_whenStorageFails_keepsSubmitResponseCallbackAlive() { + CapturingTracker tracker = new CapturingTracker(); + tracker.failProviderLink = true; + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), tracker); + StandardMessage msg = messageWithNetwork(); + msg.serial = "gateway-17"; + + String providerMessageId = worker.updateSendStatusAndProviderMessageId("smsc-17", msg); + + assertThat(providerMessageId).isEqualTo("smsc-17"); + assertThat(tracker.linkAttempts).isEqualTo(1); + } + + @Test + void updateSendStatusAndProviderMessageId_whenResponseIdIsBlank_usesInternalId() { + CapturingTracker tracker = new CapturingTracker(); + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), tracker); + StandardMessage msg = messageWithNetwork(); + msg.serial = "gateway-17"; + + String providerMessageId = worker.updateSendStatusAndProviderMessageId(" ", msg); + + assertThat(providerMessageId).isEqualTo("smppclient.test_internal_17"); + assertThat(tracker.linkAttempts).isEqualTo(1); + } + + @Test + void failMessage_whenStorageFails_attemptsDlrWithoutEscapingCallback() { + CapturingTracker tracker = new CapturingTracker(); + tracker.failProviderLink = true; + tracker.failDlrCreation = true; + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), tracker); + StandardMessage msg = messageWithNetwork(); + msg.serial = "gateway-17"; + + worker.failMessage(SmppConstants.STATUS_INVMSGLEN, "smsc-17", msg); + + assertThat(tracker.linkAttempts).isEqualTo(1); + assertThat(tracker.dlrAttempts).isEqualTo(1); + } + private static SmppClientSessionHandler handler(TestSmppClientWorker worker) { return new SmppClientSessionHandler(worker, new SmppClientWorker.ConnectionInfo( null, "localhost", 2775, SmppClientWorker.ConnectionType.NORMAL)); @@ -262,12 +354,16 @@ public void forwardMo(String forwardUrl, MoContext ctx, ForwardFormat format) { private static class CapturingTracker implements Tracker { private int dlrMqId; - private String dlrSmscId; + private String dlrProviderMessageId; private String dlrFrom; private String dlrTo; private int dlrState; private String dlrErrorCode; private HashMap dlrTlvs; + private boolean failProviderLink; + private boolean failDlrCreation; + private int linkAttempts; + private int dlrAttempts; @Override public void init() { @@ -283,7 +379,12 @@ public void configure(String key, String newValue, String oldValue) { } @Override - public int updateSendStatusAndExtID(String smsid, StandardMessage pMsg, String smscid) { + public int updateSendStatusAndExtID(String hashedProviderMessageId, StandardMessage message, + String providerMessageId) { + linkAttempts++; + if (failProviderLink) { + throw new DlrStorageException("Failed to link provider DLR ID"); + } return 1; } @@ -298,10 +399,15 @@ public String getVendorPriceGateway() { } @Override - public void createAndEnqueueDLR(int mqid, String smscid, String smsid, String from, String to, String body, - int state, String errorCode, HashMap tlvs) { + public void createAndEnqueueDLR(int mqid, String providerMessageId, String hashedProviderMessageId, + String from, String to, String body, int state, String errorCode, + HashMap tlvs) { + dlrAttempts++; + if (failDlrCreation) { + throw new DlrStorageException("Failed to resolve DLR state"); + } this.dlrMqId = mqid; - this.dlrSmscId = smscid; + this.dlrProviderMessageId = providerMessageId; this.dlrFrom = from; this.dlrTo = to; this.dlrState = state; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java index 47f46c8..7dacace 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java @@ -41,6 +41,7 @@ class InMemorySmppServerMessageStoreTest { @BeforeEach void setUp() { when(worker.getWorkerResources()).thenReturn(workerResources); + when(workerResources.isDlrPersistenceEnabled()).thenReturn(true); when(workerResources.getDlrService()).thenReturn(dlrService); when(worker.getMaxRetries()).thenReturn(5); @@ -148,6 +149,39 @@ void persistMessages_IsolatesInternalEventsWithoutReorderingCallbacks() { order.verify(worker).handlePersistedMessages(List.of(secondClient)); } + @Test + void persistMessages_WhenPersistenceDisabled_AcknowledgesWithoutStoring() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + List> events = List.of(event("gw-1", new SubmitSm())); + + assertTrue(messageStore.persistMessages(events).resultNow()); + + verify(workerResources, never()).getDlrService(); + verify(worker).handlePersistedMessages(events); + verify(worker, never()).handleMessagePersistenceFailure(anyList()); + } + + @Test + void markAsUnpushed_WhenPersistenceDisabled_LeavesRetryToWorker() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + StandardMessage msg = new StandardMessage(); + msg.type = StandardMessage.MSG_DLR; + + assertFalse(messageStore.markAsUnpushed(msg)); + + verify(workerResources, never()).getDlrService(); + } + + @Test + void onClientConnected_WhenPersistenceDisabled_DoesNotReplay() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + + messageStore.onClientConnected("sys1"); + + verify(workerResources, never()).getDlrService(); + verify(worker, never()).enqueueNoExceptions(any()); + } + @Test void getMaxAttempts_DelegatesToWorker() { int result = messageStore.getMaxAttempts(true); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java index d93ecda..451b051 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java @@ -27,10 +27,10 @@ class DlrServiceTest { @Test void resolveAndRemoveDlrMapsDeliveredStateAndForwardsCallback() { MessageState state = stateWithCallback(); - when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.DELIVERED)) + when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.DELIVERED)) .thenReturn(Optional.of(state)); - Optional result = service.resolveAndRemoveDlr("operator-1", 1); + Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 1); assertThat(result).containsSame(state); verify(forwardDlrService).forwardDlr(state); @@ -39,10 +39,10 @@ void resolveAndRemoveDlrMapsDeliveredStateAndForwardsCallback() { @Test void resolveAndRemoveDlrMapsAcceptedState() { MessageState state = stateWithoutCallback(); - when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.ACCEPTED)) + when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.ACCEPTED)) .thenReturn(Optional.of(state)); - Optional result = service.resolveAndRemoveDlr("operator-1", 9); + Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 9); assertThat(result).containsSame(state); verify(forwardDlrService, never()).forwardDlr(state); @@ -51,20 +51,20 @@ void resolveAndRemoveDlrMapsAcceptedState() { @Test void resolveAndRemoveDlrMapsUnknownStateToFailed() { MessageState state = stateWithoutCallback(); - when(storage.resolveAndRemoveDlr("operator-1", MessageState.MessageStatus.FAILED)) + when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.FAILED)) .thenReturn(Optional.of(state)); - Optional result = service.resolveAndRemoveDlr("operator-1", 0); + Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 0); assertThat(result).containsSame(state); } @Test void resolveAndRemoveDlrDoesNotForwardMissingState() { - when(storage.resolveAndRemoveDlr("unknown", MessageState.MessageStatus.DELIVERED)) + when(storage.resolveAndRemoveDlr("provider-1", "unknown", MessageState.MessageStatus.DELIVERED)) .thenReturn(Optional.empty()); - Optional result = service.resolveAndRemoveDlr("unknown", 15); + Optional result = service.resolveAndRemoveDlr("provider-1", "unknown", 15); assertThat(result).isEmpty(); verify(forwardDlrService, never()).forwardDlr(org.mockito.ArgumentMatchers.any()); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java index 47a9f48..8a6fe6a 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java @@ -21,8 +21,6 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) - -@ExtendWith(MockitoExtension.class) class InMemoryMessageTrackerTest { @Mock @@ -39,8 +37,10 @@ class InMemoryMessageTrackerTest { @BeforeEach void setUp() { when(outWorker.getWorkerResources()).thenReturn(workerResources); + when(workerResources.isDlrPersistenceEnabled()).thenReturn(true); when(workerResources.getDlrService()).thenReturn(dlrService); when(outWorker.getType()).thenReturn("testWorker"); + when(outWorker.getDlrProviderName()).thenReturn("provider-1"); tracker = new InMemoryMessageTracker(outWorker); } @@ -50,32 +50,77 @@ void updateSendStatusAndExtID_WithValidIds_Returns1() { StandardMessage pMsg = new StandardMessage(); pMsg.serial = "gw-123"; - int result = tracker.updateSendStatusAndExtID("gw-123", pMsg, "op-456"); + int result = tracker.updateSendStatusAndExtID("gw-123", pMsg, "provider-message-456"); assertEquals(1, result); - verify(dlrService).linkOperatorId("gw-123", "op-456"); + verify(dlrService).linkProviderMessageId("gw-123", "provider-1", "provider-message-456"); } @Test - void updateSendStatusAndExtID_WithNullSmsid_Returns0() { + void updateSendStatusAndExtID_WithNullGatewayMessageId_Returns0() { StandardMessage pMsg = new StandardMessage(); pMsg.serial = null; - int result = tracker.updateSendStatusAndExtID(null, pMsg, "op-456"); + int result = tracker.updateSendStatusAndExtID(null, pMsg, "provider-message-456"); assertEquals(0, result); - verify(dlrService, never()).linkOperatorId(any(), any()); + verify(dlrService, never()).linkProviderMessageId(any(), any(), any()); } @Test - void updateSendStatusAndExtID_WithNullSmscid_Returns0() { + void updateSendStatusAndExtID_WithNullProviderMessageId_Returns0() { StandardMessage pMsg = new StandardMessage(); pMsg.serial = "gw-123"; int result = tracker.updateSendStatusAndExtID("gw-123", pMsg, null); assertEquals(0, result); - verify(dlrService, never()).linkOperatorId(any(), any()); + verify(dlrService, never()).linkProviderMessageId(any(), any(), any()); + } + + @Test + void updateSendStatusAndExtID_WhenStorageFails_PropagatesToProtocolBoundary() { + StandardMessage pMsg = new StandardMessage(); + pMsg.serial = "gw-123"; + doThrow(new DlrStorageException("Failed to link provider DLR ID")) + .when(dlrService).linkProviderMessageId("gw-123", "provider-1", "provider-message-456"); + + assertThrows(DlrStorageException.class, + () -> tracker.updateSendStatusAndExtID("gw-123", pMsg, "provider-message-456")); + } + + @Test + void updateSendStatusAndExtID_WhenPersistenceDisabled_SkipsLinking() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + StandardMessage pMsg = new StandardMessage(); + pMsg.serial = "gw-123"; + + int result = tracker.updateSendStatusAndExtID("gw-123", pMsg, "provider-message-456"); + + assertEquals(0, result); + verify(workerResources, never()).getDlrService(); + } + + @Test + void createAndEnqueueDLR_WhenStorageFails_PropagatesToProtocolBoundary() throws InterruptedException { + when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 0)) + .thenThrow(new DlrStorageException("Failed to resolve DLR state")); + + assertThrows(DlrStorageException.class, () -> tracker.createAndEnqueueDLR( + 1, "provider-message-456", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>())); + + verify(outWorker, never()).enqueueToRouter(any()); + } + + @Test + void createAndEnqueueDLR_WhenPersistenceDisabled_DoesNotResolve() throws InterruptedException { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + + tracker.createAndEnqueueDLR( + 1, "provider-message-456", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); + + verify(workerResources, never()).getDlrService(); + verify(outWorker, never()).enqueueToRouter(any()); } @Test @@ -97,11 +142,13 @@ void getHashedMessageID_NullInput_ReturnsEmpty() { @Test void createAndEnqueueDLR_KnownMessage_ResolvesFromDlrService() throws InterruptedException { MessageState state = new MessageState("gw-123", "accountId", "systemId", "from", "to", null); - when(dlrService.resolveAndRemoveDlr("op-456", 0)).thenReturn(java.util.Optional.of(state)); + when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 0)) + .thenReturn(java.util.Optional.of(state)); - tracker.createAndEnqueueDLR(1, "op-456", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); + tracker.createAndEnqueueDLR( + 1, "provider-message-456", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); - verify(dlrService).resolveAndRemoveDlr("op-456", 0); + verify(dlrService).resolveAndRemoveDlr("provider-1", "provider-message-456", 0); ArgumentCaptor captor = ArgumentCaptor.forClass(StandardMessage.class); verify(outWorker).enqueueToRouter(captor.capture()); assertEquals("accountId", captor.getValue().owner_id); @@ -112,9 +159,11 @@ void createAndEnqueueDLR_KnownMessage_ResolvesFromDlrService() throws Interrupte void createAndEnqueueDLR_KnownReassembledMessage_RestoresPartIds() throws InterruptedException { MessageState state = new MessageState("gw-123", "accountId", "systemId", "from", "to", null); state.setReassembledParts(new ArrayList<>(List.of("part-1", "part-2"))); - when(dlrService.resolveAndRemoveDlr("op-456", 1)).thenReturn(java.util.Optional.of(state)); + when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 1)) + .thenReturn(java.util.Optional.of(state)); - tracker.createAndEnqueueDLR(1, "op-456", "gw-123", "from", "to", "test body", 1, "0", new HashMap<>()); + tracker.createAndEnqueueDLR( + 1, "provider-message-456", "gw-123", "from", "to", "test body", 1, "0", new HashMap<>()); ArgumentCaptor captor = ArgumentCaptor.forClass(StandardMessage.class); verify(outWorker).enqueueToRouter(captor.capture()); @@ -123,11 +172,12 @@ void createAndEnqueueDLR_KnownReassembledMessage_RestoresPartIds() throws Interr @Test void createAndEnqueueDLR_UnknownMessage_DoesNotEnqueue() { - when(dlrService.resolveAndRemoveDlr("unknown", 0)).thenReturn(java.util.Optional.empty()); + when(dlrService.resolveAndRemoveDlr("provider-1", "unknown", 0)) + .thenReturn(java.util.Optional.empty()); tracker.createAndEnqueueDLR(1, "unknown", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); - verify(dlrService).resolveAndRemoveDlr("unknown", 0); + verify(dlrService).resolveAndRemoveDlr("provider-1", "unknown", 0); } @Test diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MessageStateTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MessageStateTest.java index 303c78e..864ab3d 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MessageStateTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/MessageStateTest.java @@ -43,19 +43,28 @@ void constructor_SetsTimestampToCurrentTime() { } @Test - void getOperatorMsgId_InitiallyNull() { + void getProviderMessageId_InitiallyNull() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - assertNull(state.getOperatorMsgId()); + assertNull(state.getProviderMessageId()); } @Test - void setOperatorMsgId_UpdatesValue() { + void setProviderMessageId_UpdatesValue() { MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); - state.setOperatorMsgId("op-456"); + state.setProviderMessageId("provider-message-456"); - assertEquals("op-456", state.getOperatorMsgId()); + assertEquals("provider-message-456", state.getProviderMessageId()); + } + + @Test + void setProviderName_UpdatesValue() { + MessageState state = new MessageState("gw-123", "systemId", "from", "to", null); + + state.setProviderName("provider-1"); + + assertEquals("provider-1", state.getProviderName()); } @Test diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java index 23f8ffd..c0e8e83 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java @@ -29,6 +29,7 @@ class PostgresqlDlrStorageIT { private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; + private static final String PROVIDER = "provider-1"; private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") .withDatabaseName("sendium") .withUsername("sendium") @@ -71,7 +72,8 @@ void resetStorage() throws SQLException { @Test void saveInitialStateRoundTripsAllFields() { MessageState state = newState(); - state.setOperatorMsgId("operator-initial"); + state.setProviderName(PROVIDER); + state.setProviderMessageId("provider-message-initial"); state.setReassembledParts(List.of("part-1", "part-2")); storage.saveInitialState(state); @@ -80,7 +82,8 @@ void saveInitialStateRoundTripsAllFields() { .get() .usingRecursiveComparison() .isEqualTo(state); - assertThat(storage.resolveAndRemoveDlr("operator-initial", MessageState.MessageStatus.DELIVERED)) + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "provider-message-initial", MessageState.MessageStatus.DELIVERED)) .isPresent(); } @@ -99,36 +102,63 @@ void saveInitialStatesCommitsWholeBatch() { } @Test - void saveInitialStatesRollsBackWholeBatchOnCorrelationConflict() { + void saveInitialStatesRebindsCorrelationToNewestBatchMessage() throws SQLException { MessageState owner = newState(); - owner.setOperatorMsgId("shared-operator"); + owner.setProviderName(PROVIDER); + owner.setProviderMessageId("shared-provider-message"); storage.saveInitialState(owner); MessageState innocent = newState(); MessageState conflict = newState(); - conflict.setOperatorMsgId("shared-operator"); + conflict.setProviderName(PROVIDER); + conflict.setProviderMessageId("shared-provider-message"); + + storage.saveInitialStates(List.of(innocent, conflict)); + + assertThat(storage.getState(innocent.getGatewayMsgId())).isPresent(); + assertThat(storage.getState(conflict.getGatewayMsgId())).isPresent(); + assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getProviderMessageId()).isNull(); + assertThat(countCorrelations(owner.getGatewayMsgId())).isZero(); + assertThat(countCorrelations(conflict.getGatewayMsgId())).isOne(); + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId) + .isEqualTo(conflict.getGatewayMsgId()); + } + + @Test + void saveInitialStatesUsesFinalStateForDuplicateGatewayMessage() throws SQLException { + MessageState correlated = newState(); + correlated.setProviderName(PROVIDER); + correlated.setProviderMessageId("superseded-provider-message"); + MessageState replacement = new MessageState(correlated.getGatewayMsgId(), "replacement-account", + "replacement-system", "replacement-source", "replacement-destination", null); - assertThatThrownBy(() -> storage.saveInitialStates(List.of(innocent, conflict))) - .isInstanceOf(DlrStorageException.class); + storage.saveInitialStates(List.of(correlated, replacement)); - assertThat(storage.getState(innocent.getGatewayMsgId())).isEmpty(); - assertThat(storage.getState(conflict.getGatewayMsgId())).isEmpty(); - assertThat(storage.getState(owner.getGatewayMsgId())).isPresent(); + assertThat(storage.getState(replacement.getGatewayMsgId())) + .get() + .usingRecursiveComparison() + .isEqualTo(replacement); + assertThat(countCorrelations(replacement.getGatewayMsgId())).isZero(); + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "superseded-provider-message", MessageState.MessageStatus.DELIVERED)).isEmpty(); } @Test void trackedStateAndCorrelationSurviveAdapterRecreation() { MessageState state = newState(); storage.saveInitialState(state); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-after-restart"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-after-restart"); PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); assertThat(recreated.getState(state.getGatewayMsgId())) .get() - .extracting(MessageState::getOperatorMsgId, MessageState::getStatus) - .containsExactly("operator-after-restart", MessageState.MessageStatus.SENT); + .extracting(MessageState::getProviderMessageId, MessageState::getStatus) + .containsExactly("provider-message-after-restart", MessageState.MessageStatus.SENT); assertThat(recreated.resolveAndRemoveDlr( - "operator-after-restart", MessageState.MessageStatus.DELIVERED)) + PROVIDER, "provider-message-after-restart", MessageState.MessageStatus.DELIVERED)) .get() .extracting(MessageState::getGatewayMsgId, MessageState::getStatus) .containsExactly(state.getGatewayMsgId(), MessageState.MessageStatus.DELIVERED); @@ -138,7 +168,7 @@ void trackedStateAndCorrelationSurviveAdapterRecreation() { void saveInitialStateOverwritesExistingState() throws SQLException { MessageState initial = newState(); storage.saveInitialState(initial); - storage.linkOperatorId(initial.getGatewayMsgId(), "operator-old"); + storage.linkProviderMessageId(initial.getGatewayMsgId(), PROVIDER, "provider-message-old"); MessageState replacement = new MessageState(initial.getGatewayMsgId(), "replacement-account", "replacement-system", "replacement-source", "replacement-destination", null); @@ -150,96 +180,247 @@ void saveInitialStateOverwritesExistingState() throws SQLException { .usingRecursiveComparison() .isEqualTo(replacement); assertThat(countCorrelations(initial.getGatewayMsgId())).isZero(); - assertThat(storage.resolveAndRemoveDlr("operator-old", MessageState.MessageStatus.DELIVERED)) + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "provider-message-old", MessageState.MessageStatus.DELIVERED)) .isEmpty(); } @Test - void saveInitialStateRollsBackOnCorrelationOwnedByAnotherMessage() throws SQLException { + void saveInitialStateRebindsCorrelationOwnedByAnotherMessage() throws SQLException { MessageState owner = newState(); storage.saveInitialState(owner); - storage.linkOperatorId(owner.getGatewayMsgId(), "shared-operator"); + storage.linkProviderMessageId(owner.getGatewayMsgId(), PROVIDER, "shared-provider-message"); MessageState target = newState(); storage.saveInitialState(target); - storage.linkOperatorId(target.getGatewayMsgId(), "target-operator"); - MessageState targetBeforeReplacement = storage.getState(target.getGatewayMsgId()).orElseThrow(); - + storage.linkProviderMessageId(target.getGatewayMsgId(), PROVIDER, "target-provider-message"); MessageState replacement = new MessageState(target.getGatewayMsgId(), "replacement-account", "replacement-system", "replacement-source", "replacement-destination", null); - replacement.setOperatorMsgId("shared-operator"); - assertThatThrownBy(() -> storage.saveInitialState(replacement)) - .isInstanceOf(DlrStorageException.class); + replacement.setProviderName(PROVIDER); + replacement.setProviderMessageId("shared-provider-message"); + storage.saveInitialState(replacement); assertThat(storage.getState(target.getGatewayMsgId())) .get() .usingRecursiveComparison() - .isEqualTo(targetBeforeReplacement); - assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getOperatorMsgId()) - .isEqualTo("shared-operator"); + .isEqualTo(replacement); + assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getProviderMessageId()) + .isNull(); assertThat(countCorrelations(target.getGatewayMsgId())).isOne(); - assertThat(countCorrelations(owner.getGatewayMsgId())).isOne(); + assertThat(countCorrelations(owner.getGatewayMsgId())).isZero(); MessageState newConflict = newState(); - newConflict.setOperatorMsgId("shared-operator"); - assertThatThrownBy(() -> storage.saveInitialState(newConflict)) - .isInstanceOf(DlrStorageException.class); - assertThat(storage.getState(newConflict.getGatewayMsgId())).isEmpty(); + newConflict.setProviderName(PROVIDER); + newConflict.setProviderMessageId("shared-provider-message"); + storage.saveInitialState(newConflict); + + assertThat(storage.getState(target.getGatewayMsgId()).orElseThrow().getProviderMessageId()).isNull(); + assertThat(countCorrelations(target.getGatewayMsgId())).isZero(); + assertThat(countCorrelations(newConflict.getGatewayMsgId())).isOne(); } @Test - void linkOperatorIdUpdatesStateAndKeepsMultipleCorrelations() throws SQLException { + void linkProviderMessageIdUpdatesStateAndKeepsMultipleCorrelations() throws SQLException { MessageState state = newState(); storage.saveInitialState(state); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); MessageState linked = storage.getState(state.getGatewayMsgId()).orElseThrow(); assertThat(linked.getStatus()).isEqualTo(MessageState.MessageStatus.SENT); - assertThat(linked.getOperatorMsgId()).isEqualTo("operator-2"); + assertThat(linked.getProviderMessageId()).isEqualTo("provider-message-2"); assertThat(countCorrelations(state.getGatewayMsgId())).isEqualTo(2); } @Test - void linkOperatorIdRollsBackStateWhenCorrelationInsertFails() { + void linkProviderMessageIdRejectsInvalidCorrelation() { MessageState state = newState(); storage.saveInitialState(state); - assertThatThrownBy(() -> storage.linkOperatorId(state.getGatewayMsgId(), null)) - .isInstanceOf(DlrStorageException.class); + assertThatThrownBy(() -> storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, null)) + .isInstanceOf(IllegalArgumentException.class); MessageState unchanged = storage.getState(state.getGatewayMsgId()).orElseThrow(); assertThat(unchanged.getStatus()).isEqualTo(MessageState.MessageStatus.ACCEPTED); - assertThat(unchanged.getOperatorMsgId()).isNull(); + assertThat(unchanged.getProviderMessageId()).isNull(); + } + + @Test + void linkProviderMessageIdRebindsCorrelationToNewestMessage() throws SQLException { + MessageState first = new MessageState(UUID.randomUUID().toString(), "first-account", "first-system", + "first-source", "first-destination", null); + MessageState second = new MessageState(UUID.randomUUID().toString(), "second-account", "second-system", + "second-source", "second-destination", null); + storage.saveInitialState(first); + storage.saveInitialState(second); + storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "shared-provider-message"); + + storage.linkProviderMessageId(second.getGatewayMsgId(), PROVIDER, "shared-provider-message"); + + assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getProviderMessageId()) + .isNull(); + MessageState linked = storage.getState(second.getGatewayMsgId()).orElseThrow(); + assertThat(linked.getStatus()).isEqualTo(MessageState.MessageStatus.SENT); + assertThat(linked.getProviderMessageId()).isEqualTo("shared-provider-message"); + assertThat(countCorrelations(first.getGatewayMsgId())).isZero(); + assertThat(countCorrelations(second.getGatewayMsgId())).isOne(); + + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId, MessageState::getAccountId) + .containsExactly(second.getGatewayMsgId(), "second-account"); + assertThat(storage.getState(first.getGatewayMsgId())).isPresent(); } @Test - void linkOperatorIdRejectsCorrelationOwnedByAnotherMessage() throws SQLException { + void sameMessageIdFromDifferentProvidersResolvesIndependently() { + MessageState first = newState(); + MessageState second = newState(); + storage.saveInitialStates(List.of(first, second)); + + storage.linkProviderMessageId(first.getGatewayMsgId(), "provider-a", "shared-provider-message"); + storage.linkProviderMessageId(second.getGatewayMsgId(), "provider-b", "shared-provider-message"); + + assertThat(storage.resolveAndRemoveDlr( + "provider-a", "shared-provider-message", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId, MessageState::getProviderName, + MessageState::getProviderMessageId) + .containsExactly(first.getGatewayMsgId(), "provider-a", "shared-provider-message"); + assertThat(storage.resolveAndRemoveDlr( + "provider-b", "shared-provider-message", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId, MessageState::getProviderName, + MessageState::getProviderMessageId) + .containsExactly(second.getGatewayMsgId(), "provider-b", "shared-provider-message"); + } + + @Test + void concurrentProviderMessageIdRebindLeavesOneConsistentOwner() throws Exception { MessageState first = newState(); MessageState second = newState(); storage.saveInitialState(first); storage.saveInitialState(second); - storage.linkOperatorId(first.getGatewayMsgId(), "shared-operator"); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); - assertThatThrownBy(() -> storage.linkOperatorId(second.getGatewayMsgId(), "shared-operator")) - .isInstanceOf(DlrStorageException.class); + try { + Future firstLink = executor.submit(() -> { + linkWhenReleased(first.getGatewayMsgId(), "shared-provider-message", ready, start); + return null; + }); + Future secondLink = executor.submit(() -> { + linkWhenReleased(second.getGatewayMsgId(), "shared-provider-message", ready, start); + return null; + }); + ready.await(); + start.countDown(); + firstLink.get(); + secondLink.get(); + + MessageState firstAfter = storage.getState(first.getGatewayMsgId()).orElseThrow(); + MessageState secondAfter = storage.getState(second.getGatewayMsgId()).orElseThrow(); + assertThat(List.of(firstAfter, secondAfter).stream() + .filter(state -> "shared-provider-message".equals(state.getProviderMessageId()))) + .hasSize(1); + assertThat(countCorrelations(first.getGatewayMsgId()) + countCorrelations(second.getGatewayMsgId())) + .isOne(); + String owner = storage.resolveAndRemoveDlr( + PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED) + .orElseThrow() + .getGatewayMsgId(); + assertThat(owner).isIn(first.getGatewayMsgId(), second.getGatewayMsgId()); + } finally { + executor.shutdownNow(); + } + } - assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getOperatorMsgId()) - .isEqualTo("shared-operator"); - MessageState unchanged = storage.getState(second.getGatewayMsgId()).orElseThrow(); - assertThat(unchanged.getStatus()).isEqualTo(MessageState.MessageStatus.ACCEPTED); - assertThat(unchanged.getOperatorMsgId()).isNull(); - assertThat(countCorrelations(first.getGatewayMsgId())).isOne(); - assertThat(countCorrelations(second.getGatewayMsgId())).isZero(); + @Test + void concurrentRebindAndResolveCompleteWithoutDeadlock() throws Exception { + MessageState first = newState(); + MessageState second = newState(); + storage.saveInitialStates(List.of(first, second)); + storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "shared-provider-message"); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + + try { + Future rebind = executor.submit(() -> { + linkWhenReleased(second.getGatewayMsgId(), "shared-provider-message", ready, start); + return null; + }); + Future> resolve = executor.submit( + () -> resolveWhenReleased("shared-provider-message", ready, start)); + ready.await(); + start.countDown(); + + rebind.get(); + assertThat(resolve.get()).isPresent(); + assertThat(countCorrelations(first.getGatewayMsgId()) + + countCorrelations(second.getGatewayMsgId())).isLessThanOrEqualTo(1); + } finally { + executor.shutdownNow(); + } } @Test - void linkOperatorIdFailsWhenGatewayStateDoesNotAppear() { + void concurrentCrossedRebindsLockMessagesInConsistentOrder() throws Exception { + MessageState first = newState(); + MessageState second = newState(); + storage.saveInitialState(first); + storage.saveInitialState(second); + storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "provider-message-2"); + storage.linkProviderMessageId(second.getGatewayMsgId(), PROVIDER, "provider-message-1"); + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + + try { + Future firstLink = executor.submit(() -> { + linkWhenReleased(first.getGatewayMsgId(), "provider-message-1", ready, start); + return null; + }); + Future secondLink = executor.submit(() -> { + linkWhenReleased(second.getGatewayMsgId(), "provider-message-2", ready, start); + return null; + }); + ready.await(); + start.countDown(); + firstLink.get(); + secondLink.get(); + + assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getProviderMessageId()) + .isEqualTo("provider-message-1"); + assertThat(storage.getState(second.getGatewayMsgId()).orElseThrow().getProviderMessageId()) + .isEqualTo("provider-message-2"); + assertThat(countCorrelations(first.getGatewayMsgId())).isOne(); + assertThat(countCorrelations(second.getGatewayMsgId())).isOne(); + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "provider-message-1", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId) + .isEqualTo(first.getGatewayMsgId()); + assertThat(storage.resolveAndRemoveDlr( + PROVIDER, "provider-message-2", MessageState.MessageStatus.DELIVERED)) + .get() + .extracting(MessageState::getGatewayMsgId) + .isEqualTo(second.getGatewayMsgId()); + } finally { + executor.shutdownNow(); + } + } + + @Test + void linkProviderMessageIdFailsWhenGatewayStateDoesNotAppear() { PostgresqlDlrStorage noRetryStorage = new PostgresqlDlrStorage(dataSource, 1, 0); - assertThatThrownBy(() -> noRetryStorage.linkOperatorId(UUID.randomUUID().toString(), "operator")) + assertThatThrownBy(() -> noRetryStorage.linkProviderMessageId( + UUID.randomUUID().toString(), PROVIDER, "provider-message")) .isInstanceOf(DlrStorageException.class) .hasMessageContaining("not found"); } @@ -248,16 +429,16 @@ void linkOperatorIdFailsWhenGatewayStateDoesNotAppear() { void resolveAndRemoveDlrReturnsUpdatedStateAndDeletesAllCorrelations() throws SQLException { MessageState state = newState(); storage.saveInitialState(state); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); long beforeResolve = System.currentTimeMillis(); Optional resolved = storage.resolveAndRemoveDlr( - "operator-1", MessageState.MessageStatus.DELIVERED); + PROVIDER, "provider-message-1", MessageState.MessageStatus.DELIVERED); assertThat(resolved).isPresent(); assertThat(resolved.orElseThrow().getStatus()).isEqualTo(MessageState.MessageStatus.DELIVERED); - assertThat(resolved.orElseThrow().getOperatorMsgId()).isEqualTo("operator-1"); + assertThat(resolved.orElseThrow().getProviderMessageId()).isEqualTo("provider-message-1"); assertThat(resolved.orElseThrow().getTimestamp()).isGreaterThanOrEqualTo(beforeResolve); assertThat(storage.getState(state.getGatewayMsgId())).isEmpty(); assertThat(countCorrelations(state.getGatewayMsgId())).isZero(); @@ -267,17 +448,17 @@ void resolveAndRemoveDlrReturnsUpdatedStateAndDeletesAllCorrelations() throws SQ void concurrentResolveAcrossCorrelationsReturnsStateOnlyOnce() throws Exception { MessageState state = newState(); storage.saveInitialState(state); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-1"); - storage.linkOperatorId(state.getGatewayMsgId(), "operator-2"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); CountDownLatch ready = new CountDownLatch(2); CountDownLatch start = new CountDownLatch(1); ExecutorService executor = Executors.newFixedThreadPool(2); try { Future> first = executor.submit( - () -> resolveWhenReleased("operator-1", ready, start)); + () -> resolveWhenReleased("provider-message-1", ready, start)); Future> second = executor.submit( - () -> resolveWhenReleased("operator-2", ready, start)); + () -> resolveWhenReleased("provider-message-2", ready, start)); ready.await(); start.countDown(); @@ -305,7 +486,7 @@ void markAsFailedUpdatesExistingState() { void expiryRemovesOldCorrelationsAndMessages() throws SQLException { MessageState correlationState = newState(); storage.saveInitialState(correlationState); - storage.linkOperatorId(correlationState.getGatewayMsgId(), "old-correlation"); + storage.linkProviderMessageId(correlationState.getGatewayMsgId(), PROVIDER, "old-correlation"); ageCorrelation("old-correlation"); PostgresqlDlrStorage correlationCleanup = new PostgresqlDlrStorage(dataSource); @@ -420,6 +601,29 @@ void claimHidesReceiptUntilReleasedOrRemoved() { assertThat(storage.getUnpushedDlrs(dlr.systemId)).isEmpty(); } + @Test + void staleClaimCannotRemoveOrReleaseReplacementGeneration() { + StandardMessage original = newDlr("account-1", "system-1"); + storage.saveUnpushedDlr(original); + StandardMessage staleClaim = storage.claimUnpushedDlrs(original.systemId).getFirst(); + StandardMessage replacement = newDlr("account-2", original.systemId); + replacement.serial = original.serial; + replacement.msgId = original.msgId; + replacement.state = original.state; + replacement.errcode = original.errcode; + storage.saveUnpushedDlr(replacement); + + assertThat(storage.removeUnpushedDlr(staleClaim)).isFalse(); + List replacementClaim = storage.claimUnpushedDlrs(original.systemId); + assertThat(replacementClaim).hasSize(1); + assertThat(replacementClaim.getFirst().owner_id).isEqualTo("account-2"); + + storage.releaseUnpushedDlrClaim(staleClaim); + assertThat(storage.claimUnpushedDlrs(original.systemId)).isEmpty(); + storage.releaseUnpushedDlrClaim(replacementClaim.getFirst()); + assertThat(storage.claimUnpushedDlrs(original.systemId)).hasSize(1); + } + @Test void concurrentClaimsReturnReceiptOnlyOnce() throws Exception { StandardMessage dlr = newDlr("account-1", "system-1"); @@ -459,11 +663,18 @@ void expiryRemovesOldUnpushedDlrsAndTheirClaims() throws SQLException { assertThat(storage.claimUnpushedDlrs(dlr.systemId)).isEmpty(); } - private Optional resolveWhenReleased(String operatorMsgId, CountDownLatch ready, - CountDownLatch start) throws InterruptedException { + private Optional resolveWhenReleased(String providerMessageId, CountDownLatch ready, + CountDownLatch start) throws InterruptedException { ready.countDown(); start.await(); - return storage.resolveAndRemoveDlr(operatorMsgId, MessageState.MessageStatus.DELIVERED); + return storage.resolveAndRemoveDlr(PROVIDER, providerMessageId, MessageState.MessageStatus.DELIVERED); + } + + private void linkWhenReleased(String gatewayMsgId, String providerMessageId, CountDownLatch ready, + CountDownLatch start) throws InterruptedException { + ready.countDown(); + start.await(); + storage.linkProviderMessageId(gatewayMsgId, PROVIDER, providerMessageId); } private List claimWhenReleased(String systemId, CountDownLatch ready, @@ -499,7 +710,7 @@ private int countCorrelations(String gatewayMsgId) throws SQLException { try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(""" SELECT COUNT(*) - FROM sendium_dlr.operator_correlation + FROM sendium_dlr.provider_correlation WHERE gateway_message_id = ? """)) { statement.setObject(1, UUID.fromString(gatewayMsgId)); @@ -519,14 +730,15 @@ private int countUnpushedDlrs() throws SQLException { } } - private void ageCorrelation(String operatorMsgId) throws SQLException { + private void ageCorrelation(String providerMessageId) throws SQLException { try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(""" - UPDATE sendium_dlr.operator_correlation - SET created_at = CURRENT_TIMESTAMP - INTERVAL '4 days' - WHERE operator_message_id = ? - """)) { - statement.setString(1, operatorMsgId); + UPDATE sendium_dlr.provider_correlation + SET created_at = CURRENT_TIMESTAMP - INTERVAL '4 days' + WHERE provider_name = ? AND provider_message_id = ? + """)) { + statement.setString(1, PROVIDER); + statement.setString(2, providerMessageId); statement.executeUpdate(); } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageRetentionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageRetentionTest.java new file mode 100644 index 0000000..2f3c35c --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageRetentionTest.java @@ -0,0 +1,74 @@ +package gr.cytech.sendium.core.worker; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Retention cleanup piggybacks on storage activity, so a failing pass must stay isolated from the operation that + * triggered it: it may not reject a submission, and it may not be retried by every following call. + */ +class PostgresqlDlrStorageRetentionTest { + private static final long ONE_MINUTE_MILLIS = 60_000L; + + private final AtomicInteger retentionAttempts = new AtomicInteger(); + + private DataSource dataSource; + + @BeforeEach + void setUp() throws SQLException { + Connection connection = mock(Connection.class, RETURNS_DEEP_STUBS); + dataSource = mock(DataSource.class); + when(dataSource.getConnection()).thenReturn(connection); + doNothing().when(connection).setAutoCommit(false); + + ResultSet noRows = mock(ResultSet.class); + when(noRows.next()).thenReturn(false); + PreparedStatement read = mock(PreparedStatement.class); + when(read.executeQuery()).thenReturn(noRows); + doNothing().when(read).setObject(anyInt(), org.mockito.ArgumentMatchers.any()); + + when(connection.prepareStatement(anyString())).thenAnswer(invocation -> { + // Only the retention statements filter on an age threshold. + if (invocation.getArgument(0).contains("created_at <")) { + retentionAttempts.incrementAndGet(); + throw new SQLException("permission denied for table", "42501"); + } + return read; + }); + } + + @Test + void failedRetentionDoesNotFailTheTriggeringOperation() { + PostgresqlDlrStorage storage = new PostgresqlDlrStorage(dataSource, 1, 0, ONE_MINUTE_MILLIS); + + assertThat(storage.getState(UUID.randomUUID().toString())).isEmpty(); + assertThat(retentionAttempts).hasValue(1); + } + + @Test + void failedRetentionIsNotRetriedUntilTheNextInterval() { + PostgresqlDlrStorage storage = new PostgresqlDlrStorage(dataSource, 1, 0, ONE_MINUTE_MILLIS); + + for (int call = 0; call < 5; call++) { + assertThat(storage.getState(UUID.randomUUID().toString())).isEmpty(); + } + + assertThat(retentionAttempts).hasValue(1); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/util/MessageTraceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/util/MessageTraceTest.java index 1b16bbe..ed94409 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/util/MessageTraceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/util/MessageTraceTest.java @@ -17,7 +17,7 @@ void identifiersIncludeSafeSupportContext() { StandardMessage message = new StandardMessage(); message.serial = "gw-1"; message.msgId = 17; - message.extrid = "operator-1"; + message.extrid = "provider-message-1"; message.type = StandardMessage.MSG_DLR; message.priority = StandardMessage.HIGH_PRIORITY; message.acked = true; @@ -47,7 +47,7 @@ void identifiersIncludeSafeSupportContext() { assertThat(trace).contains( "serial=gw-1", "msgId=17", - "extrid=operator-1", + "extrid=provider-message-1", "type=l", "typeId=18", "priority=3", diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index 6c57c63..363f1a2 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -139,7 +139,7 @@ private static Process verifyHttpCorrelationSurvivesRestart(String containerName require(!gatewayId.isBlank(), "HTTP /sendsms did not return a gateway message id"); require(upstream.awaitSubmitCount(expectedSubmitCount), "Upstream SMPP server did not receive the HTTP-originated message"); - awaitSuccessfulStorageOperation("link_operator"); + awaitSuccessfulStorageOperation("link_provider"); int boundSessionsBeforeRestart = upstream.boundSessionCount(); stopContainer(containerName); @@ -173,7 +173,7 @@ private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Pa gatewayId = response.getMessageId(); require(gatewayId != null && !gatewayId.isBlank(), "SMPP restart submit_sm_resp did not contain a message id"); require(upstream.awaitSubmitCount(1), "Upstream SMPP server did not receive the restart test message"); - awaitSuccessfulStorageOperation("link_operator"); + awaitSuccessfulStorageOperation("link_provider"); } Thread.sleep(500); diff --git a/tests/quick-start-test.sh b/tests/quick-start-test.sh index 79e6eda..bbc9222 100644 --- a/tests/quick-start-test.sh +++ b/tests/quick-start-test.sh @@ -149,6 +149,25 @@ assert_equals 48 "${#http_password}" "regenerated HTTP password length" assert_equals "$old_database_password" "$database_password" "preserved PostgreSQL password" pass "explicit regeneration preserves unrelated files" +unrecoverable_dir="$test_root/unrecoverable-postgresql" +sh "$quick_start" --directory "$unrecoverable_dir" --provider local --no-start > "$test_root/unrecoverable-setup.out" 2>&1 +grep -v '^SENDIUM_DLR_POSTGRESQL_PASSWORD=' "$unrecoverable_dir/.sendium.env" > "$unrecoverable_dir/.sendium.env.stripped" +mv "$unrecoverable_dir/.sendium.env.stripped" "$unrecoverable_dir/.sendium.env" +grep -v '^SENDIUM_LOCAL_POSTGRESQL_PASSWORD=' "$unrecoverable_dir/.sendium.env" > "$unrecoverable_dir/.sendium.env.stripped" +mv "$unrecoverable_dir/.sendium.env.stripped" "$unrecoverable_dir/.sendium.env" +grep -v '^POSTGRES_PASSWORD=' "$unrecoverable_dir/.sendium.env" > "$unrecoverable_dir/.sendium.env.stripped" +mv "$unrecoverable_dir/.sendium.env.stripped" "$unrecoverable_dir/.sendium.env" +expect_failure "unrecoverable PostgreSQL password" "$test_root/unrecoverable-postgresql.out" \ + sh "$quick_start" --directory "$unrecoverable_dir" --provider local --force --no-start +assert_contains 'the existing PostgreSQL volume requires the password' "$test_root/unrecoverable-postgresql.out" +assert_contains 'down --volumes' "$test_root/unrecoverable-postgresql.out" +assert_contains 'rm -f' "$test_root/unrecoverable-postgresql.out" +rm -f "$unrecoverable_dir/compose.yml" +sh "$quick_start" --directory "$unrecoverable_dir" --provider local --force --no-start > "$test_root/recovered-postgresql.out" 2>&1 +recovered_password=$(sed -n "s/^SENDIUM_LOCAL_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$unrecoverable_dir/.sendium.env") +assert_equals 64 "${#recovered_password}" "recovered PostgreSQL password length" +pass "unrecoverable local PostgreSQL password stops regeneration" + external_dir="$test_root/external-postgresql" SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium?sslmode=require' \ SENDIUM_DLR_POSTGRESQL_USERNAME='external-user' \ @@ -161,8 +180,24 @@ assert_not_contains 'image: postgres:17-alpine' "$external_dir/compose.yml" assert_not_contains 'condition: service_healthy' "$external_dir/compose.yml" assert_not_contains 'postgres-data:' "$external_dir/compose.yml" assert_not_contains 'external-password' "$external_dir/compose.yml" +assert_contains "SENDIUM_LOCAL_POSTGRESQL_PASSWORD: ''" "$external_dir/compose.yml" pass "external PostgreSQL configuration" +switched_dir="$test_root/switched-postgresql" +sh "$quick_start" --directory "$switched_dir" --provider local --no-start > "$test_root/switched-local-setup.out" 2>&1 +original_local_password=$(sed -n "s/^SENDIUM_LOCAL_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$switched_dir/.sendium.env") +SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium' \ +SENDIUM_DLR_POSTGRESQL_USERNAME='external-user' \ +SENDIUM_DLR_POSTGRESQL_PASSWORD='external-password' \ + sh "$quick_start" --directory "$switched_dir" --provider local --force --no-start > "$test_root/switched-external.out" 2>&1 +assert_contains "SENDIUM_LOCAL_POSTGRESQL_PASSWORD='$original_local_password'" "$switched_dir/.sendium.env" +sh "$quick_start" --directory "$switched_dir" --provider local --force --no-start > "$test_root/switched-postgresql.out" 2>&1 +switched_password=$(sed -n "s/^SENDIUM_DLR_POSTGRESQL_PASSWORD='\([0-9a-f][0-9a-f]*\)'$/\1/p" "$switched_dir/.sendium.env") +assert_equals 64 "${#switched_password}" "regenerated PostgreSQL password length" +assert_equals "$original_local_password" "$switched_password" "preserved PostgreSQL password after mode switch" +assert_contains 'image: postgres:17-alpine' "$switched_dir/compose.yml" +pass "local PostgreSQL password survives an external database round trip" + expect_failure "partial external PostgreSQL configuration" "$test_root/partial-postgresql.out" \ env SENDIUM_DLR_POSTGRESQL_JDBC_URL='jdbc:postgresql://database.example.test:5432/sendium' \ sh "$quick_start" --directory "$test_root/partial-postgresql" --provider local --no-start From 45896fa1374e82631367e1aa0b49d9a17d191b2a Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 20 Aug 2026 17:06:57 +0300 Subject: [PATCH 16/20] refactor(core): rename standard message adapters --- docs/01-architecture.md | 2 +- ...va => StandardSmppServerMessageStore.java} | 10 +++++----- ...acker.java => StandardMessageTracker.java} | 19 +++++-------------- .../StandardOutgoingWorkerHandler.java | 8 ++++---- ...> StandardSmppServerMessageStoreTest.java} | 8 ++++---- ...t.java => StandardMessageTrackerTest.java} | 10 +++------- 6 files changed, 22 insertions(+), 35 deletions(-) rename sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/{InMemorySmppServerMessageStore.java => StandardSmppServerMessageStore.java} (95%) rename sendium-core/src/main/java/gr/cytech/sendium/core/worker/{InMemoryMessageTracker.java => StandardMessageTracker.java} (88%) rename sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/{InMemorySmppServerMessageStoreTest.java => StandardSmppServerMessageStoreTest.java} (97%) rename sendium-core/src/test/java/gr/cytech/sendium/core/worker/{InMemoryMessageTrackerTest.java => StandardMessageTrackerTest.java} (96%) diff --git a/docs/01-architecture.md b/docs/01-architecture.md index c8f0761..beabff7 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -157,7 +157,7 @@ Outbound HTTP messages can include a Kannel-style `dlr-url`. Sendium stores the sequenceDiagram participant SMSC as Upstream SMSC participant Worker as SmppClientWorker - participant Tracker as InMemoryMessageTracker + participant Tracker as StandardMessageTracker participant Store as DlrStorage participant Router as Router queue participant DLRHook as ForwardDlrService diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java similarity index 95% rename from sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java rename to sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java index 2d7433d..230c1cc 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java @@ -15,24 +15,24 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; -public class InMemorySmppServerMessageStore implements SmppServerMessageStore { - private static final Logger logger = LoggerFactory.getLogger(InMemorySmppServerMessageStore.class); +public class StandardSmppServerMessageStore implements SmppServerMessageStore { + private static final Logger logger = LoggerFactory.getLogger(StandardSmppServerMessageStore.class); private final SmppServerWorker worker; @Inject - public InMemorySmppServerMessageStore(SmppServerWorker worker) { + public StandardSmppServerMessageStore(SmppServerWorker worker) { this.worker = worker; } @Override public void start() { - logger.info("InMemorySmppServerMessageStore started"); + logger.info("StandardSmppServerMessageStore started"); } @Override public void stop() { - logger.info("InMemorySmppServerMessageStore stopped"); + logger.info("StandardSmppServerMessageStore stopped"); } @Override diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java similarity index 88% rename from sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java rename to sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java index e4ddcc2..986720b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/InMemoryMessageTracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java @@ -11,26 +11,24 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Optional; -import java.util.concurrent.ConcurrentLinkedQueue; -public class InMemoryMessageTracker implements Tracker { +public class StandardMessageTracker implements Tracker { - private static final Logger logger = LoggerFactory.getLogger(InMemoryMessageTracker.class); + private static final Logger logger = LoggerFactory.getLogger(StandardMessageTracker.class); AbstractOutWorker outWorker; - private final ConcurrentLinkedQueue dlrQueue = new ConcurrentLinkedQueue<>(); - public InMemoryMessageTracker(AbstractOutWorker worker) { + public StandardMessageTracker(AbstractOutWorker worker) { this.outWorker = worker; } @Override public void init() { - logger.info("InMemoryMessageTracker initialized"); + logger.info("StandardMessageTracker initialized"); } @Override public boolean stop() { - logger.info("InMemoryMessageTracker stopping"); + logger.info("StandardMessageTracker stopping"); return true; } @@ -125,11 +123,4 @@ public int getConfiguredMccMnc() { return 0; } - public StandardMessage pollDlr() { - return dlrQueue.poll(); - } - - public int getDlrQueueSize() { - return dlrQueue.size(); - } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java index 9bd70e3..3648b4b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java @@ -8,9 +8,9 @@ import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; -import gr.cytech.sendium.core.smpp.server.InMemorySmppServerMessageStore; import gr.cytech.sendium.core.smpp.server.SmppServerWorker; -import gr.cytech.sendium.core.worker.InMemoryMessageTracker; +import gr.cytech.sendium.core.smpp.server.StandardSmppServerMessageStore; +import gr.cytech.sendium.core.worker.StandardMessageTracker; import gr.cytech.sendium.core.worker.WorkerType; import gr.cytech.sendium.external.WorkerResourceProvider; import gr.cytech.sendium.external.filter.InMessageFiltering; @@ -203,10 +203,10 @@ protected AbstractOutWorker startWorker(String instName, String } worker = selectedWorker.get(); worker.setupInstance(configurationHandler, instName, queueProvider.getRouterQueue()); - worker.init(workerResourceProvider, new InMemoryMessageTracker(worker)); + worker.init(workerResourceProvider, new StandardMessageTracker(worker)); if (SmppServerWorker.TYPE_SMPP_SERVER.equals(worker.getType())) { var smppServer = (SmppServerWorker) worker; - smppServer.setMessageStore(new InMemorySmppServerMessageStore(smppServer)); + smppServer.setMessageStore(new StandardSmppServerMessageStore(smppServer)); } } catch (Exception e) { logger.error("Could not start worker {} {}", workerTypeString, instName, e); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java similarity index 97% rename from sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java index 7dacace..23683e5 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/InMemorySmppServerMessageStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java @@ -25,7 +25,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -class InMemorySmppServerMessageStoreTest { +class StandardSmppServerMessageStoreTest { @Mock private SmppServerWorker worker; @@ -36,7 +36,7 @@ class InMemorySmppServerMessageStoreTest { @Mock private DlrService dlrService; - private InMemorySmppServerMessageStore messageStore; + private StandardSmppServerMessageStore messageStore; @BeforeEach void setUp() { @@ -45,7 +45,7 @@ void setUp() { when(workerResources.getDlrService()).thenReturn(dlrService); when(worker.getMaxRetries()).thenReturn(5); - messageStore = new InMemorySmppServerMessageStore(worker); + messageStore = new StandardSmppServerMessageStore(worker); } @Test @@ -191,7 +191,7 @@ void getMaxAttempts_DelegatesToWorker() { @Test void getMaxAttempts_DefaultsTo3_WhenNoWorker() { - InMemorySmppServerMessageStore storeWithNullWorker = new InMemorySmppServerMessageStore(null); + StandardSmppServerMessageStore storeWithNullWorker = new StandardSmppServerMessageStore(null); int result = storeWithNullWorker.getMaxAttempts(true); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java similarity index 96% rename from sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java rename to sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java index 8a6fe6a..9169f0e 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/InMemoryMessageTrackerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java @@ -21,7 +21,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -class InMemoryMessageTrackerTest { +class StandardMessageTrackerTest { @Mock private AbstractOutWorker outWorker; @@ -32,7 +32,7 @@ class InMemoryMessageTrackerTest { @Mock private DlrService dlrService; - private InMemoryMessageTracker tracker; + private StandardMessageTracker tracker; @BeforeEach void setUp() { @@ -42,7 +42,7 @@ void setUp() { when(outWorker.getType()).thenReturn("testWorker"); when(outWorker.getDlrProviderName()).thenReturn("provider-1"); - tracker = new InMemoryMessageTracker(outWorker); + tracker = new StandardMessageTracker(outWorker); } @Test @@ -180,8 +180,4 @@ void createAndEnqueueDLR_UnknownMessage_DoesNotEnqueue() { verify(dlrService).resolveAndRemoveDlr("provider-1", "unknown", 0); } - @Test - void getDlrQueueSize_ReturnsQueueSize() { - assertEquals(0, tracker.getDlrQueueSize()); - } } From c9f1ff4bff294792f7999fbfe4014e3aac94e54e Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 20 Aug 2026 17:08:37 +0300 Subject: [PATCH 17/20] docs(dlr): detail provider correlation flow --- docs/01-architecture.md | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/01-architecture.md b/docs/01-architecture.md index beabff7..7bad16e 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -151,24 +151,46 @@ sequenceDiagram ## DLR Handling -Outbound HTTP messages can include a Kannel-style `dlr-url`. Sendium stores the gateway message ID and later links it to the message ID returned by the SMPP provider. Provider message IDs are scoped by the outbound provider name, so different providers may reuse the same ID without overwriting each other's correlation. When a DLR arrives, the DLR service resolves the provider and message-ID pair and forwards the callback. +Outbound HTTP messages can include a Kannel-style `dlr-url`. Before accepting a submission, Sendium stores its gateway message ID. After the upstream SMSC returns `submit_sm_resp`, the client worker links that gateway ID to the exact `(provider name, provider message ID)` pair. The provider name defaults to the worker's full name; workers sharing an SMSC message-ID namespace can use the same `msg.hash.prefix`. + +Different providers can reuse the same message ID independently. Reusing the same pair within one provider moves the correlation to the newest gateway message and clears it from the previous owner. Link and resolve transactions take a composite-key advisory lock and lock affected gateway rows in canonical UUID order, preventing crossed rebind and resolve deadlocks. Resolving a receipt transactionally consumes the tracked message and all of its correlations before callback forwarding or internal DLR queueing. ```mermaid sequenceDiagram - participant SMSC as Upstream SMSC + participant Ingress as HTTP/SMPP ingress + participant Router as Router queue participant Worker as SmppClientWorker + participant SMSC as Upstream SMSC participant Tracker as StandardMessageTracker - participant Store as DlrStorage - participant Router as Router queue + participant Service as DlrService + participant Database as PostgreSQL DLR storage participant DLRHook as ForwardDlrService participant App as Originating application + Ingress->>Service: saveInitialState(gateway message ID) + Service->>Database: Insert tracked message + Database-->>Service: Commit + Service-->>Ingress: State persisted + Ingress->>Router: Enqueue accepted message + Router->>Worker: Route outbound message + Worker->>SMSC: submit_sm + SMSC-->>Worker: submit_sm_resp(provider message ID) + Worker->>Tracker: linkProviderMessageId + Tracker->>Service: Link gateway ID and provider pair + Service->>Database: Lock and upsert provider correlation + SMSC->>Worker: deliver_sm delivery receipt Worker->>Tracker: createAndEnqueueDLR - Tracker->>Store: Resolve provider name and provider message ID - Store->>DLRHook: Forward DLR callback if URL exists - DLRHook->>App: HTTP GET callback + Tracker->>Service: resolveAndRemoveDlr(provider pair) + Service->>Database: Lock, resolve, and delete tracked state + Database-->>Service: Resolved message state + opt DLR callback URL exists + Service->>DLRHook: Forward DLR callback + DLRHook->>App: HTTP GET callback + end + Service-->>Tracker: Resolved message state Tracker->>Router: Enqueue internal MSG_DLR + Worker-->>SMSC: deliver_sm_resp ``` ## MO Handling From 92a75cc7724118f96409b7d1b797f6781b638822 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 20 Aug 2026 17:39:49 +0300 Subject: [PATCH 18/20] fix(test): harden native metric polling --- .../cytech/sendium/core/worker/Tracker.java | 6 +++++ .../src/test/java/utils/NativeE2eSmoke.java | 11 +++++--- .../test/java/utils/NativeE2eSmokeTest.java | 27 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 sendium-core/src/test/java/utils/NativeE2eSmokeTest.java diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java index d5cbd50..c1c6bec 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java @@ -12,12 +12,18 @@ public interface Tracker { void configure(String key, String newValue, String oldValue); + /** + * @param hashedProviderMessageId precomputed hash retained for tracker implementations that use hashed indexes + */ int updateSendStatusAndExtID(String hashedProviderMessageId, M message, String providerMessageId); String getHashedMessageID(String messageId); String getVendorPriceGateway(); + /** + * @param hashedProviderMessageId precomputed hash retained for tracker implementations that use hashed indexes + */ void createAndEnqueueDLR(int mqid, String providerMessageId, String hashedProviderMessageId, String from, String to, String body, int state, String errorCode, HashMap tlvs); diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index 363f1a2..8802e11 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -341,15 +341,20 @@ && hasSuccessfulStorageOperation(response.body(), operation)) { throw new IllegalStateException("Timed out waiting for successful DLR storage operation: " + operation); } - private static boolean hasSuccessfulStorageOperation(String metrics, String operation) { + static boolean hasSuccessfulStorageOperation(String metrics, String operation) { return metrics.lines() .filter(line -> line.startsWith("sendium_dlr_storage_operation_seconds_count")) .filter(line -> line.contains("backend=\"postgresql\"")) .filter(line -> line.contains("operation=\"" + operation + "\"")) .filter(line -> line.contains("outcome=\"success\"")) .map(line -> line.substring(line.lastIndexOf(' ') + 1)) - .mapToDouble(Double::parseDouble) - .anyMatch(count -> count >= 1.0); + .anyMatch(value -> { + try { + return Double.parseDouble(value) >= 1.0; + } catch (NumberFormatException ignored) { + return false; + } + }); } private static HttpResponse get(String path) throws IOException, InterruptedException { diff --git a/sendium-core/src/test/java/utils/NativeE2eSmokeTest.java b/sendium-core/src/test/java/utils/NativeE2eSmokeTest.java new file mode 100644 index 0000000..d8eb51c --- /dev/null +++ b/sendium-core/src/test/java/utils/NativeE2eSmokeTest.java @@ -0,0 +1,27 @@ +package utils; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class NativeE2eSmokeTest { + + @Test + void hasSuccessfulStorageOperationIgnoresMalformedMetricValues() { + String metrics = """ + sendium_dlr_storage_operation_seconds_count{backend="postgresql",operation="link_provider",outcome="success"} invalid + sendium_dlr_storage_operation_seconds_count{backend="postgresql",operation="link_provider",outcome="success"} 1.0 + """; + + assertThat(NativeE2eSmoke.hasSuccessfulStorageOperation(metrics, "link_provider")).isTrue(); + } + + @Test + void hasSuccessfulStorageOperationRejectsOnlyMalformedMetricValues() { + String metrics = """ + sendium_dlr_storage_operation_seconds_count{backend="postgresql",operation="link_provider",outcome="success"} invalid + """; + + assertThat(NativeE2eSmoke.hasSuccessfulStorageOperation(metrics, "link_provider")).isFalse(); + } +} From c18ab9ed189c1be5dd88cab1ae1a971f12452a88 Mon Sep 17 00:00:00 2001 From: pavlos Date: Fri, 21 Aug 2026 15:40:14 +0300 Subject: [PATCH 19/20] feat(dlr): make downstream delivery durable --- docs/01-architecture.md | 30 +- docs/13-dlr-persistence.md | 22 +- sendium-core/pom.xml | 4 + .../sendium/core/http/KannelResource.java | 2 + .../core/smpp/client/SmppClientWorker.java | 8 +- .../core/smpp/server/AccountConnections.java | 1 + .../smpp/server/DlrDeliverSmReference.java | 13 + .../core/smpp/server/DlrDeliveryBatch.java | 128 +++ .../core/smpp/server/ServerConnections.java | 3 + .../smpp/server/SmppServerBindHandler.java | 2 +- .../smpp/server/SmppServerMessageStore.java | 29 + .../smpp/server/SmppServerSessionHandler.java | 51 ++ .../core/smpp/server/SmppServerWorker.java | 69 +- .../StandardSmppServerMessageStore.java | 88 +- .../core/smpp/server/tasks/OutTask.java | 28 +- .../core/worker/DlrMessageStorage.java | 24 +- .../sendium/core/worker/DlrService.java | 55 +- .../sendium/core/worker/DlrStorage.java | 20 +- .../core/worker/ForwardDlrService.java | 240 ++++-- .../core/worker/ManagedDlrStorage.java | 41 +- .../sendium/core/worker/MessageState.java | 94 ++ .../core/worker/PostgresqlDlrStorage.java | 758 ++++++++-------- .../core/worker/StandardMessageTracker.java | 5 +- .../sendium/core/worker/UnpushedDlr.java | 61 -- .../V1__create_sendium_dlr_schema.sql | 78 +- .../core/dlr/PostgresqlMigrationIT.java | 317 +++---- .../sendium/core/http/KannelResourceTest.java | 12 + .../smpp/client/SmppClientWorkerTest.java | 23 + .../smpp/server/DlrDeliveryBatchTest.java | 113 +++ .../server/SmppServerSessionHandlerTest.java | 98 ++- .../SmppServerWorkerReassemblyTest.java | 155 +++- .../StandardSmppServerMessageStoreTest.java | 168 ++-- .../core/smpp/server/tasks/OutTaskTest.java | 86 ++ .../sendium/core/worker/DlrServiceTest.java | 51 +- .../core/worker/ForwardDlrServiceTest.java | 342 +++++++- .../core/worker/PostgresqlDlrStorageIT.java | 810 +++++------------- .../worker/StandardMessageTrackerTest.java | 43 +- .../src/test/java/utils/NativeE2eSmoke.java | 62 +- 38 files changed, 2484 insertions(+), 1650 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliverSmReference.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java delete mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/worker/UnpushedDlr.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatchTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/tasks/OutTaskTest.java diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 7bad16e..7674886 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -168,7 +168,7 @@ sequenceDiagram participant App as Originating application Ingress->>Service: saveInitialState(gateway message ID) - Service->>Database: Insert tracked message + Service->>Database: Insert DLR message Database-->>Service: Commit Service-->>Ingress: State persisted Ingress->>Router: Enqueue accepted message @@ -180,17 +180,21 @@ sequenceDiagram Service->>Database: Lock and upsert provider correlation SMSC->>Worker: deliver_sm delivery receipt - Worker->>Tracker: createAndEnqueueDLR - Tracker->>Service: resolveAndRemoveDlr(provider pair) - Service->>Database: Lock, resolve, and delete tracked state - Database-->>Service: Resolved message state - opt DLR callback URL exists - Service->>DLRHook: Forward DLR callback - DLRHook->>App: HTTP GET callback + alt ACCEPTD or ENROUTE receipt + Worker-->>SMSC: deliver_sm_resp (success) + else Terminal receipt + Worker->>Tracker: createAndEnqueueDLR + Tracker->>Service: resolveDlr(provider pair, exact state/error) + Service->>Database: Lock, resolve, consume correlations, retain pending delivery + Database-->>Service: Resolved message state + opt DLR callback URL exists + Service->>DLRHook: Forward DLR callback + DLRHook->>App: HTTP GET callback + end + Service-->>Tracker: Resolved message state + Tracker->>Router: Enqueue internal MSG_DLR + Worker-->>SMSC: deliver_sm_resp end - Service-->>Tracker: Resolved message state - Tracker->>Router: Enqueue internal MSG_DLR - Worker-->>SMSC: deliver_sm_resp ``` ## MO Handling @@ -238,9 +242,9 @@ Sendium expects runtime files in the configured `conf` directory. ## Persistence Boundaries -Most runtime queues are in memory. DLR tracking, provider correlations, and unpushed downstream SMPP receipts use PostgreSQL. Sendium completes the required storage operation before HTTP routing or successful downstream SMPP acknowledgement. Queued and in-flight messages remain process-local. +Most runtime queues are in memory. DLR messages, provider correlations, and terminal HTTP/SMPP delivery state use PostgreSQL. Sendium completes the required storage operation before HTTP routing or successful downstream SMPP acknowledgement. Queued and in-flight messages remain process-local. -PostgreSQL does not make multipart assembly, replay claims, callback retries, or router and worker queues durable. See [DLR Persistence](13-dlr-persistence.md) for retention, restart guarantees, and the remaining crash windows. +PostgreSQL does not make multipart assembly or router and worker queues durable. See [DLR Persistence](13-dlr-persistence.md) for retention, restart guarantees, and the remaining crash windows. ## Related Documentation diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 8b0d532..99e32fb 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -1,6 +1,6 @@ # DLR Persistence -Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and replay receipts that could not be delivered to a downstream SMPP client in PostgreSQL. +Sendium stores the state needed to correlate upstream delivery receipts (DLRs) and durably track terminal HTTP/SMPP delivery in PostgreSQL. This storage boundary does not make Sendium's message queues or all delivery processing durable. Review [Durability Boundaries](#durability-boundaries) before using restart recovery as a delivery guarantee. @@ -34,7 +34,7 @@ Quick Start preserves the local database password during `--force` regeneration, ## Upgrade From MVStore Builds -Older Sendium builds could store DLR state in `data/dlr-mvstore.db`. Current builds do not read or import that file. Before upgrading an MVStore-configured runtime, stop accepting submissions and allow pending provider correlations and unpushed downstream receipts to drain, or explicitly accept that the remaining state will be unavailable after the upgrade. Stop Sendium and preserve the old file before starting the PostgreSQL-only build. +Older Sendium builds could store DLR state in `data/dlr-mvstore.db`. Current builds do not read or import that file. Before upgrading an MVStore-configured runtime, stop accepting submissions and allow pending provider correlations and terminal deliveries to drain, or explicitly accept that the remaining state will be unavailable after the upgrade. Stop Sendium and preserve the old file before starting the PostgreSQL-only build. Provision PostgreSQL and require readiness to report `UP` with `backend=postgresql` before reopening traffic. State written to PostgreSQL is not available to an older MVStore build if the application is later downgraded. @@ -98,6 +98,10 @@ PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP subm Provider message IDs are correlated within the outbound provider namespace rather than globally. The worker instance name is the default namespace; workers connected to the same SMSC account can share `msg.hash.prefix` when that SMSC may deliver their receipts interchangeably. Different providers may therefore return the same message ID without overwriting each other's state. The namespace must remain stable while correlations are outstanding: changing `msg.hash.prefix` or renaming a worker using the default makes earlier receipts unresolvable. +Sendium requests final delivery receipts from upstream SMPP providers. A valid unsolicited `ACCEPTD` or `ENROUTE` receipt is acknowledged successfully but is not forwarded and does not consume its provider correlation. The first terminal receipt consumes the correlation and produces the downstream DLR; later receipts for that provider message ID cannot resolve it. Multipart submissions retain this first-terminal behavior and do not aggregate delivery states across every segment. + +A terminal receipt remains in `sendium_dlr.dlr_message` while HTTP or SMPP delivery is pending. The delivery attempt number is a fencing token: a stale completion or failure cannot mutate a newer attempt, and an adapter-local active-ID guard prevents duplicate starts within one process. + ## Retention The V1 retention thresholds are fixed application behavior, not environment settings: @@ -105,8 +109,8 @@ The V1 retention thresholds are fixed application behavior, not environment sett | State | Eligible for cleanup after | | :--- | :--- | | Provider message correlation | 3 days | -| Tracked gateway message | 7 days | -| Unpushed downstream SMPP receipt | 7 days | +| Message waiting for provider receipt | 7 days from creation | +| Pending or failed terminal delivery | 7 days from resolution | Cleanup is triggered by storage activity and runs no more than once per hour. These values are therefore eligibility thresholds, not exact physical deletion deadlines: idle records can remain in the database longer, and an active deployment can retain newly eligible state until the next cleanup pass. A provider receipt cannot be matched after its correlation has been removed. Making the thresholds or cleanup schedule configurable is outside the V1 storage replacement. @@ -117,14 +121,14 @@ Cleanup is best-effort maintenance and is isolated from message handling. One ca | State or transition | PostgreSQL guarantee | Remaining limit | | :--- | :--- | :--- | | Initial DLR state for HTTP and downstream SMPP submissions | Persisted before HTTP routing or a successful SMPP acknowledgement. | Router and worker queues remain in memory. A process crash can lose queued outbound work even though its DLR row remains until cleanup. | -| Gateway-to-provider message correlation | Survives Sendium restart after the provider message ID is linked. | Resolving a provider receipt consumes the correlation before callback or downstream delivery completes. A crash in that window can lose the resulting receipt. | -| Unpushed downstream SMPP receipt | Survives restart and is replayed when the same system ID binds again. | The row is removed after admission to the worker queue, not after confirmed downstream delivery. A crash in that window can lose the receipt. | -| Replay claim | Prevents duplicate replay within one Sendium process. | Claims are process-local. Multiple active Sendium replicas can claim and deliver the same database row. V1 supports one active gateway process. | +| Gateway-to-provider message correlation | Survives Sendium restart after the provider message ID is linked. Intermediate `ACCEPTD` and `ENROUTE` receipts leave it intact. | The first terminal receipt consumes every correlation for the gateway message. | +| Terminal HTTP/SMPP delivery | The common payload and exact provider outcome remain in one row until fenced completion. | Delivery scheduling and SMPP response batching are separate runtime concerns. | +| Active delivery attempt | The database attempt number fences stale completion, retry, and failure updates. | The active-ID guard is process-local. Adapter recreation may start a new attempt for an attempt that was active before a crash. | | Multipart submission | Each acknowledged segment has provisional DLR state; completed aggregates update the primary state. | Multipart assembly and its pending timers are process-local and are not reconstructed after restart. | -| HTTP DLR callback retry | The resolved callback is attempted up to 10 times while the process remains running. | The retry schedule is in memory and is lost on restart. There is no durable callback outbox. | +| HTTP DLR callback retry | Pending state and the next-attempt timestamp are durable. | The scheduler that consumes due rows is implemented separately. | | Database files | The Quick Start named volume survives normal container replacement and `docker compose down`. | Volume deletion, host-disk loss, and disaster recovery require backups or external PostgreSQL replication managed by the operator. | -These limits are intentional V1 boundaries. PostgreSQL provides DLR persistence; it is not a durable queue, distributed claim coordinator, or delivery outbox. +These limits are intentional V1 boundaries. PostgreSQL provides DLR persistence and delivery fencing; it is not a distributed worker coordinator or a replacement for the router and worker queues. ## Related Documentation diff --git a/sendium-core/pom.xml b/sendium-core/pom.xml index fe16063..27e485f 100644 --- a/sendium-core/pom.xml +++ b/sendium-core/pom.xml @@ -28,6 +28,10 @@ io.quarkus quarkus-arc + + io.quarkus + quarkus-scheduler + io.quarkus quarkus-smallrye-openapi diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index dc7690f..ba88759 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -214,6 +214,8 @@ public Response receiveSms( msg.serial = UUID.randomUUID().toString(); if (!dlrServices.isUnsatisfied()) { MessageState state = new MessageState(msg.serial, usr, msg.from, msg.to, dlrUrl); + state.setDeliveryChannel(dlrUrl == null || dlrUrl.isBlank() ? + MessageState.DeliveryChannel.NONE : MessageState.DeliveryChannel.HTTP); dlrServices.get().saveInitialState(state); } queueProvider.getRouterQueue().enqueue(msg); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java index d4142aa..dba724d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java @@ -32,6 +32,7 @@ import gr.cytech.sendium.core.smpp.util.CustomCharset; import gr.cytech.sendium.core.smpp.util.SmppServerUtil; import gr.cytech.sendium.core.smpp.util.VFGRCharset; +import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.ForwardMoService; import gr.cytech.sendium.core.worker.Tracker; @@ -989,7 +990,6 @@ public PduResponse parseDlrAndCreateResponse(DeliverSm deliverSm) { // the original parseShortMessage method will throw an exception if err field is more than 3 chars // among other validations it performs. The extended one does not throw exception for invalid fields var receipt = DeliveryReceipt.parseShortMessage(dlrBody, ZoneOffset.UTC, false, false); - String errcode = extractErrorCode(receipt.getRawErrorCode(), receipt.getErrorCode()); int state = SmppServerUtil.decodeFinalState(receipt.getState()); if (dlrBody.length() > 159) { dlrBody = dlrBody.substring(0, 159); @@ -999,6 +999,12 @@ public PduResponse parseDlrAndCreateResponse(DeliverSm deliverSm) { logger.warn("Invalid provider message ID, skipping unknown DLR {}", MessageTrace.pdu(deliverSm)); return deliverSm.createGenericNack(SmppConstants.STATUS_SYSERR); } + if (!DlrService.isTerminalDlrState(state)) { + logger.debug("Ignoring intermediate DLR state={} providerMessageId={}", state, + MessageTrace.value(providerMessageId)); + return deliverSm.createResponse(); + } + String errcode = extractErrorCode(receipt.getRawErrorCode(), receipt.getErrorCode()); HashMap tlvs = extractTlvs(this.tlvsDlrs, deliverSm); messageTracker.createAndEnqueueDLR(0, providerMessageId, getHashedMessageID(providerMessageId), from, to, dlrBody, state, errcode, tlvs); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/AccountConnections.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/AccountConnections.java index ac0ca7a..9b5a39a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/AccountConnections.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/AccountConnections.java @@ -38,6 +38,7 @@ protected void checkHandlerForInactivity(SmppServerSessionHandler handler, long "[SystemID:" + handler.getSession().getConfiguration().getSystemId() + "]", currentInactiveTime); if (handler.getSession() != null && SmppSession.Type.SERVER.equals(handler.getSession().getLocalType())) { + handler.failActiveDlrBatches(); handler.getSession().destroy(); serverConnections.removeConnection(handler); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliverSmReference.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliverSmReference.java new file mode 100644 index 0000000..43adfd1 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliverSmReference.java @@ -0,0 +1,13 @@ +package gr.cytech.sendium.core.smpp.server; + +import gr.cytech.sendium.core.message.StandardMessage; + +/** + * Typed reference attached to each deliver_sm belonging to a durable DLR batch. + */ +public record DlrDeliverSmReference( + SmppServerSessionHandler handler, + DlrDeliveryBatch batch, + int partOrdinal, + String receiptMessageId) { +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java new file mode 100644 index 0000000..d964894 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java @@ -0,0 +1,128 @@ +package gr.cytech.sendium.core.smpp.server; + +import gr.cytech.sendium.core.message.StandardMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; + +/** + * Tracks one durable SMPP DLR attempt across all generated deliver_sm parts. + */ +public final class DlrDeliveryBatch { + public static final String COMPLETION_STORAGE_ERROR = "completion_storage_error"; + private static final Logger logger = LoggerFactory.getLogger(DlrDeliveryBatch.class); + private static final Set FAILURE_RESULTS = Set.of( + COMPLETION_STORAGE_ERROR, + "enqueue_failed", + "generic_nack", + "non_ok_response", + "send_failed", + "session_closed", + "timeout", + "wrong_response"); + + private final M message; + private final int attempt; + private final Set expectedParts; + private final Set successfulParts = new HashSet<>(); + private final SmppServerMessageStore messageStore; + private final SmppServerSessionHandler owner; + private boolean active = true; + + public DlrDeliveryBatch(M message, int attempt, Set expectedParts, + SmppServerMessageStore messageStore, SmppServerSessionHandler owner) { + if (message == null || message.serial == null || message.serial.isBlank()) { + throw new IllegalArgumentException("DLR batch requires a gateway message ID"); + } + if (expectedParts == null || expectedParts.isEmpty()) { + throw new IllegalArgumentException("DLR batch requires at least one part"); + } + this.message = message; + this.attempt = attempt; + this.expectedParts = Set.copyOf(expectedParts); + this.messageStore = messageStore; + this.owner = owner; + } + + public synchronized boolean isActive() { + return active; + } + + public int getAttempt() { + return attempt; + } + + public String getGatewayMessageId() { + return message.serial; + } + + public void partSucceeded(int partOrdinal) { + boolean complete = false; + synchronized (this) { + if (!active || !expectedParts.contains(partOrdinal) || !successfulParts.add(partOrdinal)) { + return; + } + if (successfulParts.size() == expectedParts.size()) { + active = false; + complete = true; + } + } + if (complete) { + completeDelivery(); + } + } + + public void fail(String result) { + synchronized (this) { + if (!active) { + return; + } + active = false; + } + release(normalizeResult(result)); + unregister(); + } + + private void completeDelivery() { + boolean completed = false; + try { + completed = messageStore.completeDlrDeliveryAttempt(message, attempt); + } catch (RuntimeException e) { + logger.error("SMPP DLR completion storage error gatewayMsgId={} attempt={}", + message.serial, attempt, e); + } + if (completed) { + logger.info("SMPP DLR delivery completed gatewayMsgId={} attempt={}", message.serial, attempt); + } else { + release(COMPLETION_STORAGE_ERROR); + } + unregister(); + } + + private void release(String result) { + try { + if (!messageStore.releaseDlrDeliveryAttempt(message, attempt, result)) { + logger.error("SMPP DLR attempt release was not applied gatewayMsgId={} attempt={} result={}", + message.serial, attempt, result); + } else { + logger.warn("SMPP DLR delivery attempt failed gatewayMsgId={} attempt={} result={}", + message.serial, attempt, result); + } + } catch (RuntimeException e) { + logger.error("SMPP DLR attempt release failed gatewayMsgId={} attempt={} result={}", + message.serial, attempt, result, e); + } + } + + private void unregister() { + if (owner != null) { + owner.unregisterDlrBatch(this); + } + } + + private String normalizeResult(String result) { + return FAILURE_RESULTS.contains(result) ? result : "send_failed"; + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/ServerConnections.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/ServerConnections.java index 3b27ab9..cbe3965 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/ServerConnections.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/ServerConnections.java @@ -38,6 +38,9 @@ public synchronized void addConnection(String accountId, SmppServerSessionHandle } public synchronized boolean removeConnection(SmppServerSessionHandler handler, SmppSession session) { + if (handler != null) { + handler.failActiveDlrBatches(); + } if (session == null) { logger.warn("trying to remove a null session from account connections, rejecting"); return false; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerBindHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerBindHandler.java index ca28389..59eae34 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerBindHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerBindHandler.java @@ -141,7 +141,7 @@ public boolean isSystemIdReachable(String accountId, String systemId) { return connections.isSystemIdReachable(accountId, systemId); } - public SmppServerSessionHandler getHandlerForSending(String accountId, String systemId) { + public SmppServerSessionHandler getHandlerForSending(String accountId, String systemId) { return connections.getHandlerForSending(accountId, systemId); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java index c779a99..e90b2cc 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerMessageStore.java @@ -3,6 +3,7 @@ import gr.cytech.sendium.core.message.StandardMessage; import java.util.List; +import java.util.OptionalInt; import java.util.concurrent.Future; public interface SmppServerMessageStore { @@ -32,6 +33,34 @@ default boolean persistsBeforeAcknowledgement() { */ boolean markAsUnpushed(M msg); + /** + * Whether this store keeps DLR rows until the downstream client acknowledges them. + */ + default boolean tracksDlrDeliveryAttempts() { + return false; + } + + /** + * Starts one logical DLR delivery attempt. The default keeps existing stores source-compatible. + */ + default OptionalInt startDlrDeliveryAttempt(M msg) { + return OptionalInt.empty(); + } + + /** + * Completes a logical DLR delivery attempt after all receipt parts were acknowledged. + */ + default boolean completeDlrDeliveryAttempt(M msg, int attempt) { + return true; + } + + /** + * Releases a failed logical DLR delivery attempt for a later replay. + */ + default boolean releaseDlrDeliveryAttempt(M msg, int attempt, String result) { + return markAsUnpushed(msg); + } + /** * Called when a transmittable SMPP client session becomes available again. */ diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandler.java index 68131fd..ea06391 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandler.java @@ -4,6 +4,7 @@ import com.cloudhopper.smpp.PduAsyncResponse; import com.cloudhopper.smpp.SmppConstants; import com.cloudhopper.smpp.SmppSession; +import com.cloudhopper.smpp.pdu.DeliverSmResp; import com.cloudhopper.smpp.pdu.GenericNack; import com.cloudhopper.smpp.pdu.PartialPdu; import com.cloudhopper.smpp.pdu.Pdu; @@ -33,6 +34,9 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; public class SmppServerSessionHandler implements SmsgSmppSessionHandler { public static final String DATE_FORMAT = "yyyy-MMM-dd HH:mm:ss.SS+z"; @@ -49,6 +53,8 @@ public class SmppServerSessionHandler implements Smsg private final RateLimiter rateController; private final SmppSessionContext sessionContext; private final SubmitSmProcessor submitProcessor; + private final Set> activeDlrBatches = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(); private String apiProduct; public SmppServerSessionHandler(SmppServerWorker worker, @@ -134,6 +140,10 @@ public void firePduRequestExpired(PduRequest pduRequest) { logger.info("{}: received expired request PDU {}", this, MessageTrace.pdu(pduRequest)); if (pduRequest.getCommandId() == SmppConstants.CMD_ID_DELIVER_SM) { + if (pduRequest.getReferenceObject() instanceof DlrDeliverSmReference reference) { + reference.batch().fail("timeout"); + return; + } try { Object[] arr = (Object[]) pduRequest.getReferenceObject(); M original = (M) arr[1]; @@ -153,6 +163,7 @@ public void firePduRequestExpired(PduRequest pduRequest) { */ public void fireChannelUnexpectedlyClosed() { logger.warn("{}: closed unexpectedly", this); + failActiveDlrBatches(); worker.getBindHandler().getConnections().removeConnection(this); } @@ -172,6 +183,21 @@ public void fireChannelUnexpectedlyClosed() { public void fireExpectedPduResponseReceived(PduAsyncResponse pduAsyncResponse) { logger.trace("{}: received expected response PDU: {}", this, pduAsyncResponse.getResponse()); + if (pduAsyncResponse.getRequest() != null && + pduAsyncResponse.getRequest().getReferenceObject() instanceof DlrDeliverSmReference reference) { + PduResponse response = pduAsyncResponse.getResponse(); + if (response instanceof GenericNack) { + reference.batch().fail("generic_nack"); + } else if (!(response instanceof DeliverSmResp)) { + reference.batch().fail("wrong_response"); + } else if (response.getCommandStatus() != SmppConstants.STATUS_OK) { + reference.batch().fail("non_ok_response"); + } else { + reference.batch().partSucceeded(reference.partOrdinal()); + } + return; + } + /* * Its possible the response PDU really isn't the correct PDU we were waiting for, * so we should verify it. For example it is possible that a "Generic_Nack" could @@ -205,6 +231,7 @@ public void fireUnexpectedPduResponseReceived(PduResponse pduResponse) { * @param e The exception */ public void fireUnrecoverablePduException(UnrecoverablePduException e) { + failActiveDlrBatches(); getSession().destroy(); logger.warn("{}: destroyed because of unrecoverable pdu exception", this, e); } @@ -368,6 +395,30 @@ public SmppServerWorker getWorker() { return worker; } + public boolean registerDlrBatch(DlrDeliveryBatch batch) { + if (closed.get()) { + batch.fail("session_closed"); + return false; + } + activeDlrBatches.add(batch); + if (closed.get() && activeDlrBatches.remove(batch)) { + batch.fail("session_closed"); + return false; + } + return true; + } + + public void unregisterDlrBatch(DlrDeliveryBatch batch) { + activeDlrBatches.remove(batch); + } + + public void failActiveDlrBatches() { + closed.set(true); + for (DlrDeliveryBatch batch : activeDlrBatches) { + batch.fail("session_closed"); + } + } + public void handleSubmitSm(SubmitSm submitSm) { String userId = getAccountId(); if (userId == null) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java index 3cde255..fb2ce73 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java @@ -47,8 +47,10 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.OptionalInt; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -637,6 +639,9 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { boolean hasSystemId = !Strings.isNullOrEmpty(pMsg.systemId); if ((hasSystemId && !bindHandler.isSystemIdReachable(pMsg.owner_id, pMsg.systemId)) || !bindHandler.isConnectionReachable(pMsg.owner_id)) { + if (isDlr && messageStore.tracksDlrDeliveryAttempts()) { + return null; + } if (markAsUnpushed(pMsg)) { return null; } @@ -646,6 +651,9 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { var handler = bindHandler.getHandlerForSending(pMsg.owner_id, pMsg.systemId); var session = handler != null ? handler.getSession() : null; if (session == null || !session.isBound()) { + if (isDlr && messageStore.tracksDlrDeliveryAttempts()) { + return null; + } if (markAsUnpushed(pMsg)) { return null; } @@ -664,6 +672,11 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { return isDlr ? null : pMsg; } + if (isDlr && messageStore.tracksDlrDeliveryAttempts()) { + enqueueDlrBatch(handler, pMsg, requests); + return null; + } + for (DeliverSm deliverSm : requests) { Object deliverMsgId = deliverSm.getReferenceObject(); if (deliverMsgId instanceof String msgId) { @@ -679,7 +692,50 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { return null; } - protected List generateDeliverSmForDLR(M pMsg) { + private void enqueueDlrBatch(SmppServerSessionHandler handler, M message, List requests) { + OptionalInt started; + try { + started = messageStore.startDlrDeliveryAttempt(message); + } catch (RuntimeException e) { + logger.error("Failed to start SMPP DLR attempt gatewayMsgId={}", message.serial, e); + return; + } + if (started.isEmpty()) { + logger.debug("SMPP DLR attempt already active gatewayMsgId={}", message.serial); + return; + } + + Set expectedParts = new HashSet<>(); + for (int i = 0; i < requests.size(); i++) { + expectedParts.add(i); + } + DlrDeliveryBatch batch = new DlrDeliveryBatch<>( + message, started.getAsInt(), expectedParts, messageStore, handler); + for (int i = 0; i < requests.size(); i++) { + DeliverSm deliverSm = requests.get(i); + String receiptMessageId = (String) deliverSm.getReferenceObject(); + deliverSm.setReferenceObject(new DlrDeliverSmReference<>(handler, batch, i, receiptMessageId)); + } + if (!handler.registerDlrBatch(batch)) { + return; + } + + for (DeliverSm deliverSm : requests) { + try { + enqueueOut(deliverSm); + if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_DELIVER_ENQUEUED)) { + logger.info("message.deliver.enqueued worker={} {}", getFullName(), MessageTrace.identifiers(message)); + } + } catch (RuntimeException e) { + logger.warn("Failed to enqueue SMPP DLR gatewayMsgId={} attempt={}", + message.serial, started.getAsInt(), e); + batch.fail("enqueue_failed"); + return; + } + } + } + + protected List generateDeliverSmForDLR(M pMsg) throws SmppInvalidArgumentException { Address sender = new Address(SmppConstants.TON_UNKNOWN, SmppConstants.NPI_UNKNOWN, pMsg.from); Address receiver = new Address(SmppConstants.TON_UNKNOWN, SmppConstants.NPI_UNKNOWN, pMsg.to); @@ -709,7 +765,8 @@ protected List generateDeliverSmForDLR(M pMsg) { } protected DeliverSm getDeliverSm(M pMsg, String messageId, int errorCode, Address sender, Address receiver, - byte coding, byte requestDelivery, String charset) { + byte coding, byte requestDelivery, String charset) + throws SmppInvalidArgumentException { var submitDate = ZonedDateTime.now(ZoneOffset.UTC); // Ideally fetch from pMsg if populated var doneDate = ZonedDateTime.now(ZoneOffset.UTC); @@ -724,13 +781,7 @@ protected DeliverSm getDeliverSm(M pMsg, String messageId, int errorCode, Addres deliverSm.setPriority((byte) (pMsg.priority >= StandardMessage.LOW_PRIORITY && pMsg.priority <= StandardMessage.HIGH_PRIORITY ? pMsg.priority : StandardMessage.NORMAL_PRIORITY)); - try { - deliverSm.setShortMessage(CharsetUtil.encode(deliveryReceipt.toShortMessage(), charset)); - } catch (SmppInvalidArgumentException e) { - logger.warn("Caught SmppInvalidArgumentException", e); - markAsUnpushed(pMsg); - return null; - } + deliverSm.setShortMessage(CharsetUtil.encode(deliveryReceipt.toShortMessage(), charset)); deliverSm.setEsmClass(SmppConstants.ESM_CLASS_MT_SMSC_DELIVERY_RECEIPT); deliverSm.setReferenceObject(messageId); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java index 230c1cc..1988d97 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStore.java @@ -4,7 +4,6 @@ import gr.cytech.sendium.core.worker.DlrService; import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.MessageState; -import gr.cytech.sendium.util.MessageTrace; import gr.cytech.sendium.util.SensitiveLogSanitizer; import jakarta.inject.Inject; import org.slf4j.Logger; @@ -12,6 +11,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -71,6 +71,9 @@ private boolean persistBatch(List> eventsQueue) { StandardMessage msg = event.pMsg; if (msg != null) { MessageState state = new MessageState(msg.serial, msg.owner_id, msg.systemId, msg.from, msg.to, null); + state.setDeliveryChannel(msg.acked && worker.isForwardDlrs() ? + MessageState.DeliveryChannel.SMPP + : MessageState.DeliveryChannel.NONE); state.setReassembledParts(msg.reassembledParts); states.add(state); } @@ -103,24 +106,33 @@ public boolean persistsBeforeAcknowledgement() { @Override public boolean markAsUnpushed(StandardMessage msg) { - if (msg == null || msg.type != StandardMessage.MSG_DLR) { - return false; - } + return msg != null && msg.type == StandardMessage.MSG_DLR && isDlrPersistenceEnabled(); + } + + @Override + public boolean tracksDlrDeliveryAttempts() { + return isDlrPersistenceEnabled(); + } + + @Override + public OptionalInt startDlrDeliveryAttempt(StandardMessage msg) { if (!isDlrPersistenceEnabled()) { - //let the worker retry in memory, as documented on SmppServerMessageStore#markAsUnpushed - return false; + return SmppServerMessageStore.super.startDlrDeliveryAttempt(msg); } + return getDlrService().startDeliveryAttempt(msg.serial, MessageState.DeliveryChannel.SMPP) + .map(state -> OptionalInt.of(state.getDeliveryAttemptCount())) + .orElseGet(OptionalInt::empty); + } - try { - boolean saved = getDlrService().saveUnpushedDlr(msg); - if (!saved) { - logger.warn("Failed to save unpushed DLR {}", MessageTrace.identifiers(msg)); - } - return saved; - } catch (Exception e) { - logger.warn("Exception while saving unpushed DLR {}", MessageTrace.identifiers(msg), e); - return false; - } + @Override + public boolean completeDlrDeliveryAttempt(StandardMessage msg, int attempt) { + return !isDlrPersistenceEnabled() || getDlrService().completeDelivery(msg.serial, attempt); + } + + @Override + public boolean releaseDlrDeliveryAttempt(StandardMessage msg, int attempt, String result) { + return !isDlrPersistenceEnabled() || + getDlrService().retryDelivery(msg.serial, attempt, result, System.currentTimeMillis()); } @Override @@ -128,29 +140,37 @@ public void onClientConnected(String systemId) { if (!isDlrPersistenceEnabled()) { return; } - - DlrService dlrService = getDlrService(); - List unpushedDlrs = dlrService.claimUnpushedDlrs(systemId); - if (unpushedDlrs.isEmpty()) { - logger.info("Unpushed DLR(s) not found for systemId:{}", systemId); - return; - } - - logger.info("Re-enqueuing {} unpushed DLR(s) for systemId:{}", unpushedDlrs.size(), systemId); - for (StandardMessage msg : unpushedDlrs) { + List pending = getDlrService().listPendingSmppDeliveries(systemId); + for (MessageState state : pending) { + StandardMessage dlr = toDlrMessage(state); try { - if (worker.enqueueNoExceptions(msg)) { - dlrService.removeUnpushedDlr(msg); - } else { - dlrService.releaseUnpushedDlrClaim(msg); - } - } catch (Exception e) { - dlrService.releaseUnpushedDlrClaim(msg); - logger.warn("Failed to re-enqueue unpushed DLR {}", MessageTrace.identifiers(msg), e); + worker.enqueue(dlr); + logger.info("SMPP DLR replay enqueued gatewayMsgId={}", state.getGatewayMsgId()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("SMPP DLR replay enqueue interrupted gatewayMsgId={}", state.getGatewayMsgId()); + return; + } catch (RuntimeException e) { + logger.warn("SMPP DLR replay enqueue failed gatewayMsgId={}", state.getGatewayMsgId(), e); } } } + private StandardMessage toDlrMessage(MessageState state) { + StandardMessage dlr = new StandardMessage(); + dlr.serial = state.getGatewayMsgId(); + dlr.from = state.getDestAddr(); + dlr.to = state.getSourceAddr(); + dlr.state = state.getDlrState(); + dlr.errcode = state.getErrorCode(); + dlr.systemId = state.getSystemId(); + dlr.owner_id = state.getAccountId(); + List reassembledParts = state.getReassembledParts(); + dlr.reassembledParts = reassembledParts == null ? null : new ArrayList<>(reassembledParts); + dlr.type = StandardMessage.MSG_DLR; + return dlr; + } + private boolean isDlrPersistenceEnabled() { return worker.getWorkerResources().isDlrPersistenceEnabled(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java index 4724667..897dcd3 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java @@ -4,6 +4,7 @@ import com.cloudhopper.smpp.pdu.PduRequest; import com.cloudhopper.smpp.pdu.PduResponse; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.smpp.server.DlrDeliverSmReference; import gr.cytech.sendium.core.smpp.server.SmppServerSessionHandler; import gr.cytech.sendium.core.smpp.server.SmppServerWorker; import gr.cytech.sendium.util.MessageTrace; @@ -25,6 +26,11 @@ public OutTask(SmppServerWorker worker, Pdu pdu) { @Override public void run() { boolean success; + DlrDeliverSmReference dlrReference = + pdu.getReferenceObject() instanceof DlrDeliverSmReference reference ? reference : null; + if (dlrReference != null && !dlrReference.batch().isActive()) { + return; + } try { if (pdu.isResponse()) { //for responses, the pdu contains the handler as a reference object @@ -40,11 +46,17 @@ public void run() { success = handler.sendPduResponse((PduResponse) pdu); } else { //for requests, the pdu contains an array with the handler and possibly the original message (dlr/mo) - Object[] arr = (Object[]) pdu.getReferenceObject(); - SmppServerSessionHandler handler = (SmppServerSessionHandler) arr[0]; - msg = (M) arr[1]; - deliverMsgId = arr.length > 2 && arr[2] instanceof String id ? id : null; - success = handler.sendPduRequest((PduRequest) pdu); + if (dlrReference != null) { + msg = null; + deliverMsgId = dlrReference.receiptMessageId(); + success = dlrReference.handler().sendPduRequest((PduRequest) pdu); + } else { + Object[] arr = (Object[]) pdu.getReferenceObject(); + SmppServerSessionHandler handler = (SmppServerSessionHandler) arr[0]; + msg = (M) arr[1]; + deliverMsgId = arr.length > 2 && arr[2] instanceof String id ? id : null; + success = handler.sendPduRequest((PduRequest) pdu); + } } } catch (Exception e) { success = false; @@ -52,7 +64,11 @@ public void run() { } if (!success) { - worker.outTaskFailed(pdu, msg); + if (dlrReference != null) { + dlrReference.batch().fail("send_failed"); + } else { + worker.outTaskFailed(pdu, msg); + } } else if (!pdu.isResponse() && msg != null) { if (MessageTrace.shouldLog(worker.getConfigurationProvider(), MessageTrace.EVENT_DELIVER_SENT)) { logger.info("message.deliver.sent worker={} deliverMsgId={} {}", worker.getFullName(), diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java index 6a7116c..4d0ba09 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrMessageStorage.java @@ -12,15 +12,23 @@ default void saveInitialStates(List states) { void linkProviderMessageId(String gatewayMessageId, String providerName, String providerMessageId); - /** - * Resolves and removes one provider correlation and its tracked message. - * The returned state must contain the supplied status, provider name, linked provider message ID, and an updated - * timestamp. - */ - Optional resolveAndRemoveDlr(String providerName, String providerMessageId, - MessageState.MessageStatus status); + Optional resolveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status, int dlrState, String errorCode); Optional getState(String gatewayMsgId); - boolean markAsFailed(String gatewayMsgId); + List listPendingSmppDeliveries(String systemId); + + List listDueHttpDeliveries(int limit); + + Optional startDeliveryAttempt(String gatewayMsgId, + MessageState.DeliveryChannel expectedChannel); + + boolean completeDelivery(String gatewayMsgId, int expectedAttempt); + + boolean retryDelivery(String gatewayMsgId, int expectedAttempt, String result, long nextAttemptAt); + + boolean failDelivery(String gatewayMsgId, int expectedAttempt, String result); + + boolean failInvalidDelivery(String gatewayMsgId, String result); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java index 4c9fd68..288de8d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrService.java @@ -14,9 +14,6 @@ public class DlrService { @Inject DlrStorage storage; - @Inject - ForwardDlrService forwardDlrService; - public void saveInitialState(MessageState state) { storage.saveInitialState(state); } @@ -29,47 +26,57 @@ public void linkProviderMessageId(String gatewayMessageId, String providerName, storage.linkProviderMessageId(gatewayMessageId, providerName, providerMessageId); } - public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, int dlrState) { - Optional state = storage.resolveAndRemoveDlr( - providerName, providerMessageId, mapDlrState(dlrState)); - state.filter(messageState -> messageState.getForwardDlrUrl() != null) - .filter(messageState -> !messageState.getForwardDlrUrl().isEmpty()) - .ifPresent(forwardDlrService::forwardDlr); - return state; + public Optional resolveDlr(String providerName, String providerMessageId, int dlrState, + String errorCode) { + if (!isTerminalDlrState(dlrState)) { + return Optional.empty(); + } + return storage.resolveDlr( + providerName, providerMessageId, mapDlrState(dlrState), dlrState, errorCode); + } + + public static boolean isTerminalDlrState(int dlrState) { + return dlrState != StandardMessage.DLR_STAT_ACCEPTD && + dlrState != StandardMessage.DLR_STAT_BUFFRED; } public Optional getState(String gatewayMsgId) { return storage.getState(gatewayMsgId); } - public boolean markAsFailed(String gatewayMsgId) { - return storage.markAsFailed(gatewayMsgId); + public List listPendingSmppDeliveries(String systemId) { + return storage.listPendingSmppDeliveries(systemId); + } + + public List listDueHttpDeliveries(int limit) { + return storage.listDueHttpDeliveries(limit); } - public boolean saveUnpushedDlr(StandardMessage message) { - return storage.saveUnpushedDlr(message); + public Optional startDeliveryAttempt(String gatewayMsgId, + MessageState.DeliveryChannel expectedChannel) { + return storage.startDeliveryAttempt(gatewayMsgId, expectedChannel); } - public List getUnpushedDlrs(String systemId) { - return storage.getUnpushedDlrs(systemId); + public boolean completeDelivery(String gatewayMsgId, int expectedAttempt) { + return storage.completeDelivery(gatewayMsgId, expectedAttempt); } - public List claimUnpushedDlrs(String systemId) { - return storage.claimUnpushedDlrs(systemId); + public boolean retryDelivery(String gatewayMsgId, int expectedAttempt, String result, long nextAttemptAt) { + return storage.retryDelivery(gatewayMsgId, expectedAttempt, result, nextAttemptAt); } - public boolean removeUnpushedDlr(StandardMessage message) { - return storage.removeUnpushedDlr(message); + public boolean failDelivery(String gatewayMsgId, int expectedAttempt, String result) { + return storage.failDelivery(gatewayMsgId, expectedAttempt, result); } - public void releaseUnpushedDlrClaim(StandardMessage message) { - storage.releaseUnpushedDlrClaim(message); + public boolean failInvalidDelivery(String gatewayMsgId, String result) { + return storage.failInvalidDelivery(gatewayMsgId, result); } private MessageState.MessageStatus mapDlrState(int dlrState) { return switch (dlrState) { - case 1, 15 -> MessageState.MessageStatus.DELIVERED; - case 5, 9 -> MessageState.MessageStatus.ACCEPTED; + case StandardMessage.DLR_STAT_DELIVRD, StandardMessage.DLR_STAT_SEEN -> + MessageState.MessageStatus.DELIVERED; default -> MessageState.MessageStatus.FAILED; }; } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java index 7378952..a5b4c05 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/DlrStorage.java @@ -1,25 +1,7 @@ package gr.cytech.sendium.core.worker; -import gr.cytech.sendium.core.message.StandardMessage; - -import java.util.List; - /** - * Persistence boundary for delivery-receipt correlation and downstream SMPP replay state. + * Persistence boundary for delivery-receipt correlation and delivery state. */ public interface DlrStorage extends DlrMessageStorage { - boolean saveUnpushedDlr(StandardMessage message); - - List getUnpushedDlrs(String systemId); - - /** - * Claims replayable receipts within this storage instance. V1 targets one Sendium process and does not promise - * distributed claim coordination across multiple gateway replicas. - */ - List claimUnpushedDlrs(String systemId); - - boolean removeUnpushedDlr(StandardMessage message); - - void releaseUnpushedDlrClaim(StandardMessage message); - } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java index 29bb5fb..dc378f1 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java @@ -1,7 +1,12 @@ package gr.cytech.sendium.core.worker; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.quarkus.arc.properties.IfBuildProperty; +import io.quarkus.scheduler.Scheduled; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,14 +15,23 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Optional; @ApplicationScoped @IfBuildProperty(name = "sendium.dlr.persistence.enabled", stringValue = "true", enableIfMissing = false) public class ForwardDlrService { private static final Logger logger = LoggerFactory.getLogger(ForwardDlrService.class); - private static final int MAX_RETRIES = 10; + private static final String ATTEMPT_METRIC = "sendium.dlr.delivery.attempt"; + private static final String TERMINAL_FAILURE_METRIC = "sendium.dlr.delivery.terminal.failure"; + private static final String DISPATCH_ERROR_METRIC = "sendium.dlr.delivery.dispatch.error"; + private static final String CHANNEL_HTTP = "http"; + private static final int DUE_BATCH_SIZE = 100; + private static final int MAX_ATTEMPTS = 10; private static final long RETRY_INTERVAL_MS = 120_000; private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(5); @@ -29,93 +43,201 @@ public class ForwardDlrService { private static final String DLR_TYPE_PLACEHOLDER = "%d"; private static final String MSG_ID_PLACEHOLDER = "%s"; + private final DlrService dlrService; + private final MeterRegistry meterRegistry; private final HttpClient httpClient; - public ForwardDlrService() { - this.httpClient = HttpClient.newBuilder() + @Inject + public ForwardDlrService(DlrService dlrService, MeterRegistry meterRegistry) { + this(dlrService, meterRegistry, newHttpClient()); + } + + ForwardDlrService(DlrService dlrService, MeterRegistry meterRegistry, HttpClient httpClient) { + this.dlrService = dlrService; + this.meterRegistry = meterRegistry; + this.httpClient = httpClient; + } + + static HttpClient newHttpClient() { + return HttpClient.newBuilder() .connectTimeout(REQUEST_TIMEOUT) - .followRedirects(HttpClient.Redirect.NORMAL) + .followRedirects(HttpClient.Redirect.NEVER) .build(); } - public void forwardDlr(MessageState state) { - String forwardUrl = state.getForwardDlrUrl(); - if (forwardUrl == null || forwardUrl.isBlank()) { + @Scheduled(every = "1s", concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void dispatchDueDeliveries() { + List dueDeliveries; + try { + dueDeliveries = dlrService.listDueHttpDeliveries(DUE_BATCH_SIZE); + } catch (RuntimeException e) { + recordDispatchError("scheduler"); + logger.error("Unable to list due HTTP DLR deliveries"); return; } - int kannelType = mapToKannelType(state.getStatus()); - - Thread.startVirtualThread(() -> { + for (MessageState state : dueDeliveries) { try { - String finalUrl = buildForwardUrl(forwardUrl, state.getGatewayMsgId(), kannelType); - doForward(finalUrl, state.getGatewayMsgId(), 1); - } catch (Exception e) { - logger.error("Failed to initialize DLR forwarding for gatewayMsgId: {}", state.getGatewayMsgId(), e); + dispatch(state); + } catch (RuntimeException e) { + recordDispatchError("scheduler"); + logger.error("Unexpected HTTP DLR dispatch failure for gatewayMsgId={}", state.getGatewayMsgId()); } - }); + } } - int mapToKannelType(MessageState.MessageStatus status) { - if (status == null) { - return DLR_BUFFERED; + private void dispatch(MessageState dueState) { + String gatewayMsgId = dueState.getGatewayMsgId(); + HttpRequest request; + try { + request = buildRequest(dueState); + } catch (RuntimeException e) { + failInvalidDelivery(gatewayMsgId); + return; + } + + Optional started; + try { + started = dlrService.startDeliveryAttempt(gatewayMsgId, MessageState.DeliveryChannel.HTTP); + } catch (RuntimeException e) { + recordStorageError(gatewayMsgId, 0, "start"); + return; + } + if (started.isEmpty()) { + return; + } + + int attempt = started.orElseThrow().getDeliveryAttemptCount(); + Timer.Sample sample = Timer.start(meterRegistry); + try { + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() >= 200 && response.statusCode() < 400) { + sample.stop(attemptTimer("success")); + completeDelivery(gatewayMsgId, attempt); + } else { + sample.stop(attemptTimer("http_failure")); + handleAttemptFailure(gatewayMsgId, attempt, "http_failure"); + } + } catch (HttpTimeoutException e) { + sample.stop(attemptTimer("timeout")); + handleAttemptFailure(gatewayMsgId, attempt, "timeout"); + } catch (InterruptedException e) { + sample.stop(attemptTimer("transport_failure")); + handleAttemptFailure(gatewayMsgId, attempt, "interrupted"); + Thread.currentThread().interrupt(); + } catch (IOException | RuntimeException e) { + sample.stop(attemptTimer("transport_failure")); + handleAttemptFailure(gatewayMsgId, attempt, "transport_failure"); } - return switch (status) { - case ACCEPTED -> DLR_BUFFERED; - case SENT -> DLR_SMSC_SUBMIT; - case DELIVERED -> DLR_DELIVERED; - case FAILED -> DLR_FAILED; - }; } - String buildForwardUrl(String baseUrl, String msgId, int kannelType) { - String result = baseUrl.replace(DLR_TYPE_PLACEHOLDER, String.valueOf(kannelType)); - result = result.replace(MSG_ID_PLACEHOLDER, msgId != null ? msgId : ""); - return result; + private HttpRequest buildRequest(MessageState state) { + String callbackTemplate = state.getForwardDlrUrl(); + if (callbackTemplate == null || callbackTemplate.isBlank()) { + throw new IllegalArgumentException("Missing callback URI"); + } + String forwardUrl = buildForwardUrl( + callbackTemplate, state.getGatewayMsgId(), mapToKannelType(state.getStatus())); + URI uri = URI.create(forwardUrl); + String scheme = uri.getScheme(); + String normalizedScheme = scheme == null ? "" : scheme.toLowerCase(Locale.ROOT); + if (uri.getHost() == null || !(normalizedScheme.equals("http") || normalizedScheme.equals("https"))) { + throw new IllegalArgumentException("Callback URI must use HTTP or HTTPS"); + } + return HttpRequest.newBuilder() + .uri(uri) + .timeout(REQUEST_TIMEOUT) + .GET() + .build(); } - private void doForward(String url, String gatewayMsgId, int attempt) { + private void completeDelivery(String gatewayMsgId, int attempt) { try { - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(REQUEST_TIMEOUT) - .GET() - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); - int statusCode = response.statusCode(); + if (!dlrService.completeDelivery(gatewayMsgId, attempt)) { + recordStorageError(gatewayMsgId, attempt, "complete"); + } + } catch (RuntimeException e) { + recordStorageError(gatewayMsgId, attempt, "complete"); + } + } - if (statusCode >= 200 && statusCode < 400) { - logger.info("DLR forwarded successfully for gatewayMsgId: {}", gatewayMsgId); + private void handleAttemptFailure(String gatewayMsgId, int attempt, String result) { + logger.warn("HTTP DLR delivery attempt failed for gatewayMsgId={} attempt={} outcome={}", + gatewayMsgId, attempt, result); + try { + boolean updated; + if (attempt < MAX_ATTEMPTS) { + updated = dlrService.retryDelivery( + gatewayMsgId, attempt, result, System.currentTimeMillis() + RETRY_INTERVAL_MS); } else { - handleFailure(url, gatewayMsgId, attempt, "HTTP " + statusCode); + updated = dlrService.failDelivery(gatewayMsgId, attempt, result); + if (updated) { + terminalFailureCounter("max_attempts").increment(); + } } - } catch (IOException | InterruptedException | RuntimeException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); + if (!updated) { + recordStorageError(gatewayMsgId, attempt, "finish"); } - handleFailure(url, gatewayMsgId, attempt, e.getClass().getSimpleName() + ": " + e.getMessage()); + } catch (RuntimeException e) { + recordStorageError(gatewayMsgId, attempt, "finish"); } } - private void handleFailure(String url, String gatewayMsgId, int attempt, String error) { - if (attempt >= MAX_RETRIES) { - logger.error("DLR forward failed completely after {} retries for gatewayMsgId: {}. Last error: {}", - MAX_RETRIES, gatewayMsgId, error); - } else { - logger.warn("DLR forward attempt {} failed for gatewayMsgId: {}. Error: {}. Scheduling retry.", - attempt, gatewayMsgId, error); - scheduleRetry(url, gatewayMsgId, attempt); + private void failInvalidDelivery(String gatewayMsgId) { + logger.warn("Invalid HTTP DLR callback for gatewayMsgId={}", gatewayMsgId); + try { + if (dlrService.failInvalidDelivery(gatewayMsgId, "invalid_uri")) { + terminalFailureCounter("invalid_uri").increment(); + } else { + recordStorageError(gatewayMsgId, 0, "invalid"); + } + } catch (RuntimeException e) { + recordStorageError(gatewayMsgId, 0, "invalid"); } } - private void scheduleRetry(String url, String gatewayMsgId, int attempt) { - try { - Thread.sleep(RETRY_INTERVAL_MS); - doForward(url, gatewayMsgId, attempt + 1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("Retry sleep interrupted for gatewayMsgId: {}", gatewayMsgId); + private void recordStorageError(String gatewayMsgId, int attempt, String operation) { + recordDispatchError("storage"); + logger.error("HTTP DLR storage update failed for gatewayMsgId={} attempt={} operation={}", + gatewayMsgId, attempt, operation); + } + + private void recordDispatchError(String source) { + Counter.builder(DISPATCH_ERROR_METRIC) + .description("Sendium DLR dispatcher errors") + .tags("channel", CHANNEL_HTTP, "source", source) + .register(meterRegistry) + .increment(); + } + + private Timer attemptTimer(String outcome) { + return Timer.builder(ATTEMPT_METRIC) + .description("Sendium DLR delivery attempt latency") + .tags("channel", CHANNEL_HTTP, "outcome", outcome) + .register(meterRegistry); + } + + private Counter terminalFailureCounter(String reason) { + return Counter.builder(TERMINAL_FAILURE_METRIC) + .description("Sendium terminal DLR delivery failures") + .tags("channel", CHANNEL_HTTP, "reason", reason) + .register(meterRegistry); + } + + int mapToKannelType(MessageState.MessageStatus status) { + if (status == null) { + return DLR_BUFFERED; } + return switch (status) { + case ACCEPTED -> DLR_BUFFERED; + case SENT -> DLR_SMSC_SUBMIT; + case DELIVERED -> DLR_DELIVERED; + case FAILED -> DLR_FAILED; + }; + } + + String buildForwardUrl(String baseUrl, String msgId, int kannelType) { + String result = baseUrl.replace(DLR_TYPE_PLACEHOLDER, String.valueOf(kannelType)); + return result.replace(MSG_ID_PLACEHOLDER, msgId != null ? msgId : ""); } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java index f1dfc75..1c367dc 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ManagedDlrStorage.java @@ -1,6 +1,5 @@ package gr.cytech.sendium.core.worker; -import gr.cytech.sendium.core.message.StandardMessage; import io.agroal.api.AgroalDataSource; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; @@ -31,7 +30,7 @@ public class ManagedDlrStorage implements DlrStorage { private static final String BACKEND = "postgresql"; private static final String POSTGRESQL_PROBE_SQL = """ SELECT 1 - FROM sendium_dlr.tracked_message + FROM sendium_dlr.dlr_message WHERE FALSE """; @@ -103,9 +102,10 @@ public void linkProviderMessageId(String gatewayMessageId, String providerName, } @Override - public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, - MessageState.MessageStatus status) { - return timed("resolve", () -> delegate.resolveAndRemoveDlr(providerName, providerMessageId, status)); + public Optional resolveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status, int dlrState, String errorCode) { + return timed("resolve", () -> delegate.resolveDlr( + providerName, providerMessageId, status, dlrState, errorCode)); } @Override @@ -114,33 +114,40 @@ public Optional getState(String gatewayMsgId) { } @Override - public boolean markAsFailed(String gatewayMsgId) { - return timed("mark_failed", () -> delegate.markAsFailed(gatewayMsgId)); + public List listPendingSmppDeliveries(String systemId) { + return timed("list_pending_smpp", () -> delegate.listPendingSmppDeliveries(systemId)); } @Override - public boolean saveUnpushedDlr(StandardMessage message) { - return timed("save_unpushed", () -> delegate.saveUnpushedDlr(message)); + public List listDueHttpDeliveries(int limit) { + return timed("list_due_http", () -> delegate.listDueHttpDeliveries(limit)); } @Override - public List getUnpushedDlrs(String systemId) { - return timed("get_unpushed", () -> delegate.getUnpushedDlrs(systemId)); + public Optional startDeliveryAttempt(String gatewayMsgId, + MessageState.DeliveryChannel expectedChannel) { + return timed("start_delivery", () -> delegate.startDeliveryAttempt(gatewayMsgId, expectedChannel)); } @Override - public List claimUnpushedDlrs(String systemId) { - return timed("claim_unpushed", () -> delegate.claimUnpushedDlrs(systemId)); + public boolean completeDelivery(String gatewayMsgId, int expectedAttempt) { + return timed("complete_delivery", () -> delegate.completeDelivery(gatewayMsgId, expectedAttempt)); } @Override - public boolean removeUnpushedDlr(StandardMessage message) { - return timed("remove_unpushed", () -> delegate.removeUnpushedDlr(message)); + public boolean retryDelivery(String gatewayMsgId, int expectedAttempt, String result, long nextAttemptAt) { + return timed("retry_delivery", () -> delegate.retryDelivery( + gatewayMsgId, expectedAttempt, result, nextAttemptAt)); } @Override - public void releaseUnpushedDlrClaim(StandardMessage message) { - timed("release_claim", () -> delegate.releaseUnpushedDlrClaim(message)); + public boolean failDelivery(String gatewayMsgId, int expectedAttempt, String result) { + return timed("fail_delivery", () -> delegate.failDelivery(gatewayMsgId, expectedAttempt, result)); + } + + @Override + public boolean failInvalidDelivery(String gatewayMsgId, String result) { + return timed("fail_invalid_delivery", () -> delegate.failInvalidDelivery(gatewayMsgId, result)); } private T timed(String operation, Supplier action) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java index 180409a..e241bc3 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/MessageState.java @@ -20,6 +20,15 @@ public class MessageState implements Serializable { private String forwardDlrUrl; private List reassembledParts; private MessageStatus status; + private Integer dlrState; + private String errorCode; + private DeliveryChannel deliveryChannel = DeliveryChannel.NONE; + private DeliveryStatus deliveryStatus = DeliveryStatus.WAITING_PROVIDER; + private int deliveryAttemptCount; + private Long lastAttemptAt; + private Long nextAttemptAt; + private String lastDeliveryResult; + private Long resolvedAt; private long timestamp; public MessageState() { @@ -38,6 +47,7 @@ public MessageState(String gatewayMsgId, String accountId, String systemId, Stri this.providerName = null; this.providerMessageId = null; this.status = MessageStatus.ACCEPTED; + this.deliveryAttemptCount = 0; this.timestamp = System.currentTimeMillis(); this.forwardDlrUrl = forwardDlrUrl; } @@ -78,6 +88,42 @@ public MessageStatus getStatus() { return status; } + public Integer getDlrState() { + return dlrState; + } + + public String getErrorCode() { + return errorCode; + } + + public DeliveryChannel getDeliveryChannel() { + return deliveryChannel; + } + + public DeliveryStatus getDeliveryStatus() { + return deliveryStatus; + } + + public int getDeliveryAttemptCount() { + return deliveryAttemptCount; + } + + public Long getLastAttemptAt() { + return lastAttemptAt; + } + + public Long getNextAttemptAt() { + return nextAttemptAt; + } + + public String getLastDeliveryResult() { + return lastDeliveryResult; + } + + public Long getResolvedAt() { + return resolvedAt; + } + public List getReassembledParts() { return reassembledParts == null ? null : new ArrayList<>(reassembledParts); } @@ -98,6 +144,42 @@ public void setStatus(MessageStatus status) { this.status = status; } + public void setDlrState(Integer dlrState) { + this.dlrState = dlrState; + } + + public void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + public void setDeliveryChannel(DeliveryChannel deliveryChannel) { + this.deliveryChannel = deliveryChannel; + } + + public void setDeliveryStatus(DeliveryStatus deliveryStatus) { + this.deliveryStatus = deliveryStatus; + } + + public void setDeliveryAttemptCount(int deliveryAttemptCount) { + this.deliveryAttemptCount = deliveryAttemptCount; + } + + public void setLastAttemptAt(Long lastAttemptAt) { + this.lastAttemptAt = lastAttemptAt; + } + + public void setNextAttemptAt(Long nextAttemptAt) { + this.nextAttemptAt = nextAttemptAt; + } + + public void setLastDeliveryResult(String lastDeliveryResult) { + this.lastDeliveryResult = lastDeliveryResult; + } + + public void setResolvedAt(Long resolvedAt) { + this.resolvedAt = resolvedAt; + } + public void setReassembledParts(List reassembledParts) { this.reassembledParts = reassembledParts == null ? null : new ArrayList<>(reassembledParts); } @@ -112,4 +194,16 @@ public enum MessageStatus { DELIVERED, FAILED } + + public enum DeliveryChannel { + NONE, + HTTP, + SMPP + } + + public enum DeliveryStatus { + WAITING_PROVIDER, + PENDING, + FAILED + } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java index befae5d..b18110e 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorage.java @@ -1,6 +1,5 @@ package gr.cytech.sendium.core.worker; -import gr.cytech.sendium.core.message.StandardMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,9 +16,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,12 +32,23 @@ public class PostgresqlDlrStorage implements DlrStorage { private static final int DEFAULT_LINK_MAX_ATTEMPTS = 20; private static final long DEFAULT_LINK_RETRY_INTERVAL_MILLIS = 200; private static final long EXPIRY_CHECK_INTERVAL_MILLIS = TimeUnit.HOURS.toMillis(1); + private static final int MAX_DELIVERY_BATCH_SIZE = 1_000; + private static final int STARTING_ATTEMPT = -1; + + private static final String STATE_COLUMNS = """ + gateway_message_id, account_id, system_id, source_address, destination_address, + provider_name, provider_message_id, forward_dlr_url, reassembled_parts, provider_status, + dlr_state, error_code, delivery_channel, delivery_status, delivery_attempt_count, + last_attempt_at, next_attempt_at, last_delivery_result, resolved_at, updated_at + """; private static final String SAVE_INITIAL_STATE_SQL = """ - INSERT INTO sendium_dlr.tracked_message + INSERT INTO sendium_dlr.dlr_message (gateway_message_id, account_id, system_id, source_address, destination_address, - provider_name, provider_message_id, forward_dlr_url, reassembled_parts, status, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + provider_name, provider_message_id, forward_dlr_url, reassembled_parts, provider_status, + dlr_state, error_code, delivery_channel, delivery_status, delivery_attempt_count, + last_attempt_at, next_attempt_at, last_delivery_result, resolved_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (gateway_message_id) DO UPDATE SET account_id = EXCLUDED.account_id, system_id = EXCLUDED.system_id, @@ -51,20 +58,31 @@ ON CONFLICT (gateway_message_id) DO UPDATE SET provider_message_id = EXCLUDED.provider_message_id, forward_dlr_url = EXCLUDED.forward_dlr_url, reassembled_parts = EXCLUDED.reassembled_parts, - status = EXCLUDED.status, + provider_status = EXCLUDED.provider_status, + dlr_state = EXCLUDED.dlr_state, + error_code = EXCLUDED.error_code, + delivery_channel = EXCLUDED.delivery_channel, + delivery_status = EXCLUDED.delivery_status, + delivery_attempt_count = EXCLUDED.delivery_attempt_count, + last_attempt_at = EXCLUDED.last_attempt_at, + next_attempt_at = EXCLUDED.next_attempt_at, + last_delivery_result = EXCLUDED.last_delivery_result, + resolved_at = EXCLUDED.resolved_at, created_at = CURRENT_TIMESTAMP, updated_at = EXCLUDED.updated_at + WHERE dlr_message.delivery_status = 'WAITING_PROVIDER' """; private static final String LINK_MESSAGE_SQL = """ - UPDATE sendium_dlr.tracked_message - SET provider_name = ?, provider_message_id = ?, status = 'SENT', updated_at = CURRENT_TIMESTAMP - WHERE gateway_message_id = ? + UPDATE sendium_dlr.dlr_message + SET provider_name = ?, provider_message_id = ?, provider_status = 'SENT', + updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_status = 'WAITING_PROVIDER' """; private static final String LOCK_MESSAGE_SQL = """ SELECT 1 - FROM sendium_dlr.tracked_message + FROM sendium_dlr.dlr_message WHERE gateway_message_id = ? FOR UPDATE """; @@ -89,10 +107,11 @@ ON CONFLICT (provider_name, provider_message_id) DO UPDATE SET created_at = CURRENT_TIMESTAMP RETURNING gateway_message_id ) - UPDATE sendium_dlr.tracked_message + UPDATE sendium_dlr.dlr_message SET provider_name = NULL, provider_message_id = NULL, updated_at = CURRENT_TIMESTAMP WHERE provider_name = ? AND provider_message_id = ? AND gateway_message_id <> ? + AND delivery_status = 'WAITING_PROVIDER' AND EXISTS (SELECT 1 FROM saved_correlation) """; @@ -102,35 +121,79 @@ AND EXISTS (SELECT 1 FROM saved_correlation) """; private static final String GET_STATE_SQL = """ - SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, - tm.destination_address, tm.provider_name, tm.provider_message_id, tm.forward_dlr_url, - tm.reassembled_parts, tm.status, tm.updated_at - FROM sendium_dlr.tracked_message tm - WHERE tm.gateway_message_id = ? - """; + SELECT %s + FROM sendium_dlr.dlr_message + WHERE gateway_message_id = ? + """.formatted(STATE_COLUMNS); private static final String RESOLVE_STATE_SQL = """ - SELECT tm.gateway_message_id, tm.account_id, tm.system_id, tm.source_address, - tm.destination_address, tm.forward_dlr_url, tm.reassembled_parts, - correlation.provider_name, correlation.provider_message_id, - CURRENT_TIMESTAMP AS resolved_at - FROM sendium_dlr.provider_correlation correlation - JOIN sendium_dlr.tracked_message tm - ON tm.gateway_message_id = correlation.gateway_message_id - WHERE correlation.provider_name = ? AND correlation.provider_message_id = ? - AND correlation.gateway_message_id = ? - FOR UPDATE OF correlation - """; + UPDATE sendium_dlr.dlr_message + SET provider_name = ?, + provider_message_id = ?, + provider_status = ?, + dlr_state = ?, + error_code = ?, + delivery_status = CASE WHEN delivery_channel = 'NONE' THEN delivery_status ELSE 'PENDING' END, + next_attempt_at = CASE WHEN delivery_channel = 'HTTP' THEN CURRENT_TIMESTAMP ELSE NULL END, + resolved_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_status = 'WAITING_PROVIDER' + RETURNING %s + """.formatted(STATE_COLUMNS); private static final String DELETE_STATE_SQL = """ - DELETE FROM sendium_dlr.tracked_message + DELETE FROM sendium_dlr.dlr_message WHERE gateway_message_id = ? """; - private static final String MARK_FAILED_SQL = """ - UPDATE sendium_dlr.tracked_message - SET status = 'FAILED', updated_at = CURRENT_TIMESTAMP - WHERE gateway_message_id = ? + private static final String LIST_PENDING_SMPP_SQL = """ + SELECT %s + FROM sendium_dlr.dlr_message + WHERE system_id = ? AND delivery_channel = 'SMPP' AND delivery_status = 'PENDING' + ORDER BY resolved_at, created_at, gateway_message_id + """.formatted(STATE_COLUMNS); + + private static final String LIST_DUE_HTTP_SQL = """ + SELECT %s + FROM sendium_dlr.dlr_message + WHERE delivery_channel = 'HTTP' AND delivery_status = 'PENDING' + AND next_attempt_at <= CURRENT_TIMESTAMP + ORDER BY next_attempt_at, gateway_message_id + LIMIT ? + """.formatted(STATE_COLUMNS); + + private static final String START_DELIVERY_SQL = """ + UPDATE sendium_dlr.dlr_message + SET delivery_attempt_count = delivery_attempt_count + 1, + last_attempt_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_channel = ? AND delivery_status = 'PENDING' + RETURNING %s + """.formatted(STATE_COLUMNS); + + private static final String COMPLETE_DELIVERY_SQL = """ + DELETE FROM sendium_dlr.dlr_message + WHERE gateway_message_id = ? AND delivery_status = 'PENDING' AND delivery_attempt_count = ? + """; + + private static final String RETRY_DELIVERY_SQL = """ + UPDATE sendium_dlr.dlr_message + SET last_delivery_result = ?, next_attempt_at = ?, updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_status = 'PENDING' AND delivery_attempt_count = ? + """; + + private static final String FAIL_DELIVERY_SQL = """ + UPDATE sendium_dlr.dlr_message + SET delivery_status = 'FAILED', last_delivery_result = ?, next_attempt_at = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_status = 'PENDING' AND delivery_attempt_count = ? + """; + + private static final String FAIL_INVALID_DELIVERY_SQL = """ + UPDATE sendium_dlr.dlr_message + SET delivery_status = 'FAILED', last_delivery_result = ?, next_attempt_at = NULL, + resolved_at = COALESCE(resolved_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP + WHERE gateway_message_id = ? AND delivery_status = 'PENDING' """; private static final String DELETE_EXPIRED_CORRELATIONS_SQL = """ @@ -141,63 +204,24 @@ AND EXISTS (SELECT 1 FROM saved_correlation) private static final String DELETE_EXPIRED_MESSAGES_SQL = """ WITH expired_messages AS ( SELECT gateway_message_id - FROM sendium_dlr.tracked_message - WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' + FROM sendium_dlr.dlr_message + WHERE (delivery_status = 'WAITING_PROVIDER' + AND created_at < CURRENT_TIMESTAMP - INTERVAL '7 days') + OR (delivery_status IN ('PENDING', 'FAILED') + AND resolved_at < CURRENT_TIMESTAMP - INTERVAL '7 days') ORDER BY gateway_message_id FOR UPDATE ) - DELETE FROM sendium_dlr.tracked_message message + DELETE FROM sendium_dlr.dlr_message message USING expired_messages expired WHERE message.gateway_message_id = expired.gateway_message_id """; - private static final String SAVE_UNPUSHED_DLR_SQL = """ - INSERT INTO sendium_dlr.unpushed_dlr - (dlr_key, system_id, account_id, source_address, destination_address, serial, - message_id, dlr_state, error_code, acked, priority, reassembled_parts, generation_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (dlr_key) DO UPDATE SET - system_id = EXCLUDED.system_id, - account_id = EXCLUDED.account_id, - source_address = EXCLUDED.source_address, - destination_address = EXCLUDED.destination_address, - serial = EXCLUDED.serial, - message_id = EXCLUDED.message_id, - dlr_state = EXCLUDED.dlr_state, - error_code = EXCLUDED.error_code, - acked = EXCLUDED.acked, - priority = EXCLUDED.priority, - reassembled_parts = EXCLUDED.reassembled_parts, - generation_id = EXCLUDED.generation_id, - created_at = CURRENT_TIMESTAMP - """; - - private static final String GET_UNPUSHED_DLRS_SQL = """ - SELECT dlr_key, system_id, account_id, source_address, destination_address, serial, - message_id, dlr_state, error_code, acked, priority, reassembled_parts, generation_id - FROM sendium_dlr.unpushed_dlr - WHERE system_id = ? - ORDER BY created_at, dlr_key - """; - - private static final String DELETE_UNPUSHED_DLR_SQL = """ - DELETE FROM sendium_dlr.unpushed_dlr - WHERE dlr_key = ? AND generation_id = ? - """; - - private static final String DELETE_EXPIRED_UNPUSHED_DLRS_SQL = """ - DELETE FROM sendium_dlr.unpushed_dlr - WHERE created_at < CURRENT_TIMESTAMP - INTERVAL '7 days' - RETURNING dlr_key, generation_id - """; - private final DataSource dataSource; private final int linkMaxAttempts; private final long linkRetryIntervalMillis; private final long expiryCheckIntervalMillis; - private final Object unpushedDlrStateLock = new Object(); - private final ConcurrentHashMap claimedUnpushedDlrKeys = new ConcurrentHashMap<>(); - private final IdentityHashMap claimedUnpushedDlrGenerations = new IdentityHashMap<>(); + private final ConcurrentHashMap activeDeliveryAttempts = new ConcurrentHashMap<>(); private final AtomicBoolean expiryInProgress = new AtomicBoolean(); private volatile long lastExpiryCheck; @@ -234,74 +258,48 @@ public void saveInitialStates(List states) { return; } - List suppliedStates = states.stream() - .map(state -> Objects.requireNonNull(state, "state")) - .toList(); - suppliedStates.forEach(this::validateCorrelationFields); Map finalStatesByGateway = new LinkedHashMap<>(); - for (MessageState state : suppliedStates) { + for (MessageState state : states) { + Objects.requireNonNull(state, "state"); + validateState(state); UUID gatewayMsgId = parseGatewayId(state.getGatewayMsgId()); - // Reinsert duplicate IDs so batch order reflects each gateway's final occurrence. finalStatesByGateway.remove(gatewayMsgId); finalStatesByGateway.put(gatewayMsgId, state); } - List gatewayMsgIds = List.copyOf(finalStatesByGateway.keySet()); - List checkedStates = List.copyOf(finalStatesByGateway.values()); checkExpiry(); try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); - try (PreparedStatement saveStates = connection.prepareStatement(SAVE_INITIAL_STATE_SQL); - PreparedStatement deleteCorrelations = connection.prepareStatement(DELETE_CORRELATIONS_SQL); - PreparedStatement saveCorrelations = connection.prepareStatement(SAVE_CORRELATION_SQL)) { - List correlatedStates = checkedStates.stream() + try { + List> entries = new ArrayList<>(finalStatesByGateway.entrySet()); + List correlatedStates = entries.stream() + .map(Map.Entry::getValue) .filter(state -> state.getProviderMessageId() != null) .sorted(Comparator.comparing(MessageState::getProviderName) .thenComparing(MessageState::getProviderMessageId)) .toList(); - String previousProviderName = null; - String previousProviderMessageId = null; - for (MessageState state : correlatedStates) { - if (!state.getProviderName().equals(previousProviderName) || - !state.getProviderMessageId().equals(previousProviderMessageId)) { - lockCorrelation(connection, state.getProviderName(), state.getProviderMessageId()); - previousProviderName = state.getProviderName(); - previousProviderMessageId = state.getProviderMessageId(); + lockCorrelations(connection, correlatedStates); + lockInitialMessageOwners(connection, entries, correlatedStates); + + List> saved = new ArrayList<>(); + try (PreparedStatement statement = connection.prepareStatement(SAVE_INITIAL_STATE_SQL)) { + for (Map.Entry entry : entries) { + setStateParameters(connection, statement, entry.getKey(), entry.getValue()); + if (statement.executeUpdate() == 1) { + saved.add(entry); + } } } - List messageIdsToLock = new ArrayList<>(gatewayMsgIds); - for (MessageState state : correlatedStates) { - findCorrelationOwner(connection, state.getProviderName(), state.getProviderMessageId()) - .ifPresent(messageIdsToLock::add); - } - for (UUID messageId : messageIdsToLock.stream() - .distinct() - .sorted(Comparator.comparing(UUID::toString)) - .toList()) { - lockMessage(connection, messageId); + for (Map.Entry entry : saved) { + deleteCorrelations(connection, entry.getKey()); } - int correlationCount = 0; - for (int index = 0; index < checkedStates.size(); index++) { - MessageState state = checkedStates.get(index); - UUID gatewayMsgId = gatewayMsgIds.get(index); - setStateParameters(connection, saveStates, gatewayMsgId, state); - saveStates.addBatch(); - deleteCorrelations.setObject(1, gatewayMsgId); - deleteCorrelations.addBatch(); - if (state.getProviderMessageId() != null && - isLastCorrelationOwner(checkedStates, index, state)) { - setCorrelationParameters(saveCorrelations, state.getProviderName(), - state.getProviderMessageId(), gatewayMsgId); - saveCorrelations.addBatch(); - correlationCount++; + for (int index = 0; index < saved.size(); index++) { + MessageState state = saved.get(index).getValue(); + if (state.getProviderMessageId() != null && isLastCorrelationOwner(saved, index, state)) { + saveCorrelation(connection, state.getProviderName(), state.getProviderMessageId(), + saved.get(index).getKey()); } } - - saveStates.executeBatch(); - deleteCorrelations.executeBatch(); - if (correlationCount > 0) { - saveCorrelations.executeBatch(); - } connection.commit(); } catch (SQLException e) { rollback(connection, e); @@ -330,8 +328,8 @@ public void linkProviderMessageId(String gatewayMessageId, String providerName, } @Override - public Optional resolveAndRemoveDlr(String providerName, String providerMessageId, - MessageState.MessageStatus status) { + public Optional resolveDlr(String providerName, String providerMessageId, + MessageState.MessageStatus status, int dlrState, String errorCode) { Objects.requireNonNull(status, "status"); requireCorrelation(providerName, providerMessageId); checkExpiry(); @@ -346,13 +344,16 @@ public Optional resolveAndRemoveDlr(String providerName, String pr connection.rollback(); return Optional.empty(); } - Optional state = lockResolvedState( - connection, providerName, providerMessageId, gatewayMessageId.get(), status); + Optional state = resolveState(connection, gatewayMessageId.get(), providerName, + providerMessageId, status, dlrState, errorCode); if (state.isEmpty()) { connection.rollback(); return Optional.empty(); } - deleteState(connection, parseGatewayId(state.get().getGatewayMsgId())); + deleteCorrelations(connection, gatewayMessageId.get()); + if (state.get().getDeliveryChannel() == MessageState.DeliveryChannel.NONE) { + deleteState(connection, gatewayMessageId.get()); + } connection.commit(); return state; } catch (SQLException e) { @@ -367,12 +368,11 @@ public Optional resolveAndRemoveDlr(String providerName, String pr @Override public Optional getState(String gatewayMsgId) { checkExpiry(); - try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(GET_STATE_SQL)) { statement.setObject(1, parseGatewayId(gatewayMsgId)); try (ResultSet resultSet = statement.executeQuery()) { - return resultSet.next() ? Optional.of(readTrackedState(resultSet)) : Optional.empty(); + return resultSet.next() ? Optional.of(readState(resultSet)) : Optional.empty(); } } catch (SQLException e) { throw failure("read DLR state", e); @@ -380,160 +380,159 @@ public Optional getState(String gatewayMsgId) { } @Override - public boolean markAsFailed(String gatewayMsgId) { + public List listPendingSmppDeliveries(String systemId) { checkExpiry(); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(MARK_FAILED_SQL)) { - statement.setObject(1, parseGatewayId(gatewayMsgId)); - return statement.executeUpdate() == 1; - } catch (SQLException e) { - throw failure("mark DLR state as failed", e); + if (systemId == null || systemId.isBlank()) { + return List.of(); } + return listStates(LIST_PENDING_SMPP_SQL, statement -> statement.setString(1, systemId), + "list pending SMPP deliveries"); } @Override - public boolean saveUnpushedDlr(StandardMessage message) { + public List listDueHttpDeliveries(int limit) { checkExpiry(); - if (!isValidUnpushedDlr(message)) { - return false; + if (limit < 1) { + throw new IllegalArgumentException("Delivery limit must be positive"); + } + int boundedLimit = Math.min(limit, MAX_DELIVERY_BATCH_SIZE); + return listStates(LIST_DUE_HTTP_SQL, statement -> statement.setInt(1, boundedLimit), + "list due HTTP deliveries"); + } + + @Override + public Optional startDeliveryAttempt(String gatewayMsgId, + MessageState.DeliveryChannel expectedChannel) { + Objects.requireNonNull(expectedChannel, "expectedChannel"); + if (expectedChannel == MessageState.DeliveryChannel.NONE) { + throw new IllegalArgumentException("A delivery attempt requires HTTP or SMPP channel"); + } + UUID gatewayId = parseGatewayId(gatewayMsgId); + if (activeDeliveryAttempts.putIfAbsent(gatewayId, STARTING_ATTEMPT) != null) { + return Optional.empty(); } - UnpushedDlr dlr = UnpushedDlr.fromMessage(message); try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(SAVE_UNPUSHED_DLR_SQL)) { - statement.setString(1, getUnpushedDlrKey(message)); - statement.setString(2, dlr.systemId); - statement.setString(3, dlr.accountId); - statement.setString(4, dlr.from); - statement.setString(5, dlr.to); - statement.setString(6, dlr.serial); - statement.setInt(7, dlr.msgId); - statement.setInt(8, dlr.state); - statement.setString(9, dlr.errcode); - statement.setBoolean(10, dlr.acked); - statement.setInt(11, dlr.priority); - setStringArray(connection, statement, 12, dlr.reassembledParts); - statement.setObject(13, UUID.randomUUID()); - return statement.executeUpdate() == 1; + PreparedStatement statement = connection.prepareStatement(START_DELIVERY_SQL)) { + statement.setObject(1, gatewayId); + statement.setString(2, expectedChannel.name()); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + activeDeliveryAttempts.remove(gatewayId, STARTING_ATTEMPT); + return Optional.empty(); + } + MessageState state = readState(resultSet); + activeDeliveryAttempts.replace(gatewayId, STARTING_ATTEMPT, state.getDeliveryAttemptCount()); + return Optional.of(state); + } } catch (SQLException e) { - throw failure("save unpushed DLR", e); + activeDeliveryAttempts.remove(gatewayId, STARTING_ATTEMPT); + throw failure("start DLR delivery attempt", e); } } @Override - public List getUnpushedDlrs(String systemId) { - return loadUnpushedDlrs(systemId, false); + public boolean completeDelivery(String gatewayMsgId, int expectedAttempt) { + return finishAttempt(gatewayMsgId, expectedAttempt, COMPLETE_DELIVERY_SQL, + statement -> { + statement.setObject(1, parseGatewayId(gatewayMsgId)); + statement.setInt(2, expectedAttempt); + }, "complete DLR delivery"); } @Override - public List claimUnpushedDlrs(String systemId) { - return loadUnpushedDlrs(systemId, true); + public boolean retryDelivery(String gatewayMsgId, int expectedAttempt, String result, long nextAttemptAt) { + return finishAttempt(gatewayMsgId, expectedAttempt, RETRY_DELIVERY_SQL, + statement -> { + statement.setString(1, normalizeResult(result)); + statement.setObject(2, toOffsetDateTime(nextAttemptAt)); + statement.setObject(3, parseGatewayId(gatewayMsgId)); + statement.setInt(4, expectedAttempt); + }, "retry DLR delivery"); } @Override - public boolean removeUnpushedDlr(StandardMessage message) { - if (!isValidUnpushedDlr(message)) { - return false; - } - - String key = getUnpushedDlrKey(message); - synchronized (unpushedDlrStateLock) { - UUID generationId = claimedUnpushedDlrGenerations.get(message); - if (generationId == null) { - return false; - } - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(DELETE_UNPUSHED_DLR_SQL)) { - statement.setString(1, key); - statement.setObject(2, generationId); - boolean removed = statement.executeUpdate() == 1; - claimedUnpushedDlrGenerations.remove(message); - claimedUnpushedDlrKeys.remove(key, generationId); - return removed; - } catch (SQLException e) { - throw failure("remove unpushed DLR", e); - } - } + public boolean failDelivery(String gatewayMsgId, int expectedAttempt, String result) { + return finishAttempt(gatewayMsgId, expectedAttempt, FAIL_DELIVERY_SQL, + statement -> { + statement.setString(1, normalizeResult(result)); + statement.setObject(2, parseGatewayId(gatewayMsgId)); + statement.setInt(3, expectedAttempt); + }, "fail DLR delivery"); } @Override - public void releaseUnpushedDlrClaim(StandardMessage message) { - if (!isValidUnpushedDlr(message)) { - return; - } - - synchronized (unpushedDlrStateLock) { - UUID generationId = claimedUnpushedDlrGenerations.remove(message); - if (generationId != null) { - claimedUnpushedDlrKeys.remove(getUnpushedDlrKey(message), generationId); - } + public boolean failInvalidDelivery(String gatewayMsgId, String result) { + checkExpiry(); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(FAIL_INVALID_DELIVERY_SQL)) { + statement.setString(1, normalizeResult(result)); + statement.setObject(2, parseGatewayId(gatewayMsgId)); + return statement.executeUpdate() == 1; + } catch (SQLException e) { + throw failure("mark invalid DLR delivery failed", e); } } - private List loadUnpushedDlrs(String systemId, boolean claimForReplay) { + private boolean finishAttempt(String gatewayMsgId, int expectedAttempt, String sql, + StatementBinder binder, String operation) { + if (expectedAttempt < 1) { + throw new IllegalArgumentException("Expected attempt must be positive"); + } checkExpiry(); - if (systemId == null || systemId.isBlank()) { - return List.of(); + UUID gatewayId = parseGatewayId(gatewayMsgId); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(sql)) { + binder.bind(statement); + return statement.executeUpdate() == 1; + } catch (SQLException e) { + throw failure(operation, e); + } finally { + activeDeliveryAttempts.remove(gatewayId, expectedAttempt); } + } - synchronized (unpushedDlrStateLock) { - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(GET_UNPUSHED_DLRS_SQL)) { - statement.setString(1, systemId); - try (ResultSet resultSet = statement.executeQuery()) { - List messages = new ArrayList<>(); - while (resultSet.next()) { - String key = resultSet.getString("dlr_key"); - UUID generationId = resultSet.getObject("generation_id", UUID.class); - StandardMessage message = readUnpushedDlr(resultSet).toMessage(); - if (!claimForReplay) { - messages.add(message); - } else if (claimedUnpushedDlrKeys.putIfAbsent(key, generationId) == null) { - claimedUnpushedDlrGenerations.put(message, generationId); - messages.add(message); - } - } - return messages; + private List listStates(String sql, StatementBinder binder, String operation) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(sql)) { + binder.bind(statement); + try (ResultSet resultSet = statement.executeQuery()) { + List states = new ArrayList<>(); + while (resultSet.next()) { + states.add(readState(resultSet)); } - } catch (SQLException e) { - throw failure("read unpushed DLRs", e); + return states; } + } catch (SQLException e) { + throw failure(operation, e); } } - private UnpushedDlr readUnpushedDlr(ResultSet resultSet) throws SQLException { - UnpushedDlr dlr = new UnpushedDlr(); - dlr.systemId = resultSet.getString("system_id"); - dlr.accountId = resultSet.getString("account_id"); - dlr.from = resultSet.getString("source_address"); - dlr.to = resultSet.getString("destination_address"); - dlr.serial = resultSet.getString("serial"); - dlr.msgId = resultSet.getInt("message_id"); - dlr.state = resultSet.getInt("dlr_state"); - dlr.errcode = resultSet.getString("error_code"); - dlr.acked = resultSet.getBoolean("acked"); - dlr.priority = resultSet.getInt("priority"); - dlr.reassembledParts = readStringArray(resultSet, "reassembled_parts"); - return dlr; - } - - private boolean isValidUnpushedDlr(StandardMessage message) { - return message != null && message.type == StandardMessage.MSG_DLR && - message.systemId != null && !message.systemId.isBlank(); - } - - private String getUnpushedDlrKey(StandardMessage message) { - return String.join("|", - nullToEmpty(message.systemId), - nullToEmpty(message.serial), - String.valueOf(message.state), - nullToEmpty(message.errcode), - String.valueOf(message.msgId)); + private void lockCorrelations(Connection connection, List correlatedStates) throws SQLException { + String previousProviderName = null; + String previousProviderMessageId = null; + for (MessageState state : correlatedStates) { + if (!state.getProviderName().equals(previousProviderName) || + !state.getProviderMessageId().equals(previousProviderMessageId)) { + lockCorrelation(connection, state.getProviderName(), state.getProviderMessageId()); + previousProviderName = state.getProviderName(); + previousProviderMessageId = state.getProviderMessageId(); + } + } } - private String nullToEmpty(String value) { - return value == null ? "" : value; + private void lockInitialMessageOwners(Connection connection, List> entries, + List correlatedStates) throws SQLException { + List messageIdsToLock = entries.stream().map(Map.Entry::getKey).collect(ArrayList::new, + ArrayList::add, ArrayList::addAll); + for (MessageState state : correlatedStates) { + findCorrelationOwner(connection, state.getProviderName(), state.getProviderMessageId()) + .ifPresent(messageIdsToLock::add); + } + for (UUID messageId : messageIdsToLock.stream().distinct() + .sorted(Comparator.comparing(UUID::toString)).toList()) { + lockMessage(connection, messageId); + } } private boolean tryLinkProviderMessageId(UUID gatewayMessageId, String providerName, String providerMessageId) { @@ -546,10 +545,8 @@ private boolean tryLinkProviderMessageId(UUID gatewayMessageId, String providerN messageIdsToLock.add(gatewayMessageId); previousOwner.ifPresent(messageIdsToLock::add); boolean targetFound = false; - for (UUID messageId : messageIdsToLock.stream() - .distinct() - .sorted(Comparator.comparing(UUID::toString)) - .toList()) { + for (UUID messageId : messageIdsToLock.stream().distinct() + .sorted(Comparator.comparing(UUID::toString)).toList()) { boolean found = lockMessage(connection, messageId); if (messageId.equals(gatewayMessageId)) { targetFound = found; @@ -561,7 +558,8 @@ private boolean tryLinkProviderMessageId(UUID gatewayMessageId, String providerN } saveCorrelation(connection, providerName, providerMessageId, gatewayMessageId); if (!markAsSent(connection, gatewayMessageId, providerName, providerMessageId)) { - throw new SQLException("Gateway message state disappeared while linking provider message ID"); + connection.rollback(); + return false; } connection.commit(); return true; @@ -615,6 +613,50 @@ private boolean markAsSent(Connection connection, UUID gatewayMessageId, } } + private Optional resolveState(Connection connection, UUID gatewayMessageId, + String providerName, String providerMessageId, + MessageState.MessageStatus status, int dlrState, + String errorCode) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(RESOLVE_STATE_SQL)) { + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setString(3, status.name()); + statement.setInt(4, dlrState); + statement.setString(5, errorCode); + statement.setObject(6, gatewayMessageId); + try (ResultSet resultSet = statement.executeQuery()) { + return resultSet.next() ? Optional.of(readState(resultSet)) : Optional.empty(); + } + } + } + + private void deleteCorrelations(Connection connection, UUID gatewayMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(DELETE_CORRELATIONS_SQL)) { + statement.setObject(1, gatewayMessageId); + statement.executeUpdate(); + } + } + + private void deleteState(Connection connection, UUID gatewayMsgId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(DELETE_STATE_SQL)) { + statement.setObject(1, gatewayMsgId); + statement.executeUpdate(); + } + } + + private void saveCorrelation(Connection connection, String providerName, + String providerMessageId, UUID gatewayMessageId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(SAVE_CORRELATION_SQL)) { + statement.setString(1, providerName); + statement.setString(2, providerMessageId); + statement.setObject(3, gatewayMessageId); + statement.setString(4, providerName); + statement.setString(5, providerMessageId); + statement.setObject(6, gatewayMessageId); + statement.executeUpdate(); + } + } + private void setStateParameters(Connection connection, PreparedStatement statement, UUID gatewayMsgId, MessageState state) throws SQLException { statement.setObject(1, gatewayMsgId); @@ -627,30 +669,56 @@ private void setStateParameters(Connection connection, PreparedStatement stateme statement.setString(8, state.getForwardDlrUrl()); setStringArray(connection, statement, 9, state.getReassembledParts()); statement.setString(10, state.getStatus().name()); - statement.setObject(11, OffsetDateTime.ofInstant( - Instant.ofEpochMilli(state.getTimestamp()), ZoneOffset.UTC)); + if (state.getDlrState() == null) { + statement.setNull(11, Types.INTEGER); + } else { + statement.setInt(11, state.getDlrState()); + } + statement.setString(12, state.getErrorCode()); + statement.setString(13, state.getDeliveryChannel().name()); + statement.setString(14, state.getDeliveryStatus().name()); + statement.setInt(15, state.getDeliveryAttemptCount()); + setTimestamp(statement, 16, state.getLastAttemptAt()); + setTimestamp(statement, 17, state.getNextAttemptAt()); + statement.setString(18, state.getLastDeliveryResult()); + setTimestamp(statement, 19, state.getResolvedAt()); + statement.setObject(20, toOffsetDateTime(state.getTimestamp())); + } + + private MessageState readState(ResultSet resultSet) throws SQLException { + MessageState state = new MessageState( + resultSet.getObject("gateway_message_id", UUID.class).toString(), + resultSet.getString("account_id"), + resultSet.getString("system_id"), + resultSet.getString("source_address"), + resultSet.getString("destination_address"), + resultSet.getString("forward_dlr_url")); + state.setProviderName(resultSet.getString("provider_name")); + state.setProviderMessageId(resultSet.getString("provider_message_id")); + state.setReassembledParts(readStringArray(resultSet, "reassembled_parts")); + state.setStatus(MessageState.MessageStatus.valueOf(resultSet.getString("provider_status"))); + state.setDlrState(resultSet.getObject("dlr_state", Integer.class)); + state.setErrorCode(resultSet.getString("error_code")); + state.setDeliveryChannel(MessageState.DeliveryChannel.valueOf(resultSet.getString("delivery_channel"))); + state.setDeliveryStatus(MessageState.DeliveryStatus.valueOf(resultSet.getString("delivery_status"))); + state.setDeliveryAttemptCount(resultSet.getInt("delivery_attempt_count")); + state.setLastAttemptAt(readEpochMillis(resultSet, "last_attempt_at")); + state.setNextAttemptAt(readEpochMillis(resultSet, "next_attempt_at")); + state.setLastDeliveryResult(resultSet.getString("last_delivery_result")); + state.setResolvedAt(readEpochMillis(resultSet, "resolved_at")); + state.setTimestamp(readRequiredEpochMillis(resultSet, "updated_at")); + return state; } - private void saveCorrelation(Connection connection, String providerName, - String providerMessageId, UUID gatewayMessageId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(SAVE_CORRELATION_SQL)) { - setCorrelationParameters(statement, providerName, providerMessageId, gatewayMessageId); - statement.executeUpdate(); + private void validateState(MessageState state) { + if (state.getStatus() == null || state.getDeliveryChannel() == null || state.getDeliveryStatus() == null) { + throw new IllegalArgumentException("DLR state statuses and delivery channel are required"); + } + if (state.getDeliveryAttemptCount() < 0) { + throw new IllegalArgumentException("Delivery attempt count must not be negative"); } - } - - private void setCorrelationParameters(PreparedStatement statement, String providerName, - String providerMessageId, UUID gatewayMessageId) throws SQLException { - statement.setString(1, providerName); - statement.setString(2, providerMessageId); - statement.setObject(3, gatewayMessageId); - statement.setString(4, providerName); - statement.setString(5, providerMessageId); - statement.setObject(6, gatewayMessageId); - } - - private void validateCorrelationFields(MessageState state) { if (state.getProviderName() == null && state.getProviderMessageId() == null) { + validateDeliveryTarget(state); return; } if (state.getProviderName() == null || state.getProviderName().isBlank() || @@ -658,11 +726,24 @@ private void validateCorrelationFields(MessageState state) { throw new IllegalArgumentException( "Provider name and provider message ID must either both be set or both be absent"); } + validateDeliveryTarget(state); + } + + private void validateDeliveryTarget(MessageState state) { + if (state.getDeliveryChannel() == MessageState.DeliveryChannel.HTTP && + (state.getForwardDlrUrl() == null || state.getForwardDlrUrl().isBlank())) { + throw new IllegalArgumentException("HTTP delivery requires a nonblank callback URL"); + } + if (state.getDeliveryChannel() == MessageState.DeliveryChannel.SMPP && + (state.getSystemId() == null || state.getSystemId().isBlank())) { + throw new IllegalArgumentException("SMPP delivery requires a nonblank system ID"); + } } - private boolean isLastCorrelationOwner(List states, int index, MessageState candidate) { + private boolean isLastCorrelationOwner(List> states, int index, + MessageState candidate) { for (int laterIndex = index + 1; laterIndex < states.size(); laterIndex++) { - MessageState later = states.get(laterIndex); + MessageState later = states.get(laterIndex).getValue(); if (candidate.getProviderName().equals(later.getProviderName()) && candidate.getProviderMessageId().equals(later.getProviderMessageId())) { return false; @@ -678,65 +759,29 @@ private void requireCorrelation(String providerName, String providerMessageId) { } } - private Optional lockResolvedState(Connection connection, String providerName, - String providerMessageId, - UUID gatewayMessageId, - MessageState.MessageStatus status) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(RESOLVE_STATE_SQL)) { - statement.setString(1, providerName); - statement.setString(2, providerMessageId); - statement.setObject(3, gatewayMessageId); - try (ResultSet resultSet = statement.executeQuery()) { - return resultSet.next() ? Optional.of(readResolvedState(resultSet, status)) : Optional.empty(); - } - } + private String normalizeResult(String result) { + return result == null || result.isBlank() ? null : result.trim(); } - private void deleteState(Connection connection, UUID gatewayMsgId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(DELETE_STATE_SQL)) { - statement.setObject(1, gatewayMsgId); - statement.executeUpdate(); + private void setTimestamp(PreparedStatement statement, int index, Long epochMillis) throws SQLException { + if (epochMillis == null) { + statement.setNull(index, Types.TIMESTAMP_WITH_TIMEZONE); + } else { + statement.setObject(index, toOffsetDateTime(epochMillis)); } } - private MessageState readBaseState(ResultSet resultSet) throws SQLException { - MessageState state = new MessageState( - resultSet.getObject("gateway_message_id", UUID.class).toString(), - resultSet.getString("account_id"), - resultSet.getString("system_id"), - resultSet.getString("source_address"), - resultSet.getString("destination_address"), - resultSet.getString("forward_dlr_url")); - state.setProviderName(resultSet.getString("provider_name")); - state.setProviderMessageId(resultSet.getString("provider_message_id")); - state.setReassembledParts(readStringArray(resultSet, "reassembled_parts")); - return state; - } - - /** - * Reads a {@link #GET_STATE_SQL} row, which carries the stored status and update time. - */ - private MessageState readTrackedState(ResultSet resultSet) throws SQLException { - MessageState state = readBaseState(resultSet); - state.setStatus(MessageState.MessageStatus.valueOf(resultSet.getString("status"))); - state.setTimestamp(readEpochMillis(resultSet, "updated_at")); - return state; + private OffsetDateTime toOffsetDateTime(long epochMillis) { + return OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochMillis), ZoneOffset.UTC); } - /** - * Reads a {@link #RESOLVE_STATE_SQL} row, which carries the provider message ID and resolution time. - */ - private MessageState readResolvedState(ResultSet resultSet, - MessageState.MessageStatus status) throws SQLException { - MessageState state = readBaseState(resultSet); - state.setStatus(status); - state.setTimestamp(readEpochMillis(resultSet, "resolved_at")); - return state; + private Long readEpochMillis(ResultSet resultSet, String columnName) throws SQLException { + OffsetDateTime value = resultSet.getObject(columnName, OffsetDateTime.class); + return value == null ? null : value.toInstant().toEpochMilli(); } - private long readEpochMillis(ResultSet resultSet, String columnName) throws SQLException { - OffsetDateTime value = resultSet.getObject(columnName, OffsetDateTime.class); - return value == null ? 0L : value.toInstant().toEpochMilli(); + private long readRequiredEpochMillis(ResultSet resultSet, String columnName) throws SQLException { + return resultSet.getObject(columnName, OffsetDateTime.class).toInstant().toEpochMilli(); } private List readStringArray(ResultSet resultSet, String columnName) throws SQLException { @@ -744,7 +789,6 @@ private List readStringArray(ResultSet resultSet, String columnName) thr if (array == null) { return null; } - //Arrays.asList, not List.of: a text[] column can legally hold NULL elements return new ArrayList<>(Arrays.asList((String[]) array.getArray())); } @@ -766,23 +810,15 @@ private void sleepBeforeLinkRetry() { } } - /** - * Runs retention cleanup at most once per interval, on the thread that first observes the interval has elapsed. - * Cleanup is best-effort maintenance: at most one thread runs it, every other caller proceeds immediately, and a - * failed pass is logged and retried after the next interval instead of failing the operation that triggered it. - */ private void checkExpiry() { - if (System.currentTimeMillis() - lastExpiryCheck < expiryCheckIntervalMillis) { - return; - } - if (!expiryInProgress.compareAndSet(false, true)) { + if (System.currentTimeMillis() - lastExpiryCheck < expiryCheckIntervalMillis || + !expiryInProgress.compareAndSet(false, true)) { return; } try { - if (System.currentTimeMillis() - lastExpiryCheck < expiryCheckIntervalMillis) { - return; + if (System.currentTimeMillis() - lastExpiryCheck >= expiryCheckIntervalMillis) { + deleteExpiredState(); } - deleteExpiredState(); } catch (RuntimeException e) { logger.warn("DLR retention cleanup failed; retrying after the next interval"); } finally { @@ -792,22 +828,12 @@ private void checkExpiry() { } private void deleteExpiredState() { - Map expiredUnpushedDlrKeys = new HashMap<>(); try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); - try (PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL); - PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL); - PreparedStatement unpushedDlrs = connection.prepareStatement(DELETE_EXPIRED_UNPUSHED_DLRS_SQL)) { - // Rebinding also locks tracked messages before correlations; retain the same order to avoid deadlocks. + try (PreparedStatement messages = connection.prepareStatement(DELETE_EXPIRED_MESSAGES_SQL); + PreparedStatement correlations = connection.prepareStatement(DELETE_EXPIRED_CORRELATIONS_SQL)) { messages.executeUpdate(); correlations.executeUpdate(); - try (ResultSet resultSet = unpushedDlrs.executeQuery()) { - while (resultSet.next()) { - expiredUnpushedDlrKeys.put( - resultSet.getString("dlr_key"), - resultSet.getObject("generation_id", UUID.class)); - } - } connection.commit(); } catch (SQLException e) { rollback(connection, e); @@ -816,12 +842,6 @@ private void deleteExpiredState() { } catch (SQLException e) { throw failure("expire DLR state", e); } - synchronized (unpushedDlrStateLock) { - expiredUnpushedDlrKeys.forEach(claimedUnpushedDlrKeys::remove); - HashSet expiredGenerations = new HashSet<>(expiredUnpushedDlrKeys.values()); - claimedUnpushedDlrGenerations.entrySet() - .removeIf(entry -> expiredGenerations.contains(entry.getValue())); - } } private UUID parseGatewayId(String gatewayMsgId) { @@ -840,15 +860,15 @@ private void rollback(Connection connection, SQLException failure) { } } - /** - * Logs the database cause server-side and returns a caller-facing exception that carries no connection details. - * The one-line summary keeps an outage diagnosable without a stack trace per rejected message; the full cause is - * available at debug level. - */ private DlrStorageException failure(String operation, SQLException cause) { logger.error("Failed to {}: sqlState={} errorCode={} reason={}", operation, cause.getSQLState(), cause.getErrorCode(), cause.getMessage()); logger.debug("DLR storage failure details while attempting to {}", operation, cause); return new DlrStorageException("Failed to " + operation, cause); } + + @FunctionalInterface + private interface StatementBinder { + void bind(PreparedStatement statement) throws SQLException; + } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java index 986720b..5949308 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java @@ -87,10 +87,13 @@ public void createAndEnqueueDLR(int mqid, String providerMessageId, String hashe } Optional optState = outWorker.getWorkerResources().getDlrService() - .resolveAndRemoveDlr(outWorker.getDlrProviderName(), providerMessageId, state); + .resolveDlr(outWorker.getDlrProviderName(), providerMessageId, state, errorCode); if (optState.isPresent()) { MessageState msgState = optState.get(); + if (msgState.getDeliveryChannel() != MessageState.DeliveryChannel.SMPP) { + return; + } StandardMessage dlrMsg = new StandardMessage(); dlrMsg.serial = msgState.getGatewayMsgId(); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/UnpushedDlr.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/UnpushedDlr.java deleted file mode 100644 index b9e3d6d..0000000 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/UnpushedDlr.java +++ /dev/null @@ -1,61 +0,0 @@ -package gr.cytech.sendium.core.worker; - -import gr.cytech.sendium.core.message.StandardMessage; -import io.quarkus.runtime.annotations.RegisterForReflection; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -@RegisterForReflection -public class UnpushedDlr implements Serializable { - private static final long serialVersionUID = 1L; - - public String systemId; - public String accountId; - public String from; - public String to; - public String serial; - public int msgId; - public int state; - public String errcode; - public boolean acked; - public int priority; - public List reassembledParts; - - public UnpushedDlr() { - } - - public static UnpushedDlr fromMessage(StandardMessage msg) { - UnpushedDlr dlr = new UnpushedDlr(); - dlr.systemId = msg.systemId; - dlr.accountId = msg.owner_id; - dlr.from = msg.from; - dlr.to = msg.to; - dlr.serial = msg.serial; - dlr.msgId = msg.msgId; - dlr.state = msg.state; - dlr.errcode = msg.errcode; - dlr.acked = msg.acked; - dlr.priority = msg.priority; - dlr.reassembledParts = msg.reassembledParts == null ? null : new ArrayList<>(msg.reassembledParts); - return dlr; - } - - public StandardMessage toMessage() { - StandardMessage msg = new StandardMessage(); - msg.type = StandardMessage.MSG_DLR; - msg.systemId = systemId; - msg.owner_id = accountId; - msg.from = from; - msg.to = to; - msg.serial = serial; - msg.msgId = msgId; - msg.state = state; - msg.errcode = errcode; - msg.acked = acked; - msg.priority = priority; - msg.reassembledParts = reassembledParts == null ? null : new ArrayList<>(reassembledParts); - return msg; - } -} diff --git a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql index b41086b..8b011db 100644 --- a/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql +++ b/sendium-core/src/main/resources/db/sendium-dlr/postgresql/V1__create_sendium_dlr_schema.sql @@ -1,6 +1,6 @@ CREATE SCHEMA IF NOT EXISTS sendium_dlr; -CREATE TABLE sendium_dlr.tracked_message ( +CREATE TABLE sendium_dlr.dlr_message ( gateway_message_id UUID PRIMARY KEY, account_id TEXT, system_id TEXT, @@ -10,26 +10,55 @@ CREATE TABLE sendium_dlr.tracked_message ( provider_message_id TEXT, forward_dlr_url TEXT, reassembled_parts TEXT[], - status TEXT NOT NULL, + provider_status TEXT NOT NULL, + dlr_state INTEGER, + error_code TEXT, + delivery_channel TEXT NOT NULL DEFAULT 'NONE', + delivery_status TEXT NOT NULL DEFAULT 'WAITING_PROVIDER', + delivery_attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at TIMESTAMPTZ, + next_attempt_at TIMESTAMPTZ, + last_delivery_result TEXT, + resolved_at TIMESTAMPTZ, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT tracked_message_provider_pair_check + CONSTRAINT dlr_message_provider_pair_check CHECK ((provider_name IS NULL) = (provider_message_id IS NULL)), - CONSTRAINT tracked_message_provider_name_not_blank + CONSTRAINT dlr_message_provider_name_not_blank CHECK (provider_name IS NULL OR provider_name !~ '^[[:space:]]*$'), - CONSTRAINT tracked_message_provider_message_id_not_blank + CONSTRAINT dlr_message_provider_message_id_not_blank CHECK (provider_message_id IS NULL OR provider_message_id !~ '^[[:space:]]*$'), - CONSTRAINT tracked_message_status_check - CHECK (status IN ('ACCEPTED', 'SENT', 'DELIVERED', 'FAILED')) + CONSTRAINT dlr_message_provider_status_check + CHECK (provider_status IN ('ACCEPTED', 'SENT', 'DELIVERED', 'FAILED')), + CONSTRAINT dlr_message_delivery_channel_check + CHECK (delivery_channel IN ('NONE', 'HTTP', 'SMPP')), + CONSTRAINT dlr_message_delivery_status_check + CHECK (delivery_status IN ('WAITING_PROVIDER', 'PENDING', 'FAILED')), + CONSTRAINT dlr_message_delivery_attempt_count_check + CHECK (delivery_attempt_count >= 0), + CONSTRAINT dlr_message_http_url_check + CHECK (delivery_channel <> 'HTTP' OR + (forward_dlr_url IS NOT NULL AND forward_dlr_url !~ '^[[:space:]]*$')), + CONSTRAINT dlr_message_smpp_system_id_check + CHECK (delivery_channel <> 'SMPP' OR + (system_id IS NOT NULL AND system_id !~ '^[[:space:]]*$')) ); -CREATE INDEX tracked_message_created_at_idx - ON sendium_dlr.tracked_message (created_at); +CREATE INDEX dlr_message_created_at_idx + ON sendium_dlr.dlr_message (created_at); -CREATE INDEX tracked_message_provider_message_id_idx - ON sendium_dlr.tracked_message (provider_name, provider_message_id) +CREATE INDEX dlr_message_provider_message_id_idx + ON sendium_dlr.dlr_message (provider_name, provider_message_id) WHERE provider_message_id IS NOT NULL; +CREATE INDEX dlr_message_http_due_idx + ON sendium_dlr.dlr_message (next_attempt_at) + WHERE delivery_channel = 'HTTP' AND delivery_status = 'PENDING'; + +CREATE INDEX dlr_message_smpp_replay_idx + ON sendium_dlr.dlr_message (system_id, resolved_at) + WHERE delivery_channel = 'SMPP' AND delivery_status = 'PENDING'; + CREATE TABLE sendium_dlr.provider_correlation ( provider_name TEXT NOT NULL, provider_message_id TEXT NOT NULL, @@ -42,7 +71,7 @@ CREATE TABLE sendium_dlr.provider_correlation ( CHECK (provider_message_id !~ '^[[:space:]]*$'), CONSTRAINT provider_correlation_message_fk FOREIGN KEY (gateway_message_id) - REFERENCES sendium_dlr.tracked_message (gateway_message_id) + REFERENCES sendium_dlr.dlr_message (gateway_message_id) ON DELETE CASCADE ); @@ -51,28 +80,3 @@ CREATE INDEX provider_correlation_created_at_idx CREATE INDEX provider_correlation_gateway_message_idx ON sendium_dlr.provider_correlation (gateway_message_id); - -CREATE TABLE sendium_dlr.unpushed_dlr ( - dlr_key TEXT PRIMARY KEY, - system_id TEXT NOT NULL, - account_id TEXT, - source_address TEXT, - destination_address TEXT, - serial TEXT, - message_id INTEGER NOT NULL, - dlr_state INTEGER NOT NULL, - error_code TEXT, - acked BOOLEAN NOT NULL, - priority INTEGER NOT NULL, - reassembled_parts TEXT[], - generation_id UUID NOT NULL DEFAULT gen_random_uuid(), - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT unpushed_dlr_system_id_not_blank - CHECK (system_id !~ '^[[:space:]]*$') -); - -CREATE INDEX unpushed_dlr_system_created_at_idx - ON sendium_dlr.unpushed_dlr (system_id, created_at); - -CREATE INDEX unpushed_dlr_created_at_idx - ON sendium_dlr.unpushed_dlr (created_at); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java index a0ef084..bf39f30 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/dlr/PostgresqlMigrationIT.java @@ -22,12 +22,6 @@ class PostgresqlMigrationIT { private static final String MIGRATION_LOCATION = "classpath:db/sendium-dlr/postgresql"; - private static final UUID INVALID_STATUS_GATEWAY_ID = - UUID.fromString("00000000-0000-0000-0000-000000000001"); - private static final UUID CORRELATION_GATEWAY_ID = - UUID.fromString("00000000-0000-0000-0000-000000000002"); - private static final UUID COMPLETE_GATEWAY_ID = - UUID.fromString("00000000-0000-0000-0000-000000000003"); private static final PostgreSQLContainer POSTGRESQL = new PostgreSQLContainer("postgres:17-alpine") .withDatabaseName("sendium") .withUsername("sendium") @@ -52,28 +46,36 @@ static void stopPostgresql() { } @Test - void migrationCreatesExpectedTablesAndIndexes() throws SQLException { + void migrationCreatesDlrMessageSchemaAndPartialIndexes() throws SQLException { assertThat(initialMigration.success).isTrue(); assertThat(initialMigration.migrationsExecuted).isOne(); try (Connection connection = connection()) { assertThat(loadNames(connection, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'sendium_dlr'")) - .containsExactlyInAnyOrder("tracked_message", "provider_correlation", "unpushed_dlr"); + .containsExactlyInAnyOrder("dlr_message", "provider_correlation"); assertThat(loadNames(connection, "SELECT indexname FROM pg_indexes WHERE schemaname = 'sendium_dlr'")) - .contains("tracked_message_created_at_idx", - "tracked_message_provider_message_id_idx", + .contains("dlr_message_created_at_idx", + "dlr_message_provider_message_id_idx", + "dlr_message_http_due_idx", + "dlr_message_smpp_replay_idx", "provider_correlation_created_at_idx", - "provider_correlation_gateway_message_idx", - "unpushed_dlr_system_created_at_idx", - "unpushed_dlr_created_at_idx"); - assertThat(loadColumnType(connection, "tracked_message", "gateway_message_id")) - .isEqualTo("uuid"); - assertThat(loadColumnType(connection, "provider_correlation", "gateway_message_id")) - .isEqualTo("uuid"); - assertThat(loadColumnType(connection, "provider_correlation", "provider_name")) - .isEqualTo("text"); + "provider_correlation_gateway_message_idx"); + assertThat(loadIndexDefinition(connection, "dlr_message_http_due_idx")) + .contains("next_attempt_at") + .contains("delivery_channel = 'HTTP'") + .contains("delivery_status = 'PENDING'"); + assertThat(loadIndexDefinition(connection, "dlr_message_smpp_replay_idx")) + .contains("system_id", "resolved_at") + .contains("delivery_channel = 'SMPP'") + .contains("delivery_status = 'PENDING'"); + assertThat(loadColumnType(connection, "dlr_message", "gateway_message_id")).isEqualTo("uuid"); + assertThat(loadColumnNames(connection, "dlr_message")) + .contains("dlr_state", "error_code", "delivery_channel", "delivery_status", + "delivery_attempt_count", "last_attempt_at", "next_attempt_at", + "last_delivery_result", "resolved_at") + .doesNotContain("generation_id"); } } @@ -86,109 +88,100 @@ void migrationIsIdempotent() { } @Test - void trackedMessageRejectsUnknownStatus() throws SQLException { - try (Connection connection = connection(); - PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.tracked_message - (gateway_message_id, account_id, system_id, status) - VALUES (?, 'account', 'system', 'UNKNOWN') - """)) { - statement.setObject(1, INVALID_STATUS_GATEWAY_ID); - assertThatThrownBy(statement::executeUpdate).isInstanceOf(SQLException.class); + void schemaRejectsInvalidProviderAndDeliveryStates() throws SQLException { + try (Connection connection = connection()) { + assertInvalidMessage(connection, "UNKNOWN", "NONE", "WAITING_PROVIDER", 0, null, null); + assertInvalidMessage(connection, "ACCEPTED", "MAIL", "WAITING_PROVIDER", 0, null, null); + assertInvalidMessage(connection, "ACCEPTED", "NONE", "DONE", 0, null, null); + assertInvalidMessage(connection, "ACCEPTED", "NONE", "WAITING_PROVIDER", -1, null, null); } } @Test - void providerCorrelationFieldsRejectBlankValues() throws SQLException { + void schemaRequiresValidChannelTargets() throws SQLException { try (Connection connection = connection()) { for (String blank : List.of("", " ", "\t\n")) { - assertThatThrownBy(() -> insertTrackedMessageWithProvider( - connection, UUID.randomUUID(), blank, "provider-message")) - .isInstanceOf(SQLException.class); - assertThatThrownBy(() -> insertTrackedMessageWithProvider( - connection, UUID.randomUUID(), "provider", blank)) - .isInstanceOf(SQLException.class); + assertInvalidMessage(connection, "ACCEPTED", "HTTP", "WAITING_PROVIDER", 0, "system", blank); + assertInvalidMessage(connection, "ACCEPTED", "SMPP", "WAITING_PROVIDER", 0, blank, + "https://example.test/dlr"); } - - UUID gatewayMessageId = UUID.randomUUID(); - insertTrackedMessage(connection, gatewayMessageId); - assertThatThrownBy(() -> insertCorrelation(connection, " ", "provider-message", gatewayMessageId)) - .isInstanceOf(SQLException.class); - assertThatThrownBy(() -> insertCorrelation(connection, "provider", "\t\n", gatewayMessageId)) - .isInstanceOf(SQLException.class); + assertInvalidMessage(connection, "ACCEPTED", "HTTP", "WAITING_PROVIDER", 0, "system", null); + assertInvalidMessage(connection, "ACCEPTED", "SMPP", "WAITING_PROVIDER", 0, null, + "https://example.test/dlr"); } } @Test - void correlationIsDeletedWithTrackedMessage() throws SQLException { + void providerCorrelationReferencesDlrMessageAndCascades() throws SQLException { + UUID gatewayId = UUID.randomUUID(); try (Connection connection = connection()) { - insertTrackedMessage(connection, CORRELATION_GATEWAY_ID); - insertCorrelation(connection, "provider-1", "provider-message-1", CORRELATION_GATEWAY_ID); - insertCorrelation(connection, "provider-1", "provider-message-2", CORRELATION_GATEWAY_ID); - + insertMessage(connection, gatewayId); try (PreparedStatement statement = connection.prepareStatement(""" - DELETE FROM sendium_dlr.tracked_message WHERE gateway_message_id = ? + INSERT INTO sendium_dlr.provider_correlation + (provider_name, provider_message_id, gateway_message_id) + VALUES ('provider', 'message', ?) """)) { - statement.setObject(1, CORRELATION_GATEWAY_ID); + statement.setObject(1, gatewayId); statement.executeUpdate(); } - - assertThat(countCorrelations(connection, CORRELATION_GATEWAY_ID)).isZero(); - } - } - - @Test - void unpushedDlrRequiresNonBlankSystemId() throws SQLException { - try (Connection connection = connection()) { - assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "empty-system", "")) - .isInstanceOf(SQLException.class); - assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "whitespace-system", " ")) - .isInstanceOf(SQLException.class); - assertThatThrownBy(() -> insertMinimalUnpushedDlr(connection, "control-whitespace-system", "\t\n")) - .isInstanceOf(SQLException.class); + try (PreparedStatement statement = connection.prepareStatement( + "DELETE FROM sendium_dlr.dlr_message WHERE gateway_message_id = ?")) { + statement.setObject(1, gatewayId); + statement.executeUpdate(); + } + assertThat(loadCount(connection, "sendium_dlr.provider_correlation")).isZero(); } } @Test - void typedColumnsStoreCurrentDlrState() throws SQLException { + void defaultsWaitingProviderWithNoAttempts() throws SQLException { + UUID gatewayId = UUID.randomUUID(); try (Connection connection = connection()) { - insertCompleteTrackedMessage(connection); - insertCompleteUnpushedDlr(connection); - + insertMessage(connection, gatewayId); try (PreparedStatement statement = connection.prepareStatement(""" - SELECT gateway_message_id, provider_message_id, reassembled_parts, created_at, updated_at - FROM sendium_dlr.tracked_message - WHERE gateway_message_id = ? - """)) { - statement.setObject(1, COMPLETE_GATEWAY_ID); + SELECT delivery_channel, delivery_status, delivery_attempt_count + FROM sendium_dlr.dlr_message WHERE gateway_message_id = ? + """)) { + statement.setObject(1, gatewayId); try (ResultSet resultSet = statement.executeQuery()) { assertThat(resultSet.next()).isTrue(); - assertThat(resultSet.getObject("gateway_message_id", UUID.class)) - .isEqualTo(COMPLETE_GATEWAY_ID); - assertThat(resultSet.getString("provider_message_id")).isNull(); - assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) - .containsExactly("part-1", "part-2"); - assertThat(resultSet.getObject("created_at")).isNotNull(); - assertThat(resultSet.getObject("updated_at")).isNotNull(); + assertThat(resultSet.getString("delivery_channel")).isEqualTo("NONE"); + assertThat(resultSet.getString("delivery_status")).isEqualTo("WAITING_PROVIDER"); + assertThat(resultSet.getInt("delivery_attempt_count")).isZero(); } } + } + } - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(""" - SELECT account_id, message_id, dlr_state, error_code, acked, priority, reassembled_parts - FROM sendium_dlr.unpushed_dlr - WHERE dlr_key = 'dlr-complete' - """)) { - assertThat(resultSet.next()).isTrue(); - assertThat(resultSet.getString("account_id")).isEqualTo("account"); - assertThat(resultSet.getInt("message_id")).isEqualTo(123); - assertThat(resultSet.getInt("dlr_state")).isEqualTo(1); - assertThat(resultSet.getString("error_code")).isEqualTo("0"); - assertThat(resultSet.getBoolean("acked")).isTrue(); - assertThat(resultSet.getInt("priority")).isEqualTo(2); - assertThat((String[]) resultSet.getArray("reassembled_parts").getArray()) - .containsExactly("part-1", "part-2"); + private static void assertInvalidMessage(Connection connection, String providerStatus, String channel, + String deliveryStatus, int attempts, String systemId, + String callbackUrl) { + assertThatThrownBy(() -> { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.dlr_message + (gateway_message_id, provider_status, delivery_channel, delivery_status, + delivery_attempt_count, system_id, forward_dlr_url) + VALUES (?, ?, ?, ?, ?, ?, ?) + """)) { + statement.setObject(1, UUID.randomUUID()); + statement.setString(2, providerStatus); + statement.setString(3, channel); + statement.setString(4, deliveryStatus); + statement.setInt(5, attempts); + statement.setString(6, systemId); + statement.setString(7, callbackUrl); + statement.executeUpdate(); } + }).isInstanceOf(SQLException.class); + } + + private static void insertMessage(Connection connection, UUID gatewayId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO sendium_dlr.dlr_message (gateway_message_id, provider_status) + VALUES (?, 'ACCEPTED') + """)) { + statement.setObject(1, gatewayId); + statement.executeUpdate(); } } @@ -207,130 +200,54 @@ private static Set loadNames(Connection connection, String sql) throws S return names; } - private static void insertTrackedMessage(Connection connection, UUID gatewayMessageId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.tracked_message - (gateway_message_id, account_id, system_id, status) - VALUES (?, 'account', 'system', 'ACCEPTED') - """)) { - statement.setObject(1, gatewayMessageId); - statement.executeUpdate(); - } - } - - private static void insertTrackedMessageWithProvider(Connection connection, UUID gatewayMessageId, - String providerName, - String providerMessageId) throws SQLException { + private static Set loadColumnNames(Connection connection, String tableName) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.tracked_message - (gateway_message_id, provider_name, provider_message_id, status) - VALUES (?, ?, ?, 'ACCEPTED') + SELECT column_name FROM information_schema.columns + WHERE table_schema = 'sendium_dlr' AND table_name = ? """)) { - statement.setObject(1, gatewayMessageId); - statement.setString(2, providerName); - statement.setString(3, providerMessageId); - statement.executeUpdate(); - } - } - - private static void insertCorrelation(Connection connection, String providerName, String providerMessageId, - UUID gatewayMessageId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.provider_correlation - (provider_name, provider_message_id, gateway_message_id) - VALUES (?, ?, ?) - """)) { - statement.setString(1, providerName); - statement.setString(2, providerMessageId); - statement.setObject(3, gatewayMessageId); - statement.executeUpdate(); - } - } - - private static void insertMinimalUnpushedDlr(Connection connection, String key, - String systemId) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.unpushed_dlr - (dlr_key, system_id, message_id, dlr_state, acked, priority) - VALUES (?, ?, 1, 1, FALSE, 0) - """)) { - statement.setString(1, key); - statement.setString(2, systemId); - statement.executeUpdate(); - } - } - - private static void insertCompleteTrackedMessage(Connection connection) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.tracked_message - (gateway_message_id, account_id, system_id, source_address, destination_address, - forward_dlr_url, reassembled_parts, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """)) { - statement.setObject(1, COMPLETE_GATEWAY_ID); - statement.setString(2, "account"); - statement.setString(3, "system"); - statement.setString(4, "source"); - statement.setString(5, "destination"); - statement.setString(6, "https://example.test/dlr"); - statement.setArray(7, connection.createArrayOf("text", new String[]{"part-1", "part-2"})); - statement.setString(8, "ACCEPTED"); - statement.executeUpdate(); - } - } - - private static void insertCompleteUnpushedDlr(Connection connection) throws SQLException { - try (PreparedStatement statement = connection.prepareStatement(""" - INSERT INTO sendium_dlr.unpushed_dlr - (dlr_key, system_id, account_id, source_address, destination_address, serial, - message_id, dlr_state, error_code, acked, priority, reassembled_parts) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """)) { - statement.setString(1, "dlr-complete"); - statement.setString(2, "system"); - statement.setString(3, "account"); - statement.setString(4, "source"); - statement.setString(5, "destination"); - statement.setString(6, "serial"); - statement.setInt(7, 123); - statement.setInt(8, 1); - statement.setString(9, "0"); - statement.setBoolean(10, true); - statement.setInt(11, 2); - statement.setArray(12, connection.createArrayOf("text", new String[]{"part-1", "part-2"})); - statement.executeUpdate(); + statement.setString(1, tableName); + Set names = new HashSet<>(); + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + names.add(resultSet.getString(1)); + } + } + return names; } } - private static int countCorrelations(Connection connection, UUID gatewayMessageId) throws SQLException { + private static String loadColumnType(Connection connection, String tableName, + String columnName) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(""" - SELECT COUNT(*) - FROM sendium_dlr.provider_correlation - WHERE gateway_message_id = ? + SELECT data_type FROM information_schema.columns + WHERE table_schema = 'sendium_dlr' AND table_name = ? AND column_name = ? """)) { - statement.setObject(1, gatewayMessageId); + statement.setString(1, tableName); + statement.setString(2, columnName); try (ResultSet resultSet = statement.executeQuery()) { - resultSet.next(); - return resultSet.getInt(1); + assertThat(resultSet.next()).isTrue(); + return resultSet.getString(1); } } } - private static String loadColumnType(Connection connection, String tableName, - String columnName) throws SQLException { + private static String loadIndexDefinition(Connection connection, String indexName) throws SQLException { try (PreparedStatement statement = connection.prepareStatement(""" - SELECT data_type - FROM information_schema.columns - WHERE table_schema = 'sendium_dlr' - AND table_name = ? - AND column_name = ? + SELECT indexdef FROM pg_indexes WHERE schemaname = 'sendium_dlr' AND indexname = ? """)) { - statement.setString(1, tableName); - statement.setString(2, columnName); + statement.setString(1, indexName); try (ResultSet resultSet = statement.executeQuery()) { assertThat(resultSet.next()).isTrue(); return resultSet.getString(1); } } } + + private static int loadCount(Connection connection, String table) throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM " + table)) { + resultSet.next(); + return resultSet.getInt(1); + } + } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java index 4a4f0f4..b7f24b9 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java @@ -74,6 +74,18 @@ void persistsStateBeforeQueueAdmissionForEverySubmission() throws InterruptedExc assertThat(response.getEntity()).isEqualTo(message.serial).isEqualTo(state.getGatewayMsgId()); assertThat(message.acked).isTrue(); assertThat(state.getForwardDlrUrl()).isNull(); + assertThat(state.getDeliveryChannel()).isEqualTo(MessageState.DeliveryChannel.NONE); + } + + @Test + void callbackSubmissionUsesHttpDeliveryChannel() { + ArgumentCaptor stateCaptor = ArgumentCaptor.forClass(MessageState.class); + + Response response = submit("https://callback.test/dlr"); + + assertThat(response.getStatus()).isEqualTo(Response.Status.ACCEPTED.getStatusCode()); + verify(dlrService).saveInitialState(stateCaptor.capture()); + assertThat(stateCaptor.getValue().getDeliveryChannel()).isEqualTo(MessageState.DeliveryChannel.HTTP); } @Test diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index 4f2b540..01f8b71 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -90,6 +90,29 @@ void parseDlrAndCreateResponse_whenReceiptIsValid_enqueuesDlrWithRegisteredTlvs( assertThat(tracker.dlrTlvs).containsEntry("carrier_1400", "network-a"); } + @Test + void parseDlrAndCreateResponse_whenReceiptIsIntermediate_acknowledgesWithoutEnqueuing() throws Exception { + CapturingTracker tracker = new CapturingTracker(); + TestSmppClientWorker worker = new TestSmppClientWorker( + new TestConfigurationProvider(), new Queue<>(), tracker); + + for (String state : Set.of("ACCEPTD", "ENROUTE")) { + DeliverSm deliverSm = new DeliverSm(); + deliverSm.setSourceAddress(new Address((byte) 1, (byte) 1, "smsc")); + deliverSm.setDestAddress(new Address((byte) 1, (byte) 1, "recipient")); + deliverSm.setDataCoding(SmppConstants.DATA_CODING_DEFAULT); + deliverSm.setShortMessage(CharsetUtil.encode( + "id:abc123 sub:001 dlvrd:000 submit date:2401010000 done date: stat:" + state + + " err:000 text:pending", + CharsetUtil.NAME_GSM)); + + PduResponse response = worker.parseDlrAndCreateResponse(deliverSm); + + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK); + } + assertThat(tracker.dlrAttempts).isZero(); + } + @Test void parseDlrAndCreateResponse_whenStorageFails_returnsSystemErrorForProviderRetry() throws Exception { CapturingTracker tracker = new CapturingTracker(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatchTest.java new file mode 100644 index 0000000..4f5a39f --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatchTest.java @@ -0,0 +1,113 @@ +package gr.cytech.sendium.core.smpp.server; + +import gr.cytech.sendium.core.message.StandardMessage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DlrDeliveryBatchTest { + @Mock private SmppServerMessageStore store; + + private StandardMessage message; + + @BeforeEach + void setUp() { + message = new StandardMessage(); + message.serial = "gateway-1"; + message.type = StandardMessage.MSG_DLR; + } + + @Test + void singlePartSuccessCompletesAttempt() { + when(store.completeDlrDeliveryAttempt(message, 3)).thenReturn(true); + DlrDeliveryBatch batch = batch(3, Set.of(0)); + + batch.partSucceeded(0); + + verify(store).completeDlrDeliveryAttempt(message, 3); + verify(store, never()).releaseDlrDeliveryAttempt( + message, 3, DlrDeliveryBatch.COMPLETION_STORAGE_ERROR); + assertThat(batch.isActive()).isFalse(); + } + + @Test + void multipartCompletesAfterAllDistinctPartsAndIgnoresDuplicateResponse() { + when(store.completeDlrDeliveryAttempt(message, 4)).thenReturn(true); + DlrDeliveryBatch batch = batch(4, Set.of(0, 1, 2)); + + batch.partSucceeded(0); + batch.partSucceeded(0); + batch.partSucceeded(2); + verify(store, never()).completeDlrDeliveryAttempt(message, 4); + batch.partSucceeded(1); + + verify(store, times(1)).completeDlrDeliveryAttempt(message, 4); + } + + @Test + void firstFailureReleasesOnlyOnce() { + when(store.releaseDlrDeliveryAttempt(message, 5, "timeout")).thenReturn(true); + DlrDeliveryBatch batch = batch(5, Set.of(0, 1)); + + batch.fail("timeout"); + batch.fail("non_ok_response"); + batch.partSucceeded(0); + + verify(store, times(1)).releaseDlrDeliveryAttempt(message, 5, "timeout"); + verify(store, never()).completeDlrDeliveryAttempt(message, 5); + } + + @Test + void completionStorageFailureReleasesAttempt() { + when(store.completeDlrDeliveryAttempt(message, 6)).thenReturn(false); + when(store.releaseDlrDeliveryAttempt(message, 6, DlrDeliveryBatch.COMPLETION_STORAGE_ERROR)) + .thenReturn(true); + DlrDeliveryBatch batch = batch(6, Set.of(0)); + + batch.partSucceeded(0); + + verify(store).releaseDlrDeliveryAttempt(message, 6, DlrDeliveryBatch.COMPLETION_STORAGE_ERROR); + } + + @Test + void completionStorageExceptionReleasesAttempt() { + when(store.completeDlrDeliveryAttempt(message, 9)).thenThrow(new IllegalStateException("database down")); + when(store.releaseDlrDeliveryAttempt(message, 9, DlrDeliveryBatch.COMPLETION_STORAGE_ERROR)) + .thenReturn(true); + DlrDeliveryBatch batch = batch(9, Set.of(0)); + + batch.partSucceeded(0); + + verify(store).releaseDlrDeliveryAttempt(message, 9, DlrDeliveryBatch.COMPLETION_STORAGE_ERROR); + } + + @Test + void callbackFromTerminalOldAttemptCannotAffectNewAttempt() { + when(store.releaseDlrDeliveryAttempt(message, 7, "timeout")).thenReturn(true); + when(store.completeDlrDeliveryAttempt(message, 8)).thenReturn(true); + DlrDeliveryBatch oldBatch = batch(7, Set.of(0)); + DlrDeliveryBatch newBatch = batch(8, Set.of(0)); + + oldBatch.fail("timeout"); + newBatch.partSucceeded(0); + oldBatch.partSucceeded(0); + + verify(store, never()).completeDlrDeliveryAttempt(message, 7); + verify(store).completeDlrDeliveryAttempt(message, 8); + } + + private DlrDeliveryBatch batch(int attempt, Set expectedParts) { + return new DlrDeliveryBatch<>(message, attempt, expectedParts, store, null); + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandlerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandlerTest.java index bd2d62f..bd0cb67 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandlerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerSessionHandlerTest.java @@ -1,8 +1,12 @@ package gr.cytech.sendium.core.smpp.server; +import com.cloudhopper.smpp.PduAsyncResponse; import com.cloudhopper.smpp.SmppConstants; import com.cloudhopper.smpp.SmppSession; import com.cloudhopper.smpp.SmppSessionConfiguration; +import com.cloudhopper.smpp.pdu.DeliverSm; +import com.cloudhopper.smpp.pdu.DeliverSmResp; +import com.cloudhopper.smpp.pdu.GenericNack; import com.cloudhopper.smpp.pdu.SubmitSm; import com.cloudhopper.smpp.pdu.SubmitSmResp; import com.cloudhopper.smpp.tlv.Tlv; @@ -18,13 +22,12 @@ import java.nio.charset.StandardCharsets; import java.sql.Timestamp; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) class SmppServerSessionHandlerTest { @@ -34,6 +37,7 @@ class SmppServerSessionHandlerTest { @Mock private SmppSessionContext sessionContext; @Mock private SubmitSmProcessor submitProcessor; @Mock private SmppSessionConfiguration sessionConfiguration; + @Mock private SmppServerMessageStore messageStore; private SmppServerSessionHandler handler; @@ -153,4 +157,92 @@ void validateScheduleDeliveryTime_whenInvalid_shouldEnqueueInvalidScheduleRespon assertThat(result).isNull(); assertThat(respCaptor.getValue().getCommandStatus()).isEqualTo(SmppConstants.STATUS_INVSCHED); } + + @Test + void expectedOkDeliverSmResponseCompletesPart() { + DlrDeliveryBatch batch = batch(1); + when(messageStore.completeDlrDeliveryAttempt(any(), eq(1))).thenReturn(true); + DeliverSm request = request(batch); + DeliverSmResp response = new DeliverSmResp(); + response.setCommandStatus(SmppConstants.STATUS_OK); + + handler.fireExpectedPduResponseReceived(asyncResponse(request, response)); + + verify(messageStore).completeDlrDeliveryAttempt(any(), eq(1)); + } + + @Test + void expectedNonOkDeliverSmResponseReleasesAttempt() { + DlrDeliveryBatch batch = batch(2); + when(messageStore.releaseDlrDeliveryAttempt(any(), eq(2), eq("non_ok_response"))).thenReturn(true); + DeliverSm request = request(batch); + DeliverSmResp response = new DeliverSmResp(); + response.setCommandStatus(SmppConstants.STATUS_SYSERR); + + handler.fireExpectedPduResponseReceived(asyncResponse(request, response)); + + verify(messageStore).releaseDlrDeliveryAttempt(any(), eq(2), eq("non_ok_response")); + } + + @Test + void expectedWrongOrGenericResponseReleasesAttempt() { + DlrDeliveryBatch wrongBatch = batch(3); + DlrDeliveryBatch nackBatch = batch(4); + when(messageStore.releaseDlrDeliveryAttempt(any(), eq(3), eq("wrong_response"))).thenReturn(true); + when(messageStore.releaseDlrDeliveryAttempt(any(), eq(4), eq("generic_nack"))).thenReturn(true); + + handler.fireExpectedPduResponseReceived(asyncResponse(request(wrongBatch), new SubmitSmResp())); + handler.fireExpectedPduResponseReceived(asyncResponse(request(nackBatch), new GenericNack())); + + verify(messageStore).releaseDlrDeliveryAttempt(any(), eq(3), eq("wrong_response")); + verify(messageStore).releaseDlrDeliveryAttempt(any(), eq(4), eq("generic_nack")); + } + + @Test + void expiredDeliverSmReleasesAttemptWithoutLegacyUpsert() { + DlrDeliveryBatch batch = batch(5); + when(messageStore.releaseDlrDeliveryAttempt(any(), eq(5), eq("timeout"))).thenReturn(true); + DeliverSm request = request(batch); + + handler.firePduRequestExpired(request); + + verify(messageStore).releaseDlrDeliveryAttempt(any(), eq(5), eq("timeout")); + verify(worker, never()).markAsUnpushed(any()); + } + + @Test + void unexpectedDisconnectReleasesOutstandingBatches() { + DlrDeliveryBatch batch = batch(6); + SmppServerBindHandler bindHandler = mock(SmppServerBindHandler.class); + ServerConnections connections = mock(ServerConnections.class); + when(worker.getBindHandler()).thenReturn(bindHandler); + when(bindHandler.getConnections()).thenReturn(connections); + when(messageStore.releaseDlrDeliveryAttempt(any(), eq(6), eq("session_closed"))).thenReturn(true); + assertThat(handler.registerDlrBatch(batch)).isTrue(); + + handler.fireChannelUnexpectedlyClosed(); + + verify(messageStore).releaseDlrDeliveryAttempt(any(), eq(6), eq("session_closed")); + verify(connections).removeConnection(handler); + } + + private DlrDeliveryBatch batch(int attempt) { + StandardMessage message = new StandardMessage(); + message.serial = "gateway-" + attempt; + message.type = StandardMessage.MSG_DLR; + return new DlrDeliveryBatch<>(message, attempt, Set.of(0), messageStore, handler); + } + + private DeliverSm request(DlrDeliveryBatch batch) { + DeliverSm request = new DeliverSm(); + request.setReferenceObject(new DlrDeliverSmReference<>(handler, batch, 0, "receipt-1")); + return request; + } + + private PduAsyncResponse asyncResponse(DeliverSm request, com.cloudhopper.smpp.pdu.PduResponse response) { + PduAsyncResponse asyncResponse = mock(PduAsyncResponse.class); + when(asyncResponse.getRequest()).thenReturn(request); + when(asyncResponse.getResponse()).thenReturn(response); + return asyncResponse; + } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java index 8a34417..9a9dda9 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java @@ -2,6 +2,7 @@ import com.cloudhopper.commons.charset.CharsetUtil; import com.cloudhopper.smpp.SmppConstants; +import com.cloudhopper.smpp.SmppSession; import com.cloudhopper.smpp.pdu.DeliverSm; import com.cloudhopper.smpp.pdu.Pdu; import com.cloudhopper.smpp.pdu.SubmitSm; @@ -17,14 +18,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; class SmppServerWorkerReassemblyTest { @@ -115,6 +116,115 @@ void reassembledDlrGeneratesDeliverSmPerOriginalPartIdWithSameStatus() throws Ex assertThat(bodies).anySatisfy(body -> assertThat(body).contains("id:part-3")); } + @Test + void multipartDlrStartsOneAttemptAndUsesTypedPartReferences() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + SmppServerSessionHandler handler = reachableHandler(worker); + worker.setMessageStore(store); + when(store.tracksDlrDeliveryAttempts()).thenReturn(true); + when(store.startDlrDeliveryAttempt(any())).thenReturn(OptionalInt.of(9)); + StandardMessage dlr = dlrMessage(); + dlr.reassembledParts = new ArrayList<>(List.of("part-1", "part-2", "part-3")); + + assertThat(worker.doMessage(0, dlr)).isNull(); + + verify(store, times(1)).startDlrDeliveryAttempt(dlr); + assertThat(worker.outgoingPdus).hasSize(3).allSatisfy(pdu -> { + assertThat(pdu.getReferenceObject()).isInstanceOf(DlrDeliverSmReference.class); + DlrDeliverSmReference reference = (DlrDeliverSmReference) pdu.getReferenceObject(); + assertThat(reference.handler()).isSameAs(handler); + assertThat(reference.batch().getAttempt()).isEqualTo(9); + }); + assertThat(worker.outgoingPdus.stream() + .map(pdu -> ((DlrDeliverSmReference) pdu.getReferenceObject()).receiptMessageId())) + .containsExactly("part-1", "part-2", "part-3"); + } + + @Test + void dlrWithoutReachableSessionLeavesPendingWithoutStartingAttempt() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + SmppServerBindHandler bindHandler = mock(SmppServerBindHandler.class); + worker.setMessageStore(store); + worker.setBindHandler(bindHandler); + when(store.tracksDlrDeliveryAttempts()).thenReturn(true); + + assertThat(worker.doMessage(0, dlrMessage())).isNull(); + + verify(store, never()).startDlrDeliveryAttempt(any()); + verify(store, never()).markAsUnpushed(any()); + assertThat(worker.outgoingPdus).isEmpty(); + } + + @Test + void duplicateDlrAttemptDoesNotSend() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + reachableHandler(worker); + worker.setMessageStore(store); + when(store.tracksDlrDeliveryAttempts()).thenReturn(true); + when(store.startDlrDeliveryAttempt(any())).thenReturn(OptionalInt.empty()); + + assertThat(worker.doMessage(0, dlrMessage())).isNull(); + + assertThat(worker.outgoingPdus).isEmpty(); + verify(store, never()).releaseDlrDeliveryAttempt(any(), anyInt(), anyString()); + } + + @Test + void enqueueFailureReleasesStartedAttempt() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker( + new TestConfigurationProvider(), new Queue<>(), true); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + reachableHandler(worker); + worker.setMessageStore(store); + when(store.tracksDlrDeliveryAttempts()).thenReturn(true); + when(store.startDlrDeliveryAttempt(any())).thenReturn(OptionalInt.of(10)); + when(store.releaseDlrDeliveryAttempt(any(), eq(10), eq("enqueue_failed"))).thenReturn(true); + StandardMessage dlr = dlrMessage(); + dlr.reassembledParts = new ArrayList<>(List.of("part-1", "part-2")); + + assertThat(worker.doMessage(0, dlr)).isNull(); + + verify(store).releaseDlrDeliveryAttempt(dlr, 10, "enqueue_failed"); + assertThat(worker.outgoingPdus).hasSize(1); + assertThat(((DlrDeliverSmReference) worker.outgoingPdus.getFirst().getReferenceObject()) + .batch().isActive()).isFalse(); + } + + @Test + void generationFailureDoesNotStartOrMutateDurableAttempt() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + reachableHandler(worker); + worker.setMessageStore(store); + when(store.tracksDlrDeliveryAttempts()).thenReturn(true); + StandardMessage dlr = dlrMessage(); + dlr.errcode = "not-a-number"; + + assertThat(worker.doMessage(0, dlr)).isNull(); + + verify(store, never()).startDlrDeliveryAttempt(any()); + verify(store, never()).releaseDlrDeliveryAttempt(any(), anyInt(), anyString()); + assertThat(worker.outgoingPdus).isEmpty(); + } + + @Test + void dlrWithoutDurableTrackingKeepsExistingInMemoryRetryBehavior() throws Exception { + TestSmppServerWorker worker = new TestSmppServerWorker(new TestConfigurationProvider(), new Queue<>()); + SmppServerMessageStore store = mock(SmppServerMessageStore.class); + SmppServerBindHandler bindHandler = mock(SmppServerBindHandler.class); + worker.setMessageStore(store); + worker.setBindHandler(bindHandler); + StandardMessage dlr = dlrMessage(); + + assertThat(worker.doMessage(0, dlr)).isSameAs(dlr); + + verify(store).markAsUnpushed(dlr); + verify(store, never()).startDlrDeliveryAttempt(any()); + } + @Test void normalSubmissionRoutesAndAcknowledgesOnlyAfterPersistence() throws Exception { Queue routerQueue = new Queue<>(); @@ -260,12 +370,50 @@ private static StandardMessage messagePart(String udh, String body, String seria return message; } + private static StandardMessage dlrMessage() { + StandardMessage dlr = new StandardMessage(); + dlr.serial = "gateway-1"; + dlr.owner_id = "account-a"; + dlr.systemId = "system-a"; + dlr.from = "306900000001"; + dlr.to = "sender"; + dlr.type = StandardMessage.MSG_DLR; + dlr.state = StandardMessage.DLR_STAT_DELIVRD; + dlr.errcode = "0"; + return dlr; + } + + private SmppServerSessionHandler reachableHandler(TestSmppServerWorker worker) { + SmppServerBindHandler bindHandler = mock(SmppServerBindHandler.class); + SmppServerSessionHandler handler = mock(SmppServerSessionHandler.class); + SmppSession session = mock(SmppSession.class); + when(bindHandler.isConnectionReachable("account-a")).thenReturn(true); + when(bindHandler.isSystemIdReachable("account-a", "system-a")).thenReturn(true); + when(bindHandler.getHandlerForSending("account-a", "system-a")).thenReturn(handler); + when(handler.getSession()).thenReturn(session); + when(session.isBound()).thenReturn(true); + when(handler.registerDlrBatch(any())).thenReturn(true); + worker.setBindHandler(bindHandler); + return handler; + } + private static class TestSmppServerWorker extends SmppServerWorker { private final List workerQueueMessages = new ArrayList<>(); private final List outgoingPdus = new ArrayList<>(); + private final boolean failSecondDlrEnqueue; TestSmppServerWorker(SendiumConfigurationProvider configurationProvider, Queue routerQueue) { + this(configurationProvider, routerQueue, false); + } + + TestSmppServerWorker(SendiumConfigurationProvider configurationProvider, Queue routerQueue, + boolean failSecondDlrEnqueue) { super(configurationProvider, "smpp", routerQueue); + this.failSecondDlrEnqueue = failSecondDlrEnqueue; + } + + void setBindHandler(SmppServerBindHandler bindHandler) { + this.bindHandler = bindHandler; } @Override @@ -275,6 +423,9 @@ public void enqueue(StandardMessage pMsg) { @Override public void enqueueOut(Pdu event) { + if (failSecondDlrEnqueue && outgoingPdus.size() == 1) { + throw new IllegalStateException("queue rejected"); + } outgoingPdus.add(event); } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java index 23683e5..9d83449 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/StandardSmppServerMessageStoreTest.java @@ -44,6 +44,7 @@ void setUp() { when(workerResources.isDlrPersistenceEnabled()).thenReturn(true); when(workerResources.getDlrService()).thenReturn(dlrService); when(worker.getMaxRetries()).thenReturn(5); + when(worker.isForwardDlrs()).thenReturn(true); messageStore = new StandardSmppServerMessageStore(worker); } @@ -58,6 +59,7 @@ void persistMessages_SavesStatesAsOneBatchBeforeNotifyingWorker() { msg1.systemId = "sys1"; msg1.from = "from1"; msg1.to = "to1"; + msg1.acked = true; StandardMessage msg2 = new StandardMessage(); msg2.serial = "gw-2"; @@ -65,6 +67,7 @@ void persistMessages_SavesStatesAsOneBatchBeforeNotifyingWorker() { msg2.systemId = "sys2"; msg2.from = "from2"; msg2.to = "to2"; + msg2.acked = true; InEvent event1 = new InEvent<>(msg1, new SubmitSm(), 1, new Timestamp(System.currentTimeMillis())); @@ -82,8 +85,10 @@ void persistMessages_SavesStatesAsOneBatchBeforeNotifyingWorker() { order.verify(worker).handlePersistedMessages(events); assertEquals("account1", captor.getValue().get(0).getAccountId()); assertEquals("sys1", captor.getValue().get(0).getSystemId()); + assertEquals(MessageState.DeliveryChannel.SMPP, captor.getValue().get(0).getDeliveryChannel()); assertEquals("account2", captor.getValue().get(1).getAccountId()); assertEquals("sys2", captor.getValue().get(1).getSystemId()); + assertEquals(MessageState.DeliveryChannel.SMPP, captor.getValue().get(1).getDeliveryChannel()); } @Test @@ -103,6 +108,91 @@ void persistMessages_SavesReassembledPartIds() { assertEquals(List.of("part-1", "part-2"), captor.getValue().getFirst().getReassembledParts()); } + @Test + void persistMessages_UsesNoneChannelWhenDlrWasNotRequested() { + StandardMessage msg = new StandardMessage(); + msg.serial = "gw-1"; + msg.systemId = "sys1"; + + messageStore.persistMessages(List.of(new InEvent<>( + msg, new SubmitSm(), 1, new Timestamp(System.currentTimeMillis())))); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(dlrService).saveInitialStates(captor.capture()); + assertEquals(MessageState.DeliveryChannel.NONE, + captor.getValue().getFirst().getDeliveryChannel()); + } + + @Test + void persistMessages_UsesNoneChannelWhenDlrForwardingIsDisabled() { + when(worker.isForwardDlrs()).thenReturn(false); + StandardMessage msg = new StandardMessage(); + msg.serial = "gw-1"; + msg.systemId = "sys1"; + msg.acked = true; + + messageStore.persistMessages(List.of(new InEvent<>( + msg, new SubmitSm(), 1, new Timestamp(System.currentTimeMillis())))); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(dlrService).saveInitialStates(captor.capture()); + assertEquals(MessageState.DeliveryChannel.NONE, + captor.getValue().getFirst().getDeliveryChannel()); + } + + @Test + void durableDlrAttemptsFollowPersistenceBoundary() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false, true); + + assertFalse(messageStore.tracksDlrDeliveryAttempts()); + assertTrue(messageStore.tracksDlrDeliveryAttempts()); + } + + @Test + void markAsUnpushedFallsBackWhenPersistenceIsDisabled() { + when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); + StandardMessage dlr = new StandardMessage(); + dlr.type = StandardMessage.MSG_DLR; + + assertFalse(messageStore.markAsUnpushed(dlr)); + } + + @Test + void onClientConnected_ReconstructsAndEnqueuesPendingDlrWithoutCompletingIt() throws Exception { + MessageState state = pendingState(); + when(dlrService.listPendingSmppDeliveries("sys1")).thenReturn(List.of(state)); + + messageStore.onClientConnected("sys1"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(StandardMessage.class); + verify(worker).enqueue(captor.capture()); + StandardMessage replay = captor.getValue(); + assertAll( + () -> assertEquals(StandardMessage.MSG_DLR, replay.type), + () -> assertEquals("gw-1", replay.serial), + () -> assertEquals("destination", replay.from), + () -> assertEquals("source", replay.to), + () -> assertEquals(StandardMessage.DLR_STAT_UNDELIV, replay.state), + () -> assertEquals("42", replay.errcode), + () -> assertEquals("account1", replay.owner_id), + () -> assertEquals("sys1", replay.systemId), + () -> assertEquals(List.of("part-1", "part-2"), replay.reassembledParts)); + verify(dlrService, never()).completeDelivery(anyString(), anyInt()); + } + + @Test + void onClientConnected_WhenEnqueueFailsRetainsPendingDlr() throws Exception { + MessageState state = pendingState(); + when(dlrService.listPendingSmppDeliveries("sys1")).thenReturn(List.of(state)); + doThrow(new InterruptedException("queue stopped")).when(worker).enqueue(any()); + + messageStore.onClientConnected("sys1"); + + verify(dlrService, never()).completeDelivery(anyString(), anyInt()); + verify(dlrService, never()).retryDelivery(anyString(), anyInt(), anyString(), anyLong()); + assertTrue(Thread.interrupted()); + } + @Test void persistMessages_WithNullMessage_Skips() { List> events = new ArrayList<>(); @@ -161,27 +251,6 @@ void persistMessages_WhenPersistenceDisabled_AcknowledgesWithoutStoring() { verify(worker, never()).handleMessagePersistenceFailure(anyList()); } - @Test - void markAsUnpushed_WhenPersistenceDisabled_LeavesRetryToWorker() { - when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); - StandardMessage msg = new StandardMessage(); - msg.type = StandardMessage.MSG_DLR; - - assertFalse(messageStore.markAsUnpushed(msg)); - - verify(workerResources, never()).getDlrService(); - } - - @Test - void onClientConnected_WhenPersistenceDisabled_DoesNotReplay() { - when(workerResources.isDlrPersistenceEnabled()).thenReturn(false); - - messageStore.onClientConnected("sys1"); - - verify(workerResources, never()).getDlrService(); - verify(worker, never()).enqueueNoExceptions(any()); - } - @Test void getMaxAttempts_DelegatesToWorker() { int result = messageStore.getMaxAttempts(true); @@ -204,54 +273,15 @@ private InEvent event(String serial, SubmitSm submitSm) { return new InEvent<>(message, submitSm, 1, new Timestamp(System.currentTimeMillis())); } - @Test - void markAsUnpushed_Dlr_SavesToDlrService() { - StandardMessage msg = new StandardMessage(); - msg.type = StandardMessage.MSG_DLR; - when(dlrService.saveUnpushedDlr(msg)).thenReturn(true); - - boolean result = messageStore.markAsUnpushed(msg); - - assertTrue(result); - verify(dlrService).saveUnpushedDlr(msg); - } - - @Test - void markAsUnpushed_NonDlr_ReturnsFalse() { - StandardMessage msg = new StandardMessage(); - msg.type = StandardMessage.MSG_TEXT; - - boolean result = messageStore.markAsUnpushed(msg); - - assertFalse(result); - verify(dlrService, never()).saveUnpushedDlr(any()); + private MessageState pendingState() { + MessageState state = new MessageState( + "gw-1", "account1", "sys1", "source", "destination", null); + state.setDlrState(StandardMessage.DLR_STAT_UNDELIV); + state.setErrorCode("42"); + state.setReassembledParts(List.of("part-1", "part-2")); + state.setDeliveryChannel(MessageState.DeliveryChannel.SMPP); + state.setDeliveryStatus(MessageState.DeliveryStatus.PENDING); + return state; } - @Test - void onClientConnected_ReEnqueuesAndRemovesMatchingDlrs() { - StandardMessage dlr = new StandardMessage(); - dlr.type = StandardMessage.MSG_DLR; - dlr.owner_id = "account1"; - dlr.systemId = "sys1"; - when(dlrService.claimUnpushedDlrs("sys1")).thenReturn(List.of(dlr)); - when(worker.enqueueNoExceptions(dlr)).thenReturn(true); - - messageStore.onClientConnected("sys1"); - - verify(worker).enqueueNoExceptions(dlr); - verify(dlrService).removeUnpushedDlr(dlr); - } - - @Test - void onClientConnected_LeavesDlrStoredWhenReEnqueueFails() { - StandardMessage dlr = new StandardMessage(); - dlr.type = StandardMessage.MSG_DLR; - when(dlrService.claimUnpushedDlrs("sys1")).thenReturn(List.of(dlr)); - when(worker.enqueueNoExceptions(dlr)).thenReturn(false); - - messageStore.onClientConnected("sys1"); - - verify(dlrService, never()).removeUnpushedDlr(any()); - verify(dlrService).releaseUnpushedDlrClaim(dlr); - } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/tasks/OutTaskTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/tasks/OutTaskTest.java new file mode 100644 index 0000000..3ab01b6 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/tasks/OutTaskTest.java @@ -0,0 +1,86 @@ +package gr.cytech.sendium.core.smpp.server.tasks; + +import com.cloudhopper.smpp.pdu.DeliverSm; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.smpp.server.DlrDeliverSmReference; +import gr.cytech.sendium.core.smpp.server.DlrDeliveryBatch; +import gr.cytech.sendium.core.smpp.server.SmppServerMessageStore; +import gr.cytech.sendium.core.smpp.server.SmppServerSessionHandler; +import gr.cytech.sendium.core.smpp.server.SmppServerWorker; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Set; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class OutTaskTest { + @Mock private SmppServerWorker worker; + @Mock private SmppServerSessionHandler handler; + @Mock private SmppServerMessageStore store; + + private StandardMessage message; + + @BeforeEach + void setUp() { + message = new StandardMessage(); + message.serial = "gateway-1"; + message.type = StandardMessage.MSG_DLR; + } + + @Test + void successfulSendOnlyDispatchesAndDoesNotCompleteBatch() throws Exception { + DlrDeliveryBatch batch = batch(1); + DeliverSm request = request(batch); + when(handler.sendPduRequest(request)).thenReturn(true); + + new OutTask<>(worker, request).run(); + + verify(handler).sendPduRequest(request); + verify(store, never()).completeDlrDeliveryAttempt(any(), eq(1)); + verify(store, never()).releaseDlrDeliveryAttempt(any(), eq(1), any()); + } + + @Test + void failedSendReleasesBatchWithoutLegacyWorkerFailure() throws Exception { + DlrDeliveryBatch batch = batch(2); + DeliverSm request = request(batch); + when(handler.sendPduRequest(request)).thenReturn(false); + when(store.releaseDlrDeliveryAttempt(message, 2, "send_failed")).thenReturn(true); + + new OutTask<>(worker, request).run(); + + verify(store).releaseDlrDeliveryAttempt(message, 2, "send_failed"); + verify(worker, never()).outTaskFailed(any(), any()); + } + + @Test + void inactiveBatchIsNotSent() throws Exception { + DlrDeliveryBatch batch = batch(3); + when(store.releaseDlrDeliveryAttempt(message, 3, "timeout")).thenReturn(true); + batch.fail("timeout"); + DeliverSm request = request(batch); + + new OutTask<>(worker, request).run(); + + verify(handler, never()).sendPduRequest(any()); + } + + private DlrDeliveryBatch batch(int attempt) { + return new DlrDeliveryBatch<>(message, attempt, Set.of(0), store, handler); + } + + private DeliverSm request(DlrDeliveryBatch batch) { + DeliverSm request = new DeliverSm(); + request.setReferenceObject(new DlrDeliverSmReference<>(handler, batch, 0, "receipt-1")); + return request; + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java index 451b051..9a60fcd 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/DlrServiceTest.java @@ -1,5 +1,6 @@ package gr.cytech.sendium.core.worker; +import gr.cytech.sendium.core.message.StandardMessage; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -9,8 +10,8 @@ import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -18,56 +19,54 @@ class DlrServiceTest { @Mock DlrStorage storage; - @Mock - ForwardDlrService forwardDlrService; - @InjectMocks DlrService service; @Test - void resolveAndRemoveDlrMapsDeliveredStateAndForwardsCallback() { + void resolveDlrPassesExactDeliveredOutcome() { MessageState state = stateWithCallback(); - when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.DELIVERED)) - .thenReturn(Optional.of(state)); + when(storage.resolveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_DELIVRD, "007")).thenReturn(Optional.of(state)); - Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 1); + Optional result = service.resolveDlr( + "provider-1", "provider-message-1", StandardMessage.DLR_STAT_DELIVRD, "007"); assertThat(result).containsSame(state); - verify(forwardDlrService).forwardDlr(state); + verify(storage).resolveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_DELIVRD, "007"); } @Test - void resolveAndRemoveDlrMapsAcceptedState() { - MessageState state = stateWithoutCallback(); - when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.ACCEPTED)) - .thenReturn(Optional.of(state)); + void resolveDlrPreservesFinalOnlyGuard() { + assertThat(service.resolveDlr( + "provider-1", "accepted", StandardMessage.DLR_STAT_ACCEPTD, "000")).isEmpty(); + assertThat(service.resolveDlr( + "provider-1", "buffered", StandardMessage.DLR_STAT_BUFFRED, "000")).isEmpty(); - Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 9); - - assertThat(result).containsSame(state); - verify(forwardDlrService, never()).forwardDlr(state); + verifyNoInteractions(storage); } @Test - void resolveAndRemoveDlrMapsUnknownStateToFailed() { + void resolveDlrMapsTerminalFailureAndPassesExactError() { MessageState state = stateWithoutCallback(); - when(storage.resolveAndRemoveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.FAILED)) - .thenReturn(Optional.of(state)); + when(storage.resolveDlr("provider-1", "provider-message-1", MessageState.MessageStatus.FAILED, + StandardMessage.DLR_STAT_REJECTD, " exact ")).thenReturn(Optional.of(state)); - Optional result = service.resolveAndRemoveDlr("provider-1", "provider-message-1", 0); + Optional result = service.resolveDlr( + "provider-1", "provider-message-1", StandardMessage.DLR_STAT_REJECTD, " exact "); assertThat(result).containsSame(state); } @Test - void resolveAndRemoveDlrDoesNotForwardMissingState() { - when(storage.resolveAndRemoveDlr("provider-1", "unknown", MessageState.MessageStatus.DELIVERED)) - .thenReturn(Optional.empty()); + void resolveDlrReturnsMissingState() { + when(storage.resolveDlr("provider-1", "unknown", MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_SEEN, null)).thenReturn(Optional.empty()); - Optional result = service.resolveAndRemoveDlr("provider-1", "unknown", 15); + Optional result = service.resolveDlr( + "provider-1", "unknown", StandardMessage.DLR_STAT_SEEN, null); assertThat(result).isEmpty(); - verify(forwardDlrService, never()).forwardDlr(org.mockito.ArgumentMatchers.any()); } private MessageState stateWithCallback() { diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java index 9b65f6d..c625724 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java @@ -1,88 +1,350 @@ package gr.cytech.sendium.core.worker; +import com.sun.net.httpserver.HttpServer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; -import static org.junit.jupiter.api.Assertions.*; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) class ForwardDlrServiceTest { + private static final String GATEWAY_ID = "3fdac55b-a953-4a36-8d0f-0273e3537502"; + + @Mock + DlrService dlrService; - private ForwardDlrService forwardDlrService; + @Mock + HttpClient httpClient; + + private SimpleMeterRegistry meterRegistry; + private ForwardDlrService service; + private HttpServer server; @BeforeEach void setUp() { - forwardDlrService = new ForwardDlrService(); + meterRegistry = new SimpleMeterRegistry(); + service = new ForwardDlrService(dlrService, meterRegistry, httpClient); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + meterRegistry.close(); + } + + @Test + void schedulerUsesBoundedBatchAndDoesNothingWhenNoDeliveryIsDue() throws Exception { + when(dlrService.listDueHttpDeliveries(100)).thenReturn(List.of()); + + service.dispatchDueDeliveries(); + + verify(dlrService).listDueHttpDeliveries(100); + verifyNoInteractions(httpClient); + } + + @Test + void successfulResponseCompletesExpectedAttempt() throws Exception { + MessageState due = dueState("https://example.test/dlr?id=%s&type=%d"); + dueAttempt(due, 1); + respondWith(204); + when(dlrService.completeDelivery(GATEWAY_ID, 1)).thenReturn(true); + + service.dispatchDueDeliveries(); + + ArgumentCaptor request = ArgumentCaptor.forClass(HttpRequest.class); + verify(httpClient).send(request.capture(), anyBodyHandler()); + assertThat(request.getValue().uri().toString()) + .isEqualTo("https://example.test/dlr?id=" + GATEWAY_ID + "&type=1"); + assertThat(request.getValue().timeout()).contains(java.time.Duration.ofSeconds(5)); + verify(dlrService).completeDelivery(GATEWAY_ID, 1); + var order = inOrder(dlrService, httpClient); + order.verify(dlrService).startDeliveryAttempt(GATEWAY_ID, MessageState.DeliveryChannel.HTTP); + order.verify(httpClient).send(any(HttpRequest.class), anyBodyHandler()); + assertAttemptMetric("success", 1); } @Test - void mapToKannelType_Accepted_ReturnsBuffered() { - int result = forwardDlrService.mapToKannelType(MessageState.MessageStatus.ACCEPTED); - assertEquals(4, result); + void directRedirectCompletesAndIsNotFollowed() throws Exception { + AtomicInteger redirectTargetRequests = new AtomicInteger(); + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/redirect", exchange -> { + exchange.getResponseHeaders().add("Location", "/target"); + exchange.sendResponseHeaders(302, -1); + exchange.close(); + }); + server.createContext("/target", exchange -> { + redirectTargetRequests.incrementAndGet(); + exchange.sendResponseHeaders(204, -1); + exchange.close(); + }); + server.start(); + MessageState due = dueState("http://" + server.getAddress().getHostString() + ':' + + server.getAddress().getPort() + "/redirect"); + dueAttempt(due, 1); + when(dlrService.completeDelivery(GATEWAY_ID, 1)).thenReturn(true); + service = new ForwardDlrService(dlrService, meterRegistry, ForwardDlrService.newHttpClient()); + + service.dispatchDueDeliveries(); + + verify(dlrService).completeDelivery(GATEWAY_ID, 1); + assertThat(redirectTargetRequests).hasValue(0); + assertThat(ForwardDlrService.newHttpClient().followRedirects()).isEqualTo(HttpClient.Redirect.NEVER); + assertAttemptMetric("success", 1); } @Test - void mapToKannelType_Sent_ReturnsSmscSubmit() { - int result = forwardDlrService.mapToKannelType(MessageState.MessageStatus.SENT); - assertEquals(8, result); + void clientErrorSchedulesRetryWithNormalizedResult() throws Exception { + assertHttpFailureSchedulesRetry(404); } @Test - void mapToKannelType_Delivered_ReturnsSuccess() { - int result = forwardDlrService.mapToKannelType(MessageState.MessageStatus.DELIVERED); - assertEquals(1, result); + void serverErrorSchedulesRetryWithNormalizedResult() throws Exception { + assertHttpFailureSchedulesRetry(503); + } + + @Test + void tenthFailureMarksDeliveryFailed() throws Exception { + MessageState due = dueState("https://example.test/dlr"); + dueAttempt(due, 10); + respondWith(500); + when(dlrService.failDelivery(GATEWAY_ID, 10, "http_failure")).thenReturn(true); + + service.dispatchDueDeliveries(); + + verify(dlrService).failDelivery(GATEWAY_ID, 10, "http_failure"); + verify(dlrService, never()).retryDelivery(eq(GATEWAY_ID), eq(10), any(), anyLong()); + assertThat(meterRegistry.get("sendium.dlr.delivery.terminal.failure") + .tags("channel", "http", "reason", "max_attempts").counter().count()).isEqualTo(1); } @Test - void mapToKannelType_Failed_ReturnsFailure() { - int result = forwardDlrService.mapToKannelType(MessageState.MessageStatus.FAILED); - assertEquals(2, result); + void timeoutSchedulesRetry() throws Exception { + MessageState due = dueState("https://secret.example.test/dlr?token=do-not-expose"); + dueAttempt(due, 2); + when(httpClient.send(any(HttpRequest.class), anyBodyHandler())) + .thenThrow(new HttpTimeoutException("request timed out")); + when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(2), eq("timeout"), anyLong())) + .thenReturn(true); + + service.dispatchDueDeliveries(); + + verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(2), eq("timeout"), anyLong()); + assertAttemptMetric("timeout", 1); + assertThat(deliveryResult().getValue()).doesNotContain("secret", "token", "http"); } @Test - void mapToKannelType_Null_ReturnsBuffered() { - int result = forwardDlrService.mapToKannelType(null); - assertEquals(4, result); + void ioFailureSchedulesTransportRetry() throws Exception { + MessageState due = dueState("https://example.test/dlr"); + dueAttempt(due, 3); + when(httpClient.send(any(HttpRequest.class), anyBodyHandler())) + .thenThrow(new IOException("connection refused")); + when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(3), eq("transport_failure"), anyLong())) + .thenReturn(true); + + service.dispatchDueDeliveries(); + + verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(3), eq("transport_failure"), anyLong()); + assertAttemptMetric("transport_failure", 1); } @Test - void buildForwardUrl_ReplacesDlrTypePlaceholder() { - String result = forwardDlrService.buildForwardUrl("http://example.com/dlr?type=%d", "msg-123", 1); - assertEquals("http://example.com/dlr?type=1", result); + void runtimeTransportFailureSchedulesRetry() throws Exception { + MessageState due = dueState("https://example.test/dlr"); + dueAttempt(due, 4); + when(httpClient.send(any(HttpRequest.class), anyBodyHandler())) + .thenThrow(new IllegalStateException("transport unavailable")); + when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(4), eq("transport_failure"), anyLong())) + .thenReturn(true); + + service.dispatchDueDeliveries(); + + verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(4), eq("transport_failure"), anyLong()); + assertAttemptMetric("transport_failure", 1); } @Test - void buildForwardUrl_ReplacesMsgIdPlaceholder() { - String result = forwardDlrService.buildForwardUrl("http://example.com/dlr?id=%s", "msg-123", 1); - assertEquals("http://example.com/dlr?id=msg-123", result); + void interruptionSchedulesRetryAndRestoresInterrupt() throws Exception { + MessageState due = dueState("https://example.test/dlr"); + dueAttempt(due, 5); + when(httpClient.send(any(HttpRequest.class), anyBodyHandler())) + .thenThrow(new InterruptedException("interrupted")); + when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(5), eq("interrupted"), anyLong())) + .thenReturn(true); + + try { + service.dispatchDueDeliveries(); + + verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(5), eq("interrupted"), anyLong()); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + assertAttemptMetric("transport_failure", 1); + } finally { + Thread.interrupted(); + } } @Test - void buildForwardUrl_BothPlaceholders() { - String result = forwardDlrService.buildForwardUrl("http://example.com/dlr?id=%s&type=%d", "msg-123", 2); - assertEquals("http://example.com/dlr?id=msg-123&type=2", result); + void invalidUriFailsWithoutStartingAnAttempt() { + MessageState due = dueState("https://example.test/%ZZ?secret=value"); + when(dlrService.listDueHttpDeliveries(100)).thenReturn(List.of(due)); + when(dlrService.failInvalidDelivery(GATEWAY_ID, "invalid_uri")).thenReturn(true); + + service.dispatchDueDeliveries(); + + verify(dlrService).failInvalidDelivery(GATEWAY_ID, "invalid_uri"); + verify(dlrService, never()).startDeliveryAttempt(any(), any()); + verifyNoInteractions(httpClient); + assertThat(meterRegistry.get("sendium.dlr.delivery.terminal.failure") + .tags("channel", "http", "reason", "invalid_uri").counter().count()).isEqualTo(1); } @Test - void buildForwardUrl_NoPlaceholders_Unchanged() { - String result = forwardDlrService.buildForwardUrl("http://example.com/dlr?id=123", "msg-123", 1); - assertEquals("http://example.com/dlr?id=123", result); + void activeAttemptIsSkippedBeforeSending() { + MessageState due = dueState("https://example.test/dlr"); + when(dlrService.listDueHttpDeliveries(100)).thenReturn(List.of(due)); + when(dlrService.startDeliveryAttempt(GATEWAY_ID, MessageState.DeliveryChannel.HTTP)) + .thenReturn(Optional.empty()); + + service.dispatchDueDeliveries(); + + verifyNoInteractions(httpClient); } @Test - void forwardDlr_NullUrl_NoAction() { - MessageState state = new MessageState("msg-1", "system", "from", "to", null); - assertDoesNotThrow(() -> forwardDlrService.forwardDlr(state)); + void completionStorageFailureLeavesDeliveryForLaterRunAndRecordsBoundedError() throws Exception { + MessageState due = dueState("https://example.test/dlr"); + dueAttempt(due, 1); + respondWith(200); + doThrow(new DlrStorageException("database unavailable")) + .when(dlrService).completeDelivery(GATEWAY_ID, 1); + + service.dispatchDueDeliveries(); + + verify(dlrService).completeDelivery(GATEWAY_ID, 1); + assertThat(meterRegistry.get("sendium.dlr.delivery.dispatch.error") + .tags("channel", "http", "source", "storage").counter().count()).isEqualTo(1); } @Test - void forwardDlr_EmptyUrl_NoAction() { - MessageState state = new MessageState("msg-1", "system", "from", "to", ""); - assertDoesNotThrow(() -> forwardDlrService.forwardDlr(state)); + void schedulerStorageFailureIsCountedWithoutDynamicMetricTags() { + when(dlrService.listDueHttpDeliveries(100)).thenThrow(new DlrStorageException("database unavailable")); + + service.dispatchDueDeliveries(); + + assertThat(meterRegistry.get("sendium.dlr.delivery.dispatch.error") + .tags("channel", "http", "source", "scheduler").counter().count()).isEqualTo(1); + assertBoundedMetricTags(); } @Test - void forwardDlr_WhitespaceUrl_NoAction() { - MessageState state = new MessageState("msg-1", "system", "from", "to", " "); - assertDoesNotThrow(() -> forwardDlrService.forwardDlr(state)); + void statusAndUrlPlaceholderMappingsRemainStable() { + assertThat(service.mapToKannelType(MessageState.MessageStatus.ACCEPTED)).isEqualTo(4); + assertThat(service.mapToKannelType(MessageState.MessageStatus.SENT)).isEqualTo(8); + assertThat(service.mapToKannelType(MessageState.MessageStatus.DELIVERED)).isEqualTo(1); + assertThat(service.mapToKannelType(MessageState.MessageStatus.FAILED)).isEqualTo(2); + assertThat(service.mapToKannelType(null)).isEqualTo(4); + assertThat(service.buildForwardUrl("https://example.test?id=%s&type=%d", "msg-1", 2)) + .isEqualTo("https://example.test?id=msg-1&type=2"); + } + + private void assertHttpFailureSchedulesRetry(int statusCode) throws Exception { + MessageState due = dueState("https://secret.example.test/dlr?token=do-not-expose"); + dueAttempt(due, 1); + respondWith(statusCode); + when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(1), eq("http_failure"), anyLong())) + .thenReturn(true); + long beforeFailure = System.currentTimeMillis(); + + service.dispatchDueDeliveries(); + + ArgumentCaptor nextAttempt = ArgumentCaptor.forClass(Long.class); + verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(1), eq("http_failure"), nextAttempt.capture()); + assertThat(nextAttempt.getValue()).isBetween(beforeFailure + 120_000, System.currentTimeMillis() + 120_000); + assertThat(deliveryResult().getValue()).isEqualTo("http_failure"); + assertAttemptMetric("http_failure", 1); + assertBoundedMetricTags(); + } + + private ArgumentCaptor deliveryResult() { + ArgumentCaptor result = ArgumentCaptor.forClass(String.class); + verify(dlrService).retryDelivery(eq(GATEWAY_ID), anyInt(), result.capture(), anyLong()); + return result; + } + + private void dueAttempt(MessageState due, int attempt) { + MessageState started = dueState(due.getForwardDlrUrl()); + started.setDeliveryAttemptCount(attempt); + when(dlrService.listDueHttpDeliveries(100)).thenReturn(List.of(due)); + when(dlrService.startDeliveryAttempt(GATEWAY_ID, MessageState.DeliveryChannel.HTTP)) + .thenReturn(Optional.of(started)); + } + + private MessageState dueState(String callbackUrl) { + MessageState state = new MessageState(GATEWAY_ID, "account", "system", "source", "destination", + callbackUrl); + state.setStatus(MessageState.MessageStatus.DELIVERED); + state.setDeliveryChannel(MessageState.DeliveryChannel.HTTP); + state.setDeliveryStatus(MessageState.DeliveryStatus.PENDING); + return state; + } + + private void respondWith(int statusCode) throws Exception { + HttpResponse response = mock(HttpResponse.class); + when(response.statusCode()).thenReturn(statusCode); + when(httpClient.send(any(HttpRequest.class), anyBodyHandler())).thenReturn(response); + } + + @SuppressWarnings("unchecked") + private HttpResponse.BodyHandler anyBodyHandler() { + return any(HttpResponse.BodyHandler.class); + } + + private void assertAttemptMetric(String outcome, long expectedCount) { + assertThat(meterRegistry.get("sendium.dlr.delivery.attempt") + .tags("channel", "http", "outcome", outcome).timer().count()).isEqualTo(expectedCount); + } + + private void assertBoundedMetricTags() { + Set permittedTags = Set.of("channel", "outcome", "reason", "source"); + assertThat(meterRegistry.getMeters()) + .filteredOn(meter -> meter.getId().getName().startsWith("sendium.dlr.delivery")) + .allSatisfy(meter -> assertThat(meter.getId().getTags()) + .extracting(io.micrometer.core.instrument.Tag::getKey) + .allMatch(permittedTags::contains)); + assertThat(meterRegistry.getMeters()) + .flatExtracting(meter -> meter.getId().getTags()) + .extracting(io.micrometer.core.instrument.Tag::getKey) + .doesNotContain("id", "url", "host", "provider", "attempt"); } -} \ No newline at end of file +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java index c0e8e83..5defe17 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/PostgresqlDlrStorageIT.java @@ -15,7 +15,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -64,16 +64,14 @@ static void stopPostgresql() { void resetStorage() throws SQLException { try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) { - statement.execute("TRUNCATE sendium_dlr.tracked_message, sendium_dlr.unpushed_dlr CASCADE"); + statement.execute("TRUNCATE sendium_dlr.dlr_message CASCADE"); } storage = new PostgresqlDlrStorage(dataSource); } @Test - void saveInitialStateRoundTripsAllFields() { - MessageState state = newState(); - state.setProviderName(PROVIDER); - state.setProviderMessageId("provider-message-initial"); + void initialStateRoundTripsAndDefaultsToNoDelivery() { + MessageState state = state(MessageState.DeliveryChannel.NONE, "system", null); state.setReassembledParts(List.of("part-1", "part-2")); storage.saveInitialState(state); @@ -82,371 +80,261 @@ void saveInitialStateRoundTripsAllFields() { .get() .usingRecursiveComparison() .isEqualTo(state); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "provider-message-initial", MessageState.MessageStatus.DELIVERED)) - .isPresent(); + assertThat(state.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.WAITING_PROVIDER); + assertThat(state.getDeliveryAttemptCount()).isZero(); } @Test - void saveInitialStatesCommitsWholeBatch() { - List states = List.of(newState(), newState(), newState()); - - storage.saveInitialStates(states); + void terminalHttpStateIsRetainedWithExactOutcomeAndAllCorrelationsConsumed() throws SQLException { + MessageState state = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/dlr"); + storage.saveInitialState(state); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); - for (MessageState state : states) { - assertThat(storage.getState(state.getGatewayMsgId())) - .get() - .usingRecursiveComparison() - .isEqualTo(state); - } - } + Optional resolved = storage.resolveDlr(PROVIDER, "provider-message-1", + MessageState.MessageStatus.FAILED, StandardMessage.DLR_STAT_REJECTD, " exact-101 "); - @Test - void saveInitialStatesRebindsCorrelationToNewestBatchMessage() throws SQLException { - MessageState owner = newState(); - owner.setProviderName(PROVIDER); - owner.setProviderMessageId("shared-provider-message"); - storage.saveInitialState(owner); - MessageState innocent = newState(); - MessageState conflict = newState(); - conflict.setProviderName(PROVIDER); - conflict.setProviderMessageId("shared-provider-message"); - - storage.saveInitialStates(List.of(innocent, conflict)); - - assertThat(storage.getState(innocent.getGatewayMsgId())).isPresent(); - assertThat(storage.getState(conflict.getGatewayMsgId())).isPresent(); - assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getProviderMessageId()).isNull(); - assertThat(countCorrelations(owner.getGatewayMsgId())).isZero(); - assertThat(countCorrelations(conflict.getGatewayMsgId())).isOne(); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId) - .isEqualTo(conflict.getGatewayMsgId()); + assertThat(resolved).get().satisfies(actual -> { + assertThat(actual.getStatus()).isEqualTo(MessageState.MessageStatus.FAILED); + assertThat(actual.getDlrState()).isEqualTo(StandardMessage.DLR_STAT_REJECTD); + assertThat(actual.getErrorCode()).isEqualTo(" exact-101 "); + assertThat(actual.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.PENDING); + assertThat(actual.getResolvedAt()).isNotNull(); + assertThat(actual.getNextAttemptAt()).isNotNull(); + }); + assertThat(storage.getState(state.getGatewayMsgId())).isPresent(); + assertThat(countCorrelations(state.getGatewayMsgId())).isZero(); + assertThat(storage.resolveDlr(PROVIDER, "provider-message-2", MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_DELIVRD, "000")).isEmpty(); } @Test - void saveInitialStatesUsesFinalStateForDuplicateGatewayMessage() throws SQLException { - MessageState correlated = newState(); - correlated.setProviderName(PROVIDER); - correlated.setProviderMessageId("superseded-provider-message"); - MessageState replacement = new MessageState(correlated.getGatewayMsgId(), "replacement-account", - "replacement-system", "replacement-source", "replacement-destination", null); + void terminalStateWithoutDeliveryChannelIsDeletedAfterResolution() throws SQLException { + MessageState state = state(MessageState.DeliveryChannel.NONE, "system", null); + saveAndLink(state, "provider-message"); - storage.saveInitialStates(List.of(correlated, replacement)); + MessageState resolved = resolve(state, "provider-message").orElseThrow(); - assertThat(storage.getState(replacement.getGatewayMsgId())) - .get() - .usingRecursiveComparison() - .isEqualTo(replacement); - assertThat(countCorrelations(replacement.getGatewayMsgId())).isZero(); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "superseded-provider-message", MessageState.MessageStatus.DELIVERED)).isEmpty(); + assertThat(resolved.getDlrState()).isEqualTo(StandardMessage.DLR_STAT_DELIVRD); + assertThat(storage.getState(state.getGatewayMsgId())).isEmpty(); + assertThat(countCorrelations(state.getGatewayMsgId())).isZero(); } @Test - void trackedStateAndCorrelationSurviveAdapterRecreation() { - MessageState state = newState(); - storage.saveInitialState(state); - storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-after-restart"); - - PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); - - assertThat(recreated.getState(state.getGatewayMsgId())) - .get() - .extracting(MessageState::getProviderMessageId, MessageState::getStatus) - .containsExactly("provider-message-after-restart", MessageState.MessageStatus.SENT); - assertThat(recreated.resolveAndRemoveDlr( - PROVIDER, "provider-message-after-restart", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId, MessageState::getStatus) - .containsExactly(state.getGatewayMsgId(), MessageState.MessageStatus.DELIVERED); + void pendingSmppDeliveriesAreFilteredBySystemAndOrderedOldestFirst() throws SQLException { + MessageState newer = state(MessageState.DeliveryChannel.SMPP, "system-a", null); + MessageState otherSystem = state(MessageState.DeliveryChannel.SMPP, "system-b", null); + MessageState older = state(MessageState.DeliveryChannel.SMPP, "system-a", null); + saveResolve(newer, "newer"); + saveResolve(otherSystem, "other"); + saveResolve(older, "older"); + setResolvedAt(older.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '2 hours'"); + setResolvedAt(newer.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '1 hour'"); + + assertThat(storage.listPendingSmppDeliveries("system-a")) + .extracting(MessageState::getGatewayMsgId) + .containsExactly(older.getGatewayMsgId(), newer.getGatewayMsgId()); + assertThat(storage.listPendingSmppDeliveries("system-b")) + .extracting(MessageState::getGatewayMsgId) + .containsExactly(otherSystem.getGatewayMsgId()); + assertThat(storage.listPendingSmppDeliveries(" ")).isEmpty(); } @Test - void saveInitialStateOverwritesExistingState() throws SQLException { - MessageState initial = newState(); - storage.saveInitialState(initial); - storage.linkProviderMessageId(initial.getGatewayMsgId(), PROVIDER, "provider-message-old"); - - MessageState replacement = new MessageState(initial.getGatewayMsgId(), "replacement-account", - "replacement-system", "replacement-source", "replacement-destination", null); - replacement.setStatus(MessageState.MessageStatus.FAILED); - storage.saveInitialState(replacement); - - assertThat(storage.getState(initial.getGatewayMsgId())) - .get() - .usingRecursiveComparison() - .isEqualTo(replacement); - assertThat(countCorrelations(initial.getGatewayMsgId())).isZero(); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "provider-message-old", MessageState.MessageStatus.DELIVERED)) - .isEmpty(); + void dueHttpDeliveriesAreFilteredOrderedAndLimited() throws SQLException { + MessageState later = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/later"); + MessageState first = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/first"); + MessageState future = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/future"); + MessageState smpp = state(MessageState.DeliveryChannel.SMPP, "system", null); + saveResolve(later, "later"); + saveResolve(first, "first"); + saveResolve(future, "future"); + saveResolve(smpp, "smpp"); + setNextAttemptAt(first.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '2 hours'"); + setNextAttemptAt(later.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '1 hour'"); + setNextAttemptAt(future.getGatewayMsgId(), "CURRENT_TIMESTAMP + INTERVAL '1 hour'"); + + assertThat(storage.listDueHttpDeliveries(1)) + .extracting(MessageState::getGatewayMsgId) + .containsExactly(first.getGatewayMsgId()); + assertThat(storage.listDueHttpDeliveries(10)) + .extracting(MessageState::getGatewayMsgId) + .containsExactly(first.getGatewayMsgId(), later.getGatewayMsgId()); + assertThatThrownBy(() -> storage.listDueHttpDeliveries(0)) + .isInstanceOf(IllegalArgumentException.class); } @Test - void saveInitialStateRebindsCorrelationOwnedByAnotherMessage() throws SQLException { - MessageState owner = newState(); - storage.saveInitialState(owner); - storage.linkProviderMessageId(owner.getGatewayMsgId(), PROVIDER, "shared-provider-message"); - - MessageState target = newState(); - storage.saveInitialState(target); - storage.linkProviderMessageId(target.getGatewayMsgId(), PROVIDER, "target-provider-message"); - MessageState replacement = new MessageState(target.getGatewayMsgId(), "replacement-account", - "replacement-system", "replacement-source", "replacement-destination", null); - replacement.setProviderName(PROVIDER); - replacement.setProviderMessageId("shared-provider-message"); - storage.saveInitialState(replacement); + void deliveryAttemptIncrementsOnceAndLocalGuardPreventsDuplicateStart() { + MessageState state = pendingHttp("attempt-once"); - assertThat(storage.getState(target.getGatewayMsgId())) - .get() - .usingRecursiveComparison() - .isEqualTo(replacement); - assertThat(storage.getState(owner.getGatewayMsgId()).orElseThrow().getProviderMessageId()) - .isNull(); - assertThat(countCorrelations(target.getGatewayMsgId())).isOne(); - assertThat(countCorrelations(owner.getGatewayMsgId())).isZero(); - - MessageState newConflict = newState(); - newConflict.setProviderName(PROVIDER); - newConflict.setProviderMessageId("shared-provider-message"); - storage.saveInitialState(newConflict); - - assertThat(storage.getState(target.getGatewayMsgId()).orElseThrow().getProviderMessageId()).isNull(); - assertThat(countCorrelations(target.getGatewayMsgId())).isZero(); - assertThat(countCorrelations(newConflict.getGatewayMsgId())).isOne(); + assertThat(storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.SMPP)).isEmpty(); + MessageState attempt = storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); + + assertThat(attempt.getDeliveryAttemptCount()).isOne(); + assertThat(attempt.getLastAttemptAt()).isNotNull(); + assertThat(storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP)).isEmpty(); + assertThat(storage.getState(state.getGatewayMsgId()).orElseThrow().getDeliveryAttemptCount()).isOne(); } @Test - void linkProviderMessageIdUpdatesStateAndKeepsMultipleCorrelations() throws SQLException { - MessageState state = newState(); - storage.saveInitialState(state); - - storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); - storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); - - MessageState linked = storage.getState(state.getGatewayMsgId()).orElseThrow(); - assertThat(linked.getStatus()).isEqualTo(MessageState.MessageStatus.SENT); - assertThat(linked.getProviderMessageId()).isEqualTo("provider-message-2"); - assertThat(countCorrelations(state.getGatewayMsgId())).isEqualTo(2); + void staleAttemptCannotCompleteOrFailNewerAttempt() { + MessageState state = pendingHttp("stale-fence"); + MessageState first = storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); + assertThat(storage.retryDelivery(state.getGatewayMsgId(), first.getDeliveryAttemptCount(), + " first retry ", System.currentTimeMillis())).isTrue(); + MessageState second = storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); + + assertThat(storage.retryDelivery(state.getGatewayMsgId(), first.getDeliveryAttemptCount(), + "stale", System.currentTimeMillis())).isFalse(); + assertThat(storage.completeDelivery(state.getGatewayMsgId(), first.getDeliveryAttemptCount())).isFalse(); + assertThat(storage.failDelivery(state.getGatewayMsgId(), first.getDeliveryAttemptCount(), "stale")) + .isFalse(); + assertThat(storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP)).isEmpty(); + assertThat(storage.failDelivery(state.getGatewayMsgId(), second.getDeliveryAttemptCount(), " final ")) + .isTrue(); + assertThat(storage.getState(state.getGatewayMsgId())).get().satisfies(actual -> { + assertThat(actual.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.FAILED); + assertThat(actual.getLastDeliveryResult()).isEqualTo("final"); + assertThat(actual.getDeliveryAttemptCount()).isEqualTo(2); + }); } @Test - void linkProviderMessageIdRejectsInvalidCorrelation() { - MessageState state = newState(); - storage.saveInitialState(state); + void matchingCompletionDeletesPendingDelivery() { + MessageState state = pendingHttp("complete"); + MessageState attempt = storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); - assertThatThrownBy(() -> storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, null)) - .isInstanceOf(IllegalArgumentException.class); - - MessageState unchanged = storage.getState(state.getGatewayMsgId()).orElseThrow(); - assertThat(unchanged.getStatus()).isEqualTo(MessageState.MessageStatus.ACCEPTED); - assertThat(unchanged.getProviderMessageId()).isNull(); + assertThat(storage.completeDelivery(state.getGatewayMsgId(), attempt.getDeliveryAttemptCount())).isTrue(); + assertThat(storage.getState(state.getGatewayMsgId())).isEmpty(); + assertThat(storage.completeDelivery(state.getGatewayMsgId(), attempt.getDeliveryAttemptCount())).isFalse(); } @Test - void linkProviderMessageIdRebindsCorrelationToNewestMessage() throws SQLException { - MessageState first = new MessageState(UUID.randomUUID().toString(), "first-account", "first-system", - "first-source", "first-destination", null); - MessageState second = new MessageState(UUID.randomUUID().toString(), "second-account", "second-system", - "second-source", "second-destination", null); - storage.saveInitialState(first); - storage.saveInitialState(second); - storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "shared-provider-message"); - - storage.linkProviderMessageId(second.getGatewayMsgId(), PROVIDER, "shared-provider-message"); - - assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getProviderMessageId()) - .isNull(); - MessageState linked = storage.getState(second.getGatewayMsgId()).orElseThrow(); - assertThat(linked.getStatus()).isEqualTo(MessageState.MessageStatus.SENT); - assertThat(linked.getProviderMessageId()).isEqualTo("shared-provider-message"); - assertThat(countCorrelations(first.getGatewayMsgId())).isZero(); - assertThat(countCorrelations(second.getGatewayMsgId())).isOne(); - - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId, MessageState::getAccountId) - .containsExactly(second.getGatewayMsgId(), "second-account"); - assertThat(storage.getState(first.getGatewayMsgId())).isPresent(); + void retryStoresNormalizedResultAndSchedulesNextAttempt() { + MessageState state = pendingHttp("retry"); + MessageState attempt = storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); + long nextAttemptAt = System.currentTimeMillis() + Duration.ofHours(1).toMillis(); + + assertThat(storage.retryDelivery(state.getGatewayMsgId(), attempt.getDeliveryAttemptCount(), + " timeout ", nextAttemptAt)).isTrue(); + + assertThat(storage.getState(state.getGatewayMsgId())).get().satisfies(actual -> { + assertThat(actual.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.PENDING); + assertThat(actual.getLastDeliveryResult()).isEqualTo("timeout"); + assertThat(actual.getNextAttemptAt()).isEqualTo(nextAttemptAt); + }); + assertThat(storage.listDueHttpDeliveries(10)).isEmpty(); } @Test - void sameMessageIdFromDifferentProvidersResolvesIndependently() { - MessageState first = newState(); - MessageState second = newState(); - storage.saveInitialStates(List.of(first, second)); + void invalidDeliveryFailsWithoutIncrementingAttempts() { + MessageState state = pendingHttp("invalid"); - storage.linkProviderMessageId(first.getGatewayMsgId(), "provider-a", "shared-provider-message"); - storage.linkProviderMessageId(second.getGatewayMsgId(), "provider-b", "shared-provider-message"); + assertThat(storage.failInvalidDelivery(state.getGatewayMsgId(), " missing URL ")).isTrue(); - assertThat(storage.resolveAndRemoveDlr( - "provider-a", "shared-provider-message", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId, MessageState::getProviderName, - MessageState::getProviderMessageId) - .containsExactly(first.getGatewayMsgId(), "provider-a", "shared-provider-message"); - assertThat(storage.resolveAndRemoveDlr( - "provider-b", "shared-provider-message", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId, MessageState::getProviderName, - MessageState::getProviderMessageId) - .containsExactly(second.getGatewayMsgId(), "provider-b", "shared-provider-message"); + assertThat(storage.getState(state.getGatewayMsgId())).get().satisfies(actual -> { + assertThat(actual.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.FAILED); + assertThat(actual.getDeliveryAttemptCount()).isZero(); + assertThat(actual.getLastDeliveryResult()).isEqualTo("missing URL"); + }); } @Test - void concurrentProviderMessageIdRebindLeavesOneConsistentOwner() throws Exception { - MessageState first = newState(); - MessageState second = newState(); - storage.saveInitialState(first); - storage.saveInitialState(second); - CountDownLatch ready = new CountDownLatch(2); - CountDownLatch start = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); + void adapterRecreationCanRetryAttemptThatWasActiveBeforeCrash() { + MessageState state = pendingHttp("adapter-recreation"); + assertThat(storage.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP)).isPresent(); - try { - Future firstLink = executor.submit(() -> { - linkWhenReleased(first.getGatewayMsgId(), "shared-provider-message", ready, start); - return null; - }); - Future secondLink = executor.submit(() -> { - linkWhenReleased(second.getGatewayMsgId(), "shared-provider-message", ready, start); - return null; - }); - ready.await(); - start.countDown(); - firstLink.get(); - secondLink.get(); - - MessageState firstAfter = storage.getState(first.getGatewayMsgId()).orElseThrow(); - MessageState secondAfter = storage.getState(second.getGatewayMsgId()).orElseThrow(); - assertThat(List.of(firstAfter, secondAfter).stream() - .filter(state -> "shared-provider-message".equals(state.getProviderMessageId()))) - .hasSize(1); - assertThat(countCorrelations(first.getGatewayMsgId()) + countCorrelations(second.getGatewayMsgId())) - .isOne(); - String owner = storage.resolveAndRemoveDlr( - PROVIDER, "shared-provider-message", MessageState.MessageStatus.DELIVERED) - .orElseThrow() - .getGatewayMsgId(); - assertThat(owner).isIn(first.getGatewayMsgId(), second.getGatewayMsgId()); - } finally { - executor.shutdownNow(); - } + PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); + MessageState retried = recreated.startDeliveryAttempt( + state.getGatewayMsgId(), MessageState.DeliveryChannel.HTTP).orElseThrow(); + + assertThat(retried.getDeliveryAttemptCount()).isEqualTo(2); } @Test - void concurrentRebindAndResolveCompleteWithoutDeadlock() throws Exception { - MessageState first = newState(); - MessageState second = newState(); - storage.saveInitialStates(List.of(first, second)); - storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "shared-provider-message"); - CountDownLatch ready = new CountDownLatch(2); - CountDownLatch start = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); + void initialSaveAndLinkCannotOverwriteOrRelinkTerminalRow() throws SQLException { + MessageState terminal = pendingHttp("terminal-guard"); + MessageState replacement = new MessageState(terminal.getGatewayMsgId(), "replacement-account", + "replacement-system", "replacement-source", "replacement-destination", + "https://example.test/replacement"); + replacement.setDeliveryChannel(MessageState.DeliveryChannel.HTTP); - try { - Future rebind = executor.submit(() -> { - linkWhenReleased(second.getGatewayMsgId(), "shared-provider-message", ready, start); - return null; - }); - Future> resolve = executor.submit( - () -> resolveWhenReleased("shared-provider-message", ready, start)); - ready.await(); - start.countDown(); + storage.saveInitialState(replacement); + PostgresqlDlrStorage noRetryStorage = new PostgresqlDlrStorage(dataSource, 1, 0); - rebind.get(); - assertThat(resolve.get()).isPresent(); - assertThat(countCorrelations(first.getGatewayMsgId()) - + countCorrelations(second.getGatewayMsgId())).isLessThanOrEqualTo(1); - } finally { - executor.shutdownNow(); - } + assertThatThrownBy(() -> noRetryStorage.linkProviderMessageId( + terminal.getGatewayMsgId(), PROVIDER, "new-provider-message")) + .isInstanceOf(DlrStorageException.class); + assertThat(storage.getState(terminal.getGatewayMsgId())).get().satisfies(actual -> { + assertThat(actual.getAccountId()).isEqualTo(terminal.getAccountId()); + assertThat(actual.getDlrState()).isEqualTo(StandardMessage.DLR_STAT_DELIVRD); + assertThat(actual.getDeliveryStatus()).isEqualTo(MessageState.DeliveryStatus.PENDING); + }); + assertThat(countCorrelations(terminal.getGatewayMsgId())).isZero(); } @Test - void concurrentCrossedRebindsLockMessagesInConsistentOrder() throws Exception { - MessageState first = newState(); - MessageState second = newState(); - storage.saveInitialState(first); - storage.saveInitialState(second); - storage.linkProviderMessageId(first.getGatewayMsgId(), PROVIDER, "provider-message-2"); - storage.linkProviderMessageId(second.getGatewayMsgId(), PROVIDER, "provider-message-1"); - CountDownLatch ready = new CountDownLatch(2); - CountDownLatch start = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); + void retentionUsesCreatedAtWhileWaitingAndResolvedAtAfterResolution() throws SQLException { + MessageState oldWaiting = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/waiting"); + storage.saveInitialState(oldWaiting); + setCreatedAt(oldWaiting.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '8 days'"); - try { - Future firstLink = executor.submit(() -> { - linkWhenReleased(first.getGatewayMsgId(), "provider-message-1", ready, start); - return null; - }); - Future secondLink = executor.submit(() -> { - linkWhenReleased(second.getGatewayMsgId(), "provider-message-2", ready, start); - return null; - }); - ready.await(); - start.countDown(); - firstLink.get(); - secondLink.get(); - - assertThat(storage.getState(first.getGatewayMsgId()).orElseThrow().getProviderMessageId()) - .isEqualTo("provider-message-1"); - assertThat(storage.getState(second.getGatewayMsgId()).orElseThrow().getProviderMessageId()) - .isEqualTo("provider-message-2"); - assertThat(countCorrelations(first.getGatewayMsgId())).isOne(); - assertThat(countCorrelations(second.getGatewayMsgId())).isOne(); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "provider-message-1", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId) - .isEqualTo(first.getGatewayMsgId()); - assertThat(storage.resolveAndRemoveDlr( - PROVIDER, "provider-message-2", MessageState.MessageStatus.DELIVERED)) - .get() - .extracting(MessageState::getGatewayMsgId) - .isEqualTo(second.getGatewayMsgId()); - } finally { - executor.shutdownNow(); - } - } + MessageState freshPendingWithOldCreation = pendingHttp("fresh-pending"); + setCreatedAt(freshPendingWithOldCreation.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '8 days'"); - @Test - void linkProviderMessageIdFailsWhenGatewayStateDoesNotAppear() { - PostgresqlDlrStorage noRetryStorage = - new PostgresqlDlrStorage(dataSource, 1, 0); + MessageState oldPending = pendingHttp("old-pending"); + setResolvedAt(oldPending.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '8 days'"); - assertThatThrownBy(() -> noRetryStorage.linkProviderMessageId( - UUID.randomUUID().toString(), PROVIDER, "provider-message")) - .isInstanceOf(DlrStorageException.class) - .hasMessageContaining("not found"); + MessageState freshFailed = pendingHttp("fresh-failed"); + storage.failInvalidDelivery(freshFailed.getGatewayMsgId(), "invalid"); + + MessageState oldFailed = pendingHttp("old-failed"); + storage.failInvalidDelivery(oldFailed.getGatewayMsgId(), "invalid"); + setResolvedAt(oldFailed.getGatewayMsgId(), "CURRENT_TIMESTAMP - INTERVAL '8 days'"); + + PostgresqlDlrStorage cleanup = new PostgresqlDlrStorage(dataSource, 1, 0, 0); + cleanup.getState(freshPendingWithOldCreation.getGatewayMsgId()); + + assertThat(cleanup.getState(oldWaiting.getGatewayMsgId())).isEmpty(); + assertThat(cleanup.getState(oldPending.getGatewayMsgId())).isEmpty(); + assertThat(cleanup.getState(oldFailed.getGatewayMsgId())).isEmpty(); + assertThat(cleanup.getState(freshPendingWithOldCreation.getGatewayMsgId())).isPresent(); + assertThat(cleanup.getState(freshFailed.getGatewayMsgId())).isPresent(); } @Test - void resolveAndRemoveDlrReturnsUpdatedStateAndDeletesAllCorrelations() throws SQLException { - MessageState state = newState(); - storage.saveInitialState(state); - storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); - storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); - long beforeResolve = System.currentTimeMillis(); - - Optional resolved = storage.resolveAndRemoveDlr( - PROVIDER, "provider-message-1", MessageState.MessageStatus.DELIVERED); + void correlationRetentionRemainsThreeDays() throws SQLException { + MessageState state = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/dlr"); + saveAndLink(state, "old-correlation"); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + UPDATE sendium_dlr.provider_correlation + SET created_at = CURRENT_TIMESTAMP - INTERVAL '4 days' + WHERE provider_name = ? AND provider_message_id = ? + """)) { + statement.setString(1, PROVIDER); + statement.setString(2, "old-correlation"); + statement.executeUpdate(); + } - assertThat(resolved).isPresent(); - assertThat(resolved.orElseThrow().getStatus()).isEqualTo(MessageState.MessageStatus.DELIVERED); - assertThat(resolved.orElseThrow().getProviderMessageId()).isEqualTo("provider-message-1"); - assertThat(resolved.orElseThrow().getTimestamp()).isGreaterThanOrEqualTo(beforeResolve); - assertThat(storage.getState(state.getGatewayMsgId())).isEmpty(); + PostgresqlDlrStorage cleanup = new PostgresqlDlrStorage(dataSource, 1, 0, 0); + assertThat(cleanup.getState(state.getGatewayMsgId())).isPresent(); assertThat(countCorrelations(state.getGatewayMsgId())).isZero(); } @Test - void concurrentResolveAcrossCorrelationsReturnsStateOnlyOnce() throws Exception { - MessageState state = newState(); + void concurrentTerminalReceiptsResolveMessageOnlyOnce() throws Exception { + MessageState state = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/dlr"); storage.saveInitialState(state); storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-1"); storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, "provider-message-2"); @@ -462,256 +350,52 @@ void concurrentResolveAcrossCorrelationsReturnsStateOnlyOnce() throws Exception ready.await(); start.countDown(); - assertThat(List.of(first.get(), second.get()).stream().filter(Optional::isPresent).count()) - .isOne(); + assertThat(List.of(first.get(), second.get()).stream().filter(Optional::isPresent).count()).isOne(); } finally { executor.shutdownNow(); } } - @Test - void markAsFailedUpdatesExistingState() { - MessageState state = newState(); - storage.saveInitialState(state); - - boolean updated = storage.markAsFailed(state.getGatewayMsgId()); - - assertThat(updated).isTrue(); - assertThat(storage.getState(state.getGatewayMsgId()).orElseThrow().getStatus()) - .isEqualTo(MessageState.MessageStatus.FAILED); - assertThat(storage.markAsFailed(UUID.randomUUID().toString())).isFalse(); - } - - @Test - void expiryRemovesOldCorrelationsAndMessages() throws SQLException { - MessageState correlationState = newState(); - storage.saveInitialState(correlationState); - storage.linkProviderMessageId(correlationState.getGatewayMsgId(), PROVIDER, "old-correlation"); - ageCorrelation("old-correlation"); - - PostgresqlDlrStorage correlationCleanup = new PostgresqlDlrStorage(dataSource); - assertThat(correlationCleanup.getState(correlationState.getGatewayMsgId())).isPresent(); - assertThat(countCorrelations(correlationState.getGatewayMsgId())).isZero(); - - MessageState oldMessage = newState(); - storage.saveInitialState(oldMessage); - ageMessage(oldMessage.getGatewayMsgId()); - - PostgresqlDlrStorage messageCleanup = new PostgresqlDlrStorage(dataSource); - assertThat(messageCleanup.getState(oldMessage.getGatewayMsgId())).isEmpty(); - } - - @Test - void saveUnpushedDlrRoundTripsAllPersistedFields() { - StandardMessage dlr = newDlr("account-1", "system-1"); - - assertThat(storage.saveUnpushedDlr(dlr)).isTrue(); - - List stored = storage.getUnpushedDlrs("system-1"); - assertThat(stored).singleElement().satisfies(actual -> { - assertThat(actual.type).isEqualTo(StandardMessage.MSG_DLR); - assertThat(actual.systemId).isEqualTo(dlr.systemId); - assertThat(actual.owner_id).isEqualTo(dlr.owner_id); - assertThat(actual.from).isEqualTo(dlr.from); - assertThat(actual.to).isEqualTo(dlr.to); - assertThat(actual.serial).isEqualTo(dlr.serial); - assertThat(actual.msgId).isEqualTo(dlr.msgId); - assertThat(actual.state).isEqualTo(dlr.state); - assertThat(actual.errcode).isEqualTo(dlr.errcode); - assertThat(actual.acked).isEqualTo(dlr.acked); - assertThat(actual.priority).isEqualTo(dlr.priority); - assertThat(actual.reassembledParts).containsExactlyElementsOf(dlr.reassembledParts); - }); - } - - @Test - void saveUnpushedDlrRejectsInvalidMessages() throws SQLException { - StandardMessage wrongType = newDlr("account-1", "system-1"); - wrongType.type = StandardMessage.MSG_TEXT; - StandardMessage blankSystem = newDlr("account-1", " "); - - assertThat(storage.saveUnpushedDlr(null)).isFalse(); - assertThat(storage.saveUnpushedDlr(wrongType)).isFalse(); - assertThat(storage.saveUnpushedDlr(blankSystem)).isFalse(); - assertThat(countUnpushedDlrs()).isZero(); - } - - @Test - void saveUnpushedDlrOverwritesSameReplayKey() throws SQLException { - StandardMessage initial = newDlr("account-1", "system-1"); - storage.saveUnpushedDlr(initial); - - StandardMessage replacement = newDlr("replacement-account", initial.systemId); - replacement.serial = initial.serial; - replacement.msgId = initial.msgId; - replacement.state = initial.state; - replacement.errcode = initial.errcode; - replacement.priority = 9; - replacement.reassembledParts = new ArrayList<>(List.of("replacement-part")); - storage.saveUnpushedDlr(replacement); - - assertThat(countUnpushedDlrs()).isOne(); - assertThat(storage.getUnpushedDlrs(initial.systemId)) - .singleElement() - .satisfies(actual -> { - assertThat(actual.owner_id).isEqualTo("replacement-account"); - assertThat(actual.priority).isEqualTo(9); - assertThat(actual.reassembledParts).containsExactly("replacement-part"); - }); - } - - @Test - void unpushedDlrsAreFilteredAndSurviveAdapterRecreation() { - StandardMessage first = newDlr("account-1", "system-1"); - StandardMessage second = newDlr("account-2", "system-2"); - storage.saveUnpushedDlr(first); - storage.saveUnpushedDlr(second); - - PostgresqlDlrStorage recreated = new PostgresqlDlrStorage(dataSource); - - assertThat(recreated.getUnpushedDlrs("system-1")) - .extracting(message -> message.serial) - .containsExactly(first.serial); - assertThat(recreated.getUnpushedDlrs("system-2")) - .extracting(message -> message.serial) - .containsExactly(second.serial); - assertThat(recreated.getUnpushedDlrs("missing-system")).isEmpty(); - assertThat(recreated.claimUnpushedDlrs("system-1")) - .extracting(message -> message.serial) - .containsExactly(first.serial); - assertThat(recreated.claimUnpushedDlrs("system-1")).isEmpty(); - } - - @Test - void claimHidesReceiptUntilReleasedOrRemoved() { - StandardMessage dlr = newDlr("account-1", "system-1"); - storage.saveUnpushedDlr(dlr); - - List firstClaim = storage.claimUnpushedDlrs(dlr.systemId); - - assertThat(firstClaim).hasSize(1); - assertThat(storage.claimUnpushedDlrs(dlr.systemId)).isEmpty(); - assertThat(storage.getUnpushedDlrs(dlr.systemId)).hasSize(1); - - storage.releaseUnpushedDlrClaim(firstClaim.getFirst()); - List releasedClaim = storage.claimUnpushedDlrs(dlr.systemId); - assertThat(releasedClaim).hasSize(1); - assertThat(storage.removeUnpushedDlr(releasedClaim.getFirst())).isTrue(); - assertThat(storage.removeUnpushedDlr(releasedClaim.getFirst())).isFalse(); - assertThat(storage.getUnpushedDlrs(dlr.systemId)).isEmpty(); + private MessageState pendingHttp(String providerMessageId) { + MessageState state = state(MessageState.DeliveryChannel.HTTP, "system", "https://example.test/dlr"); + saveResolve(state, providerMessageId); + return state; } - @Test - void staleClaimCannotRemoveOrReleaseReplacementGeneration() { - StandardMessage original = newDlr("account-1", "system-1"); - storage.saveUnpushedDlr(original); - StandardMessage staleClaim = storage.claimUnpushedDlrs(original.systemId).getFirst(); - StandardMessage replacement = newDlr("account-2", original.systemId); - replacement.serial = original.serial; - replacement.msgId = original.msgId; - replacement.state = original.state; - replacement.errcode = original.errcode; - storage.saveUnpushedDlr(replacement); - - assertThat(storage.removeUnpushedDlr(staleClaim)).isFalse(); - List replacementClaim = storage.claimUnpushedDlrs(original.systemId); - assertThat(replacementClaim).hasSize(1); - assertThat(replacementClaim.getFirst().owner_id).isEqualTo("account-2"); - - storage.releaseUnpushedDlrClaim(staleClaim); - assertThat(storage.claimUnpushedDlrs(original.systemId)).isEmpty(); - storage.releaseUnpushedDlrClaim(replacementClaim.getFirst()); - assertThat(storage.claimUnpushedDlrs(original.systemId)).hasSize(1); + private void saveResolve(MessageState state, String providerMessageId) { + saveAndLink(state, providerMessageId); + resolve(state, providerMessageId).orElseThrow(); } - @Test - void concurrentClaimsReturnReceiptOnlyOnce() throws Exception { - StandardMessage dlr = newDlr("account-1", "system-1"); - storage.saveUnpushedDlr(dlr); - CountDownLatch ready = new CountDownLatch(2); - CountDownLatch start = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); - - try { - Future> first = executor.submit( - () -> claimWhenReleased(dlr.systemId, ready, start)); - Future> second = executor.submit( - () -> claimWhenReleased(dlr.systemId, ready, start)); - ready.await(); - start.countDown(); - - assertThat(List.of(first.get(), second.get()).stream().filter(claim -> !claim.isEmpty()).count()) - .isOne(); - } finally { - executor.shutdownNow(); - } + private void saveAndLink(MessageState state, String providerMessageId) { + storage.saveInitialState(state); + storage.linkProviderMessageId(state.getGatewayMsgId(), PROVIDER, providerMessageId); } - @Test - void expiryRemovesOldUnpushedDlrsAndTheirClaims() throws SQLException { - StandardMessage dlr = newDlr("account-1", "system-1"); - storage.saveUnpushedDlr(dlr); - storage = new PostgresqlDlrStorage(dataSource, 20, 200, 0); - assertThat(storage.claimUnpushedDlrs(dlr.systemId)).hasSize(1); - ageUnpushedDlr(dlr.serial); - - assertThat(storage.getUnpushedDlrs(dlr.systemId)).isEmpty(); - assertThat(countUnpushedDlrs()).isZero(); - - assertThat(storage.saveUnpushedDlr(dlr)).isTrue(); - assertThat(storage.claimUnpushedDlrs(dlr.systemId)).hasSize(1); - assertThat(storage.claimUnpushedDlrs(dlr.systemId)).isEmpty(); + private Optional resolve(MessageState state, String providerMessageId) { + return storage.resolveDlr(PROVIDER, providerMessageId, MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_DELIVRD, "000"); } private Optional resolveWhenReleased(String providerMessageId, CountDownLatch ready, - CountDownLatch start) throws InterruptedException { - ready.countDown(); - start.await(); - return storage.resolveAndRemoveDlr(PROVIDER, providerMessageId, MessageState.MessageStatus.DELIVERED); - } - - private void linkWhenReleased(String gatewayMsgId, String providerMessageId, CountDownLatch ready, - CountDownLatch start) throws InterruptedException { + CountDownLatch start) throws InterruptedException { ready.countDown(); start.await(); - storage.linkProviderMessageId(gatewayMsgId, PROVIDER, providerMessageId); + return storage.resolveDlr(PROVIDER, providerMessageId, MessageState.MessageStatus.DELIVERED, + StandardMessage.DLR_STAT_DELIVRD, "000"); } - private List claimWhenReleased(String systemId, CountDownLatch ready, - CountDownLatch start) throws InterruptedException { - ready.countDown(); - start.await(); - return storage.claimUnpushedDlrs(systemId); - } - - private MessageState newState() { - return new MessageState(UUID.randomUUID().toString(), "account", "system", "source", "destination", - "https://example.test/dlr"); - } - - private StandardMessage newDlr(String accountId, String systemId) { - StandardMessage dlr = new StandardMessage(); - dlr.type = StandardMessage.MSG_DLR; - dlr.systemId = systemId; - dlr.owner_id = accountId; - dlr.from = "source"; - dlr.to = "destination"; - dlr.serial = UUID.randomUUID().toString(); - dlr.msgId = 42; - dlr.state = 1; - dlr.errcode = "000"; - dlr.acked = true; - dlr.priority = 3; - dlr.reassembledParts = new ArrayList<>(List.of("part-1", "part-2")); - return dlr; + private MessageState state(MessageState.DeliveryChannel channel, String systemId, String callbackUrl) { + MessageState state = new MessageState(UUID.randomUUID().toString(), "account", systemId, + "source", "destination", callbackUrl); + state.setDeliveryChannel(channel); + return state; } private int countCorrelations(String gatewayMsgId) throws SQLException { try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(""" - SELECT COUNT(*) - FROM sendium_dlr.provider_correlation - WHERE gateway_message_id = ? + SELECT COUNT(*) FROM sendium_dlr.provider_correlation WHERE gateway_message_id = ? """)) { statement.setObject(1, UUID.fromString(gatewayMsgId)); try (ResultSet resultSet = statement.executeQuery()) { @@ -721,48 +405,24 @@ SELECT COUNT(*) } } - private int countUnpushedDlrs() throws SQLException { - try (Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM sendium_dlr.unpushed_dlr")) { - resultSet.next(); - return resultSet.getInt(1); - } + private void setCreatedAt(String gatewayMsgId, String expression) throws SQLException { + updateTimestamp(gatewayMsgId, "created_at", expression); } - private void ageCorrelation(String providerMessageId) throws SQLException { - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(""" - UPDATE sendium_dlr.provider_correlation - SET created_at = CURRENT_TIMESTAMP - INTERVAL '4 days' - WHERE provider_name = ? AND provider_message_id = ? - """)) { - statement.setString(1, PROVIDER); - statement.setString(2, providerMessageId); - statement.executeUpdate(); - } + private void setResolvedAt(String gatewayMsgId, String expression) throws SQLException { + updateTimestamp(gatewayMsgId, "resolved_at", expression); } - private void ageMessage(String gatewayMsgId) throws SQLException { - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(""" - UPDATE sendium_dlr.tracked_message - SET created_at = CURRENT_TIMESTAMP - INTERVAL '8 days' - WHERE gateway_message_id = ? - """)) { - statement.setObject(1, UUID.fromString(gatewayMsgId)); - statement.executeUpdate(); - } + private void setNextAttemptAt(String gatewayMsgId, String expression) throws SQLException { + updateTimestamp(gatewayMsgId, "next_attempt_at", expression); } - private void ageUnpushedDlr(String serial) throws SQLException { + private void updateTimestamp(String gatewayMsgId, String column, String expression) throws SQLException { + String sql = "UPDATE sendium_dlr.dlr_message SET " + column + " = " + expression + + " WHERE gateway_message_id = ?"; try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(""" - UPDATE sendium_dlr.unpushed_dlr - SET created_at = CURRENT_TIMESTAMP - INTERVAL '8 days' - WHERE serial = ? - """)) { - statement.setString(1, serial); + PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setObject(1, UUID.fromString(gatewayMsgId)); statement.executeUpdate(); } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java index 9169f0e..3d7a5c8 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java @@ -103,7 +103,7 @@ void updateSendStatusAndExtID_WhenPersistenceDisabled_SkipsLinking() { @Test void createAndEnqueueDLR_WhenStorageFails_PropagatesToProtocolBoundary() throws InterruptedException { - when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 0)) + when(dlrService.resolveDlr("provider-1", "provider-message-456", 0, "0")) .thenThrow(new DlrStorageException("Failed to resolve DLR state")); assertThrows(DlrStorageException.class, () -> tracker.createAndEnqueueDLR( @@ -140,15 +140,16 @@ void getHashedMessageID_NullInput_ReturnsEmpty() { } @Test - void createAndEnqueueDLR_KnownMessage_ResolvesFromDlrService() throws InterruptedException { + void createAndEnqueueDLR_SmppMessage_ResolvesAndEnqueues() throws InterruptedException { MessageState state = new MessageState("gw-123", "accountId", "systemId", "from", "to", null); - when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 0)) + state.setDeliveryChannel(MessageState.DeliveryChannel.SMPP); + when(dlrService.resolveDlr("provider-1", "provider-message-456", 0, "0")) .thenReturn(java.util.Optional.of(state)); tracker.createAndEnqueueDLR( 1, "provider-message-456", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); - verify(dlrService).resolveAndRemoveDlr("provider-1", "provider-message-456", 0); + verify(dlrService).resolveDlr("provider-1", "provider-message-456", 0, "0"); ArgumentCaptor captor = ArgumentCaptor.forClass(StandardMessage.class); verify(outWorker).enqueueToRouter(captor.capture()); assertEquals("accountId", captor.getValue().owner_id); @@ -158,8 +159,9 @@ void createAndEnqueueDLR_KnownMessage_ResolvesFromDlrService() throws Interrupte @Test void createAndEnqueueDLR_KnownReassembledMessage_RestoresPartIds() throws InterruptedException { MessageState state = new MessageState("gw-123", "accountId", "systemId", "from", "to", null); + state.setDeliveryChannel(MessageState.DeliveryChannel.SMPP); state.setReassembledParts(new ArrayList<>(List.of("part-1", "part-2"))); - when(dlrService.resolveAndRemoveDlr("provider-1", "provider-message-456", 1)) + when(dlrService.resolveDlr("provider-1", "provider-message-456", 1, "0")) .thenReturn(java.util.Optional.of(state)); tracker.createAndEnqueueDLR( @@ -170,14 +172,41 @@ void createAndEnqueueDLR_KnownReassembledMessage_RestoresPartIds() throws Interr assertEquals(List.of("part-1", "part-2"), captor.getValue().reassembledParts); } + @Test + void createAndEnqueueDLR_HttpMessage_DoesNotEnqueue() throws InterruptedException { + MessageState state = new MessageState("gw-http", "accountId", "systemId", "from", "to", + "https://example.test/dlr"); + state.setDeliveryChannel(MessageState.DeliveryChannel.HTTP); + when(dlrService.resolveDlr("provider-1", "provider-http", 1, "0")) + .thenReturn(java.util.Optional.of(state)); + + tracker.createAndEnqueueDLR( + 1, "provider-http", "gw-http", "from", "to", "test body", 1, "0", new HashMap<>()); + + verify(outWorker, never()).enqueueToRouter(any()); + } + + @Test + void createAndEnqueueDLR_NoneMessage_DoesNotEnqueue() throws InterruptedException { + MessageState state = new MessageState("gw-none", "accountId", "systemId", "from", "to", null); + state.setDeliveryChannel(MessageState.DeliveryChannel.NONE); + when(dlrService.resolveDlr("provider-1", "provider-none", 1, "0")) + .thenReturn(java.util.Optional.of(state)); + + tracker.createAndEnqueueDLR( + 1, "provider-none", "gw-none", "from", "to", "test body", 1, "0", new HashMap<>()); + + verify(outWorker, never()).enqueueToRouter(any()); + } + @Test void createAndEnqueueDLR_UnknownMessage_DoesNotEnqueue() { - when(dlrService.resolveAndRemoveDlr("provider-1", "unknown", 0)) + when(dlrService.resolveDlr("provider-1", "unknown", 0, "0")) .thenReturn(java.util.Optional.empty()); tracker.createAndEnqueueDLR(1, "unknown", "gw-123", "from", "to", "test body", 0, "0", new HashMap<>()); - verify(dlrService).resolveAndRemoveDlr("provider-1", "unknown", 0); + verify(dlrService).resolveDlr("provider-1", "unknown", 0, "0"); } } diff --git a/sendium-core/src/test/java/utils/NativeE2eSmoke.java b/sendium-core/src/test/java/utils/NativeE2eSmoke.java index 8802e11..cc25b19 100644 --- a/sendium-core/src/test/java/utils/NativeE2eSmoke.java +++ b/sendium-core/src/test/java/utils/NativeE2eSmoke.java @@ -62,7 +62,6 @@ public class NativeE2eSmoke { private static final int SENDIUM_SMPP_PORT = 27777; private static final int UPSTREAM_SMPP_PORT = 27779; private static final Duration TIMEOUT = Duration.ofSeconds(90); - private static final Duration NO_DELIVERY_TIMEOUT = Duration.ofSeconds(3); public static void main(String[] args) throws Exception { String containerName = "sendium-e2e-" + UUID.randomUUID().toString().substring(0, 8); @@ -80,9 +79,8 @@ public static void main(String[] args) throws Exception { waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); require(upstream.awaitSessionBound(), "Sendium container did not bind to the upstream SMPP server"); - container = verifyUnpushedDlrSurvivesRestart(containerName, workDir, upstream); - container = verifyHttpCorrelationSurvivesRestart(containerName, workDir, upstream, callbackServer, 2); - verifySmppSubmitGetsDeliverSm(upstream, 3); + container = verifyHttpCorrelationSurvivesRestart(containerName, workDir, upstream, callbackServer, 1); + verifySmppSubmitGetsDeliverSm(upstream, 2); } catch (Throwable t) { printDockerLogs(containerName); throw t; @@ -161,62 +159,6 @@ private static Process verifyHttpCorrelationSurvivesRestart(String containerName } } - private static Process verifyUnpushedDlrSurvivesRestart(String containerName, Path workDir, - UpstreamSmppServer upstream) throws Exception { - upstream.setAutomaticDelivery(false); - String gatewayId; - try (DownstreamSmppClient client = new DownstreamSmppClient()) { - client.start(); - SubmitSmResp response = client.sendSms("smpp-sender", "306900000003", "container restart dlr e2e"); - require(response.getCommandStatus() == SmppConstants.STATUS_OK, - "SMPP restart submit_sm_resp status was " + response.getCommandStatus()); - gatewayId = response.getMessageId(); - require(gatewayId != null && !gatewayId.isBlank(), "SMPP restart submit_sm_resp did not contain a message id"); - require(upstream.awaitSubmitCount(1), "Upstream SMPP server did not receive the restart test message"); - awaitSuccessfulStorageOperation("link_provider"); - } - - Thread.sleep(500); - upstream.sendDeliveryReceipt(1); - awaitSuccessfulStorageOperation("save_unpushed"); - upstream.setAutomaticDelivery(true); - int boundSessionsBeforeRestart = upstream.boundSessionCount(); - stopContainer(containerName); - - Process replayContainer = startSendiumContainer(containerName, workDir); - waitForPostgresqlReadiness(); - waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); - require(upstream.awaitSessionBoundAfter(boundSessionsBeforeRestart), - "Sendium container did not rebind to upstream after restart"); - - try (DownstreamSmppClient reconnectedClient = new DownstreamSmppClient()) { - reconnectedClient.start(); - DeliverSm deliverSm = reconnectedClient.awaitDeliverSm(); - require(deliverSm != null, "Reconnected downstream SMPP client did not receive persisted unpushed DLR"); - String body = new String(deliverSm.getShortMessage(), StandardCharsets.UTF_8); - require(body.contains("DELIVRD"), "Persisted unpushed DLR was not delivered: " + body); - require(body.contains("id:" + gatewayId), - "Persisted unpushed DLR did not contain gateway id " + gatewayId + ": " + body); - awaitSuccessfulStorageOperation("remove_unpushed"); - } - - boundSessionsBeforeRestart = upstream.boundSessionCount(); - stopContainer(containerName); - replayContainer.destroyForcibly(); - Process container = startSendiumContainer(containerName, workDir); - waitForPostgresqlReadiness(); - waitForPort("localhost", SENDIUM_SMPP_PORT, TIMEOUT); - require(upstream.awaitSessionBoundAfter(boundSessionsBeforeRestart), - "Sendium container did not rebind to upstream after replay check"); - try (DownstreamSmppClient client = new DownstreamSmppClient()) { - client.start(); - awaitSuccessfulStorageOperation("claim_unpushed"); - require(client.awaitDeliverSm(NO_DELIVERY_TIMEOUT) == null, - "Replayed unpushed DLR was delivered more than once after restart"); - } - return container; - } - private static Process startSendiumContainer(String containerName, Path workDir) throws Exception { List command = List.of( "docker", "run", "--rm", "-d", From 24f08ad0595a6a7c4606d2f73bb950c289dd1c8c Mon Sep 17 00:00:00 2001 From: pavlos Date: Fri, 21 Aug 2026 17:07:14 +0300 Subject: [PATCH 20/20] refactor(dlr): defer delivery observability --- docs/01-architecture.md | 32 +++++++--- docs/07-webhooks.md | 12 ++-- docs/08-monitoring-observability.md | 2 +- docs/13-dlr-persistence.md | 21 +++++-- .../core/smpp/server/DlrDeliveryBatch.java | 4 ++ .../core/smpp/server/tasks/OutTask.java | 9 ++- .../core/worker/ForwardDlrService.java | 56 +++-------------- .../core/worker/ForwardDlrServiceTest.java | 60 +++++-------------- 8 files changed, 80 insertions(+), 116 deletions(-) diff --git a/docs/01-architecture.md b/docs/01-architecture.md index 7674886..b3ad2b2 100644 --- a/docs/01-architecture.md +++ b/docs/01-architecture.md @@ -151,9 +151,9 @@ sequenceDiagram ## DLR Handling -Outbound HTTP messages can include a Kannel-style `dlr-url`. Before accepting a submission, Sendium stores its gateway message ID. After the upstream SMSC returns `submit_sm_resp`, the client worker links that gateway ID to the exact `(provider name, provider message ID)` pair. The provider name defaults to the worker's full name; workers sharing an SMSC message-ID namespace can use the same `msg.hash.prefix`. +Outbound HTTP messages can include a Kannel-style `dlr-url`, while downstream SMPP submissions request receipts through `registered_delivery`. Before accepting either submission, Sendium stores one `dlr_message` row containing the gateway message ID and downstream delivery target. After the upstream SMSC returns `submit_sm_resp`, the client worker links that gateway ID to the exact `(provider name, provider message ID)` pair in `provider_correlation`. The provider name defaults to the worker's full name; workers sharing an SMSC message-ID namespace can use the same `msg.hash.prefix`. -Different providers can reuse the same message ID independently. Reusing the same pair within one provider moves the correlation to the newest gateway message and clears it from the previous owner. Link and resolve transactions take a composite-key advisory lock and lock affected gateway rows in canonical UUID order, preventing crossed rebind and resolve deadlocks. Resolving a receipt transactionally consumes the tracked message and all of its correlations before callback forwarding or internal DLR queueing. +Different providers can reuse the same message ID independently. Reusing the same pair within one provider moves the correlation to the newest gateway message and clears it from the previous owner. Link and resolve transactions take a composite-key advisory lock and lock affected gateway rows in canonical UUID order, preventing crossed rebind and resolve deadlocks. Intermediate `ACCEPTD` and `ENROUTE` receipts are acknowledged without invoking the tracker or consuming correlation. The first terminal receipt records its exact state and error, consumes every correlation for the gateway message, and either deletes a `NONE` delivery row or retains an HTTP/SMPP row as `PENDING`. ```mermaid sequenceDiagram @@ -164,8 +164,9 @@ sequenceDiagram participant Tracker as StandardMessageTracker participant Service as DlrService participant Database as PostgreSQL DLR storage - participant DLRHook as ForwardDlrService - participant App as Originating application + participant HTTP as HTTP DLR dispatcher + participant App as Originating HTTP application + participant SMPPApp as Originating SMPP client Ingress->>Service: saveInitialState(gateway message ID) Service->>Database: Insert DLR message @@ -187,16 +188,29 @@ sequenceDiagram Tracker->>Service: resolveDlr(provider pair, exact state/error) Service->>Database: Lock, resolve, consume correlations, retain pending delivery Database-->>Service: Resolved message state - opt DLR callback URL exists - Service->>DLRHook: Forward DLR callback - DLRHook->>App: HTTP GET callback + alt persistence succeeds + Worker-->>SMSC: deliver_sm_resp (success) + else persistence fails + Worker-->>SMSC: deliver_sm_resp (SYSERR) end - Service-->>Tracker: Resolved message state + end + + alt HTTP delivery channel + loop Poll durable due rows + HTTP->>Database: Start fenced attempt + HTTP->>App: HTTP GET callback + HTTP->>Database: Delete on success or persist retry/failure + end + else SMPP delivery channel Tracker->>Router: Enqueue internal MSG_DLR - Worker-->>SMSC: deliver_sm_resp + Router->>SMPPApp: deliver_sm receipt part(s) + SMPPApp-->>Router: deliver_sm_resp for every part + Router->>Database: Delete only after all responses succeed end ``` +HTTP and SMPP delivery are acknowledgement-driven and at-least-once. A crash after the receiver accepts a callback or response can cause the same receipt, including already acknowledged multipart SMPP parts, to be delivered again. + ## MO Handling Mobile-originated messages received from upstream SMPP providers are handled by the SMPP client worker. If the worker instance has an MO forwarding URL configured, `SmppClientWorker` forwards the MO through `ForwardMoService` using the configured forwarding format. diff --git a/docs/07-webhooks.md b/docs/07-webhooks.md index 802db88..826c33f 100644 --- a/docs/07-webhooks.md +++ b/docs/07-webhooks.md @@ -4,7 +4,7 @@ Sendium can call external HTTP endpoints for delivery receipts and mobile-origin ## Delivery Receipt Callbacks -HTTP submissions can include a `dlr-url` query parameter. Sendium stores the callback URL with the submitted message and calls it when the message state changes. +HTTP submissions can include a `dlr-url` query parameter. Sendium stores the callback URL with the submitted message and calls it for the first terminal provider outcome. Intermediate `ACCEPTD` and `ENROUTE` receipts are acknowledged to the provider but are not forwarded. Example HTTP submission: @@ -31,10 +31,12 @@ curl -G http://localhost:8080/sendsms \ | :--- | :--- | | `1` | Delivered. | | `2` | Failed. | -| `4` | Buffered or accepted for processing. | -| `8` | Submitted to SMSC. | +| `4` | Buffered or accepted for processing; retained as a compatibility mapping and not normally emitted by final-only receipt handling. | +| `8` | Submitted to SMSC; retained as a compatibility mapping and not normally emitted by final-only receipt handling. | -DLR callbacks are sent as HTTP `GET` requests. HTTP status codes from `200` to `399` are treated as successful. Sendium makes up to 10 attempts with a 120 second delay between failed attempts. The retry schedule is process-local and does not survive a Sendium restart. +DLR callbacks are sent as HTTP `GET` requests. The durable dispatcher checks PostgreSQL on a one-second schedule in serial batches of up to 100; a running batch delays the next check rather than overlapping it. Each request has a five-second timeout, and redirects are not followed; the original response status from `200` to `399` is treated as successful. Failures on attempts 1 through 9 are scheduled 120 seconds later. A failure on attempt 10 marks the row `FAILED`, and pending or failed rows are eligible for cleanup seven days after provider resolution. A malformed callback URI fails immediately without starting an HTTP attempt. + +The retry schedule and attempt count survive a Sendium restart, but delivery is at-least-once rather than exactly-once. A crash or storage failure after the receiver accepts a callback can cause another request. Callback handlers must be idempotent and should use the gateway message ID supplied through `%s` as their deduplication key. ## Mobile-Originated Message Forwarding @@ -85,7 +87,7 @@ outSms.instance.testRoute.forward.mo.url = https://example.com/mo?from=%p&to=%P& outSms.instance.testRoute.forward.mo.format = FORM ``` -MO callbacks are sent as HTTP `POST` requests. HTTP status codes from `200` to `399` are treated as successful. Sendium makes up to 10 attempts with a 120 second delay between failed attempts. The retry schedule is process-local and does not survive a Sendium restart. +Unlike durable DLR callbacks, MO callbacks are sent as process-local HTTP `POST` requests. HTTP status codes from `200` to `399` are treated as successful. Sendium makes up to 10 attempts with a 120 second delay between failed attempts. The MO retry schedule does not survive a Sendium restart. ## Security Notes diff --git a/docs/08-monitoring-observability.md b/docs/08-monitoring-observability.md index eca4662..975df08 100644 --- a/docs/08-monitoring-observability.md +++ b/docs/08-monitoring-observability.md @@ -18,7 +18,7 @@ For a local Docker or development run, verify the endpoint with: curl http://localhost:8080/q/metrics ``` -The endpoint exposes Quarkus, JVM, HTTP server, and Micrometer runtime metrics. Sendium-specific business metrics require explicit instrumentation in code, such as counters, timers, or gauges registered through Micrometer. +The endpoint exposes Quarkus, JVM, HTTP server, and Micrometer runtime metrics. Sendium's PostgreSQL DLR subsystem also registers its selected-backend gauge and storage-operation timers. ## Prometheus Configuration diff --git a/docs/13-dlr-persistence.md b/docs/13-dlr-persistence.md index 99e32fb..e86e1ab 100644 --- a/docs/13-dlr-persistence.md +++ b/docs/13-dlr-persistence.md @@ -12,6 +12,12 @@ The `sendium.dlr.persistence.enabled` build-time property controls this boundary Message paths degrade rather than fail when the subsystem is absent. HTTP and downstream SMPP submissions are accepted and routed without gateway DLR state, undelivered downstream receipts fall back to the worker's in-memory retry, and provider receipts are not correlated, so Sendium emits no delivery receipts of its own. An application that embeds `sendium-core` without this subsystem is expected to supply its own `Tracker` and message store if it needs delivery receipts. +## Storage Model + +`sendium_dlr.dlr_message` contains one row per gateway UUID. The row holds ingress metadata, the exact terminal provider outcome, the downstream delivery channel and status, the common HTTP/SMPP payload, the retry schedule, and the monotonically increasing attempt number used for fencing. + +`sendium_dlr.provider_correlation` maps the exact `(provider_name, provider_message_id)` pair to the gateway UUID. Multiple provider correlations, including multipart provider IDs, can point to one message. Terminal resolution stores the resolving provider pair and outcome in `dlr_message`, removes every correlation for that gateway message, and retains the message row only when HTTP or SMPP delivery is required. + ## Quick Start PostgreSQL The generated Quick Start runtime creates: @@ -98,9 +104,9 @@ PostgreSQL is fail-closed. If required persistence is unavailable, new HTTP subm Provider message IDs are correlated within the outbound provider namespace rather than globally. The worker instance name is the default namespace; workers connected to the same SMSC account can share `msg.hash.prefix` when that SMSC may deliver their receipts interchangeably. Different providers may therefore return the same message ID without overwriting each other's state. The namespace must remain stable while correlations are outstanding: changing `msg.hash.prefix` or renaming a worker using the default makes earlier receipts unresolvable. -Sendium requests final delivery receipts from upstream SMPP providers. A valid unsolicited `ACCEPTD` or `ENROUTE` receipt is acknowledged successfully but is not forwarded and does not consume its provider correlation. The first terminal receipt consumes the correlation and produces the downstream DLR; later receipts for that provider message ID cannot resolve it. Multipart submissions retain this first-terminal behavior and do not aggregate delivery states across every segment. +Sendium requests final delivery receipts from upstream SMPP providers. A valid unsolicited `ACCEPTD` or `ENROUTE` receipt, including a receipt identified through its SMPP ESM class, is acknowledged successfully but is not forwarded and does not consume its provider correlation. The first terminal receipt consumes every correlation for the gateway message and produces the downstream DLR; later receipts for those provider message IDs cannot resolve it. Multipart submissions retain this first-terminal behavior and do not aggregate delivery states across every segment. A terminal persistence failure returns `deliver_sm_resp` with `STATUS_SYSERR` so the provider can retry. A successful provider acknowledgement confirms durable resolution only; it does not wait for downstream HTTP or SMPP delivery. -A terminal receipt remains in `sendium_dlr.dlr_message` while HTTP or SMPP delivery is pending. The delivery attempt number is a fencing token: a stale completion or failure cannot mutate a newer attempt, and an adapter-local active-ID guard prevents duplicate starts within one process. +A terminal receipt remains in `sendium_dlr.dlr_message` while HTTP or SMPP delivery is pending or after HTTP delivery reaches terminal `FAILED` status. The delivery attempt number is a fencing token: a stale completion or failure cannot mutate a newer attempt, and a process-local storage guard prevents duplicate starts within one Sendium instance. ## Retention @@ -114,6 +120,8 @@ The V1 retention thresholds are fixed application behavior, not environment sett Cleanup is triggered by storage activity and runs no more than once per hour. These values are therefore eligibility thresholds, not exact physical deletion deadlines: idle records can remain in the database longer, and an active deployment can retain newly eligible state until the next cleanup pass. A provider receipt cannot be matched after its correlation has been removed. Making the thresholds or cleanup schedule configurable is outside the V1 storage replacement. +An HTTP row marked `FAILED` after attempt 10 remains eligible based on the original provider-resolution time, not the time of its final request. SMPP delivery has no fixed attempt cap, but a still-pending SMPP row is also eligible for cleanup seven days after resolution. + Cleanup is best-effort maintenance and is isolated from message handling. One caller at a time runs a pass while every other caller proceeds immediately, and a failed pass is logged and left until the next interval rather than rejecting the submission that triggered it. ## Durability Boundaries @@ -122,13 +130,16 @@ Cleanup is best-effort maintenance and is isolated from message handling. One ca | :--- | :--- | :--- | | Initial DLR state for HTTP and downstream SMPP submissions | Persisted before HTTP routing or a successful SMPP acknowledgement. | Router and worker queues remain in memory. A process crash can lose queued outbound work even though its DLR row remains until cleanup. | | Gateway-to-provider message correlation | Survives Sendium restart after the provider message ID is linked. Intermediate `ACCEPTD` and `ENROUTE` receipts leave it intact. | The first terminal receipt consumes every correlation for the gateway message. | -| Terminal HTTP/SMPP delivery | The common payload and exact provider outcome remain in one row until fenced completion. | Delivery scheduling and SMPP response batching are separate runtime concerns. | +| Terminal HTTP/SMPP delivery | The common payload and exact provider outcome remain in one row until fenced completion. | Delivery is at-least-once; acknowledgement can be received before the final delete commits. | | Active delivery attempt | The database attempt number fences stale completion, retry, and failure updates. | The active-ID guard is process-local. Adapter recreation may start a new attempt for an attempt that was active before a crash. | | Multipart submission | Each acknowledged segment has provisional DLR state; completed aggregates update the primary state. | Multipart assembly and its pending timers are process-local and are not reconstructed after restart. | -| HTTP DLR callback retry | Pending state and the next-attempt timestamp are durable. | The scheduler that consumes due rows is implemented separately. | +| HTTP DLR callback retry | Pending state, attempt count, and next-attempt timestamp are durable. Checks are scheduled every second in non-overlapping serial batches; failures retry after 120 seconds and attempt 10 failures become `FAILED`. | A request accepted before a crash or failed completion update can be repeated. A slow batch delays later due callbacks. | +| SMPP DLR delivery | One attempt covers every generated receipt part and completes only after matching successful `deliver_sm_resp` PDUs for all parts. Pending rows are enqueued when the same `system_id` binds. | Timeout, `generic_nack`, wrong/non-OK response, enqueue/send failure, or session closure releases the attempt. Replay is bind-driven rather than periodic, and partial success is not checkpointed. | | Database files | The Quick Start named volume survives normal container replacement and `docker compose down`. | Volume deletion, host-disk loss, and disaster recovery require backups or external PostgreSQL replication managed by the operator. | -These limits are intentional V1 boundaries. PostgreSQL provides DLR persistence and delivery fencing; it is not a distributed worker coordinator or a replacement for the router and worker queues. +Downstream delivery uses bounded at-least-once attempt semantics, not exactly-once delivery. A crash or storage failure after an HTTP receiver accepts a callback, or after an SMPP client sends a successful `deliver_sm_resp`, can cause the receipt to be delivered again. Multipart SMPP replay can repeat already acknowledged parts. Consumers must be idempotent using the gateway or receipted message ID. HTTP retry limits, SMPP bind availability, and seven-day retention mean this is not an unlimited eventual-success guarantee. + +These limits are intentional V1 boundaries. PostgreSQL provides DLR persistence and delivery fencing; it is not a distributed worker coordinator or a replacement for the router and worker queues. Attempt guards are process-local, so multiple active Sendium replicas sharing one database can start duplicate deliveries. ## Related Documentation diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java index d964894..548f11f 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/DlrDeliveryBatch.java @@ -58,6 +58,10 @@ public String getGatewayMessageId() { return message.serial; } + public M getMessage() { + return message; + } + public void partSucceeded(int partOrdinal) { boolean complete = false; synchronized (this) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java index 897dcd3..10741e8 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/tasks/OutTask.java @@ -28,6 +28,7 @@ public void run() { boolean success; DlrDeliverSmReference dlrReference = pdu.getReferenceObject() instanceof DlrDeliverSmReference reference ? reference : null; + StandardMessage durableDlr = dlrReference == null ? null : dlrReference.batch().getMessage(); if (dlrReference != null && !dlrReference.batch().isActive()) { return; } @@ -65,14 +66,18 @@ public void run() { if (!success) { if (dlrReference != null) { + if (MessageTrace.shouldLog(worker.getConfigurationProvider(), MessageTrace.EVENT_DELIVER_FAILED)) { + logger.warn("message.deliver.failed worker={} {}", worker.getFullName(), + MessageTrace.identifiers(durableDlr)); + } dlrReference.batch().fail("send_failed"); } else { worker.outTaskFailed(pdu, msg); } - } else if (!pdu.isResponse() && msg != null) { + } else if (!pdu.isResponse() && (msg != null || durableDlr != null)) { if (MessageTrace.shouldLog(worker.getConfigurationProvider(), MessageTrace.EVENT_DELIVER_SENT)) { logger.info("message.deliver.sent worker={} deliverMsgId={} {}", worker.getFullName(), - MessageTrace.value(deliverMsgId), MessageTrace.identifiers(msg)); + MessageTrace.value(deliverMsgId), MessageTrace.identifiers(msg != null ? msg : durableDlr)); } } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java index dc378f1..492af3b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/ForwardDlrService.java @@ -1,8 +1,5 @@ package gr.cytech.sendium.core.worker; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Timer; import io.quarkus.arc.properties.IfBuildProperty; import io.quarkus.scheduler.Scheduled; import jakarta.enterprise.context.ApplicationScoped; @@ -26,10 +23,6 @@ public class ForwardDlrService { private static final Logger logger = LoggerFactory.getLogger(ForwardDlrService.class); - private static final String ATTEMPT_METRIC = "sendium.dlr.delivery.attempt"; - private static final String TERMINAL_FAILURE_METRIC = "sendium.dlr.delivery.terminal.failure"; - private static final String DISPATCH_ERROR_METRIC = "sendium.dlr.delivery.dispatch.error"; - private static final String CHANNEL_HTTP = "http"; private static final int DUE_BATCH_SIZE = 100; private static final int MAX_ATTEMPTS = 10; private static final long RETRY_INTERVAL_MS = 120_000; @@ -44,17 +37,15 @@ public class ForwardDlrService { private static final String MSG_ID_PLACEHOLDER = "%s"; private final DlrService dlrService; - private final MeterRegistry meterRegistry; private final HttpClient httpClient; @Inject - public ForwardDlrService(DlrService dlrService, MeterRegistry meterRegistry) { - this(dlrService, meterRegistry, newHttpClient()); + public ForwardDlrService(DlrService dlrService) { + this(dlrService, newHttpClient()); } - ForwardDlrService(DlrService dlrService, MeterRegistry meterRegistry, HttpClient httpClient) { + ForwardDlrService(DlrService dlrService, HttpClient httpClient) { this.dlrService = dlrService; - this.meterRegistry = meterRegistry; this.httpClient = httpClient; } @@ -71,7 +62,6 @@ void dispatchDueDeliveries() { try { dueDeliveries = dlrService.listDueHttpDeliveries(DUE_BATCH_SIZE); } catch (RuntimeException e) { - recordDispatchError("scheduler"); logger.error("Unable to list due HTTP DLR deliveries"); return; } @@ -79,8 +69,10 @@ void dispatchDueDeliveries() { for (MessageState state : dueDeliveries) { try { dispatch(state); + if (Thread.currentThread().isInterrupted()) { + return; + } } catch (RuntimeException e) { - recordDispatchError("scheduler"); logger.error("Unexpected HTTP DLR dispatch failure for gatewayMsgId={}", state.getGatewayMsgId()); } } @@ -108,25 +100,19 @@ private void dispatch(MessageState dueState) { } int attempt = started.orElseThrow().getDeliveryAttemptCount(); - Timer.Sample sample = Timer.start(meterRegistry); try { HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); if (response.statusCode() >= 200 && response.statusCode() < 400) { - sample.stop(attemptTimer("success")); completeDelivery(gatewayMsgId, attempt); } else { - sample.stop(attemptTimer("http_failure")); handleAttemptFailure(gatewayMsgId, attempt, "http_failure"); } } catch (HttpTimeoutException e) { - sample.stop(attemptTimer("timeout")); handleAttemptFailure(gatewayMsgId, attempt, "timeout"); } catch (InterruptedException e) { - sample.stop(attemptTimer("transport_failure")); handleAttemptFailure(gatewayMsgId, attempt, "interrupted"); Thread.currentThread().interrupt(); } catch (IOException | RuntimeException e) { - sample.stop(attemptTimer("transport_failure")); handleAttemptFailure(gatewayMsgId, attempt, "transport_failure"); } } @@ -171,9 +157,6 @@ private void handleAttemptFailure(String gatewayMsgId, int attempt, String resul gatewayMsgId, attempt, result, System.currentTimeMillis() + RETRY_INTERVAL_MS); } else { updated = dlrService.failDelivery(gatewayMsgId, attempt, result); - if (updated) { - terminalFailureCounter("max_attempts").increment(); - } } if (!updated) { recordStorageError(gatewayMsgId, attempt, "finish"); @@ -186,9 +169,7 @@ private void handleAttemptFailure(String gatewayMsgId, int attempt, String resul private void failInvalidDelivery(String gatewayMsgId) { logger.warn("Invalid HTTP DLR callback for gatewayMsgId={}", gatewayMsgId); try { - if (dlrService.failInvalidDelivery(gatewayMsgId, "invalid_uri")) { - terminalFailureCounter("invalid_uri").increment(); - } else { + if (!dlrService.failInvalidDelivery(gatewayMsgId, "invalid_uri")) { recordStorageError(gatewayMsgId, 0, "invalid"); } } catch (RuntimeException e) { @@ -197,33 +178,10 @@ private void failInvalidDelivery(String gatewayMsgId) { } private void recordStorageError(String gatewayMsgId, int attempt, String operation) { - recordDispatchError("storage"); logger.error("HTTP DLR storage update failed for gatewayMsgId={} attempt={} operation={}", gatewayMsgId, attempt, operation); } - private void recordDispatchError(String source) { - Counter.builder(DISPATCH_ERROR_METRIC) - .description("Sendium DLR dispatcher errors") - .tags("channel", CHANNEL_HTTP, "source", source) - .register(meterRegistry) - .increment(); - } - - private Timer attemptTimer(String outcome) { - return Timer.builder(ATTEMPT_METRIC) - .description("Sendium DLR delivery attempt latency") - .tags("channel", CHANNEL_HTTP, "outcome", outcome) - .register(meterRegistry); - } - - private Counter terminalFailureCounter(String reason) { - return Counter.builder(TERMINAL_FAILURE_METRIC) - .description("Sendium terminal DLR delivery failures") - .tags("channel", CHANNEL_HTTP, "reason", reason) - .register(meterRegistry); - } - int mapToKannelType(MessageState.MessageStatus status) { if (status == null) { return DLR_BUFFERED; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java index c625724..0975e8a 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/ForwardDlrServiceTest.java @@ -1,7 +1,6 @@ package gr.cytech.sendium.core.worker; import com.sun.net.httpserver.HttpServer; -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -19,7 +18,6 @@ import java.net.http.HttpTimeoutException; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -45,14 +43,12 @@ class ForwardDlrServiceTest { @Mock HttpClient httpClient; - private SimpleMeterRegistry meterRegistry; private ForwardDlrService service; private HttpServer server; @BeforeEach void setUp() { - meterRegistry = new SimpleMeterRegistry(); - service = new ForwardDlrService(dlrService, meterRegistry, httpClient); + service = new ForwardDlrService(dlrService, httpClient); } @AfterEach @@ -60,7 +56,6 @@ void tearDown() { if (server != null) { server.stop(0); } - meterRegistry.close(); } @Test @@ -91,7 +86,6 @@ void successfulResponseCompletesExpectedAttempt() throws Exception { var order = inOrder(dlrService, httpClient); order.verify(dlrService).startDeliveryAttempt(GATEWAY_ID, MessageState.DeliveryChannel.HTTP); order.verify(httpClient).send(any(HttpRequest.class), anyBodyHandler()); - assertAttemptMetric("success", 1); } @Test @@ -113,14 +107,13 @@ void directRedirectCompletesAndIsNotFollowed() throws Exception { + server.getAddress().getPort() + "/redirect"); dueAttempt(due, 1); when(dlrService.completeDelivery(GATEWAY_ID, 1)).thenReturn(true); - service = new ForwardDlrService(dlrService, meterRegistry, ForwardDlrService.newHttpClient()); + service = new ForwardDlrService(dlrService, ForwardDlrService.newHttpClient()); service.dispatchDueDeliveries(); verify(dlrService).completeDelivery(GATEWAY_ID, 1); assertThat(redirectTargetRequests).hasValue(0); assertThat(ForwardDlrService.newHttpClient().followRedirects()).isEqualTo(HttpClient.Redirect.NEVER); - assertAttemptMetric("success", 1); } @Test @@ -144,8 +137,6 @@ void tenthFailureMarksDeliveryFailed() throws Exception { verify(dlrService).failDelivery(GATEWAY_ID, 10, "http_failure"); verify(dlrService, never()).retryDelivery(eq(GATEWAY_ID), eq(10), any(), anyLong()); - assertThat(meterRegistry.get("sendium.dlr.delivery.terminal.failure") - .tags("channel", "http", "reason", "max_attempts").counter().count()).isEqualTo(1); } @Test @@ -160,7 +151,6 @@ void timeoutSchedulesRetry() throws Exception { service.dispatchDueDeliveries(); verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(2), eq("timeout"), anyLong()); - assertAttemptMetric("timeout", 1); assertThat(deliveryResult().getValue()).doesNotContain("secret", "token", "http"); } @@ -176,7 +166,6 @@ void ioFailureSchedulesTransportRetry() throws Exception { service.dispatchDueDeliveries(); verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(3), eq("transport_failure"), anyLong()); - assertAttemptMetric("transport_failure", 1); } @Test @@ -191,13 +180,19 @@ void runtimeTransportFailureSchedulesRetry() throws Exception { service.dispatchDueDeliveries(); verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(4), eq("transport_failure"), anyLong()); - assertAttemptMetric("transport_failure", 1); } @Test - void interruptionSchedulesRetryAndRestoresInterrupt() throws Exception { + void interruptionSchedulesRetryRestoresInterruptAndStopsBatch() throws Exception { MessageState due = dueState("https://example.test/dlr"); - dueAttempt(due, 5); + String laterGatewayId = "1e5fc768-c60d-4417-95bf-d39642381a1c"; + MessageState later = new MessageState(laterGatewayId, "account", "system", "source", "destination", + "https://example.test/later"); + MessageState started = dueState(due.getForwardDlrUrl()); + started.setDeliveryAttemptCount(5); + when(dlrService.listDueHttpDeliveries(100)).thenReturn(List.of(due, later)); + when(dlrService.startDeliveryAttempt(GATEWAY_ID, MessageState.DeliveryChannel.HTTP)) + .thenReturn(Optional.of(started)); when(httpClient.send(any(HttpRequest.class), anyBodyHandler())) .thenThrow(new InterruptedException("interrupted")); when(dlrService.retryDelivery(eq(GATEWAY_ID), eq(5), eq("interrupted"), anyLong())) @@ -207,8 +202,8 @@ void interruptionSchedulesRetryAndRestoresInterrupt() throws Exception { service.dispatchDueDeliveries(); verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(5), eq("interrupted"), anyLong()); + verify(dlrService, never()).startDeliveryAttempt(laterGatewayId, MessageState.DeliveryChannel.HTTP); assertThat(Thread.currentThread().isInterrupted()).isTrue(); - assertAttemptMetric("transport_failure", 1); } finally { Thread.interrupted(); } @@ -225,8 +220,6 @@ void invalidUriFailsWithoutStartingAnAttempt() { verify(dlrService).failInvalidDelivery(GATEWAY_ID, "invalid_uri"); verify(dlrService, never()).startDeliveryAttempt(any(), any()); verifyNoInteractions(httpClient); - assertThat(meterRegistry.get("sendium.dlr.delivery.terminal.failure") - .tags("channel", "http", "reason", "invalid_uri").counter().count()).isEqualTo(1); } @Test @@ -242,7 +235,7 @@ void activeAttemptIsSkippedBeforeSending() { } @Test - void completionStorageFailureLeavesDeliveryForLaterRunAndRecordsBoundedError() throws Exception { + void completionStorageFailureLeavesDeliveryForLaterRun() throws Exception { MessageState due = dueState("https://example.test/dlr"); dueAttempt(due, 1); respondWith(200); @@ -252,19 +245,15 @@ void completionStorageFailureLeavesDeliveryForLaterRunAndRecordsBoundedError() t service.dispatchDueDeliveries(); verify(dlrService).completeDelivery(GATEWAY_ID, 1); - assertThat(meterRegistry.get("sendium.dlr.delivery.dispatch.error") - .tags("channel", "http", "source", "storage").counter().count()).isEqualTo(1); } @Test - void schedulerStorageFailureIsCountedWithoutDynamicMetricTags() { + void schedulerStorageFailureDoesNotSend() { when(dlrService.listDueHttpDeliveries(100)).thenThrow(new DlrStorageException("database unavailable")); service.dispatchDueDeliveries(); - assertThat(meterRegistry.get("sendium.dlr.delivery.dispatch.error") - .tags("channel", "http", "source", "scheduler").counter().count()).isEqualTo(1); - assertBoundedMetricTags(); + verifyNoInteractions(httpClient); } @Test @@ -292,8 +281,6 @@ private void assertHttpFailureSchedulesRetry(int statusCode) throws Exception { verify(dlrService).retryDelivery(eq(GATEWAY_ID), eq(1), eq("http_failure"), nextAttempt.capture()); assertThat(nextAttempt.getValue()).isBetween(beforeFailure + 120_000, System.currentTimeMillis() + 120_000); assertThat(deliveryResult().getValue()).isEqualTo("http_failure"); - assertAttemptMetric("http_failure", 1); - assertBoundedMetricTags(); } private ArgumentCaptor deliveryResult() { @@ -330,21 +317,4 @@ private HttpResponse.BodyHandler anyBodyHandler() { return any(HttpResponse.BodyHandler.class); } - private void assertAttemptMetric(String outcome, long expectedCount) { - assertThat(meterRegistry.get("sendium.dlr.delivery.attempt") - .tags("channel", "http", "outcome", outcome).timer().count()).isEqualTo(expectedCount); - } - - private void assertBoundedMetricTags() { - Set permittedTags = Set.of("channel", "outcome", "reason", "source"); - assertThat(meterRegistry.getMeters()) - .filteredOn(meter -> meter.getId().getName().startsWith("sendium.dlr.delivery")) - .allSatisfy(meter -> assertThat(meter.getId().getTags()) - .extracting(io.micrometer.core.instrument.Tag::getKey) - .allMatch(permittedTags::contains)); - assertThat(meterRegistry.getMeters()) - .flatExtracting(meter -> meter.getId().getTags()) - .extracting(io.micrometer.core.instrument.Tag::getKey) - .doesNotContain("id", "url", "host", "provider", "attempt"); - } }