From 070c6904e4c82be583b5e5ee4edab8ca6f6e3516 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 14:58:47 +0800 Subject: [PATCH 01/15] add connector it test in mysql, mongodb; add github action workflow for it test. --- .github/workflows/connector-it.yml | 42 ++++++ connectors-common/mysql-core/pom.xml | 2 +- connectors/mongodb-connector/pom.xml | 67 +++++++++ .../tapdata/mongodb/MongoDBConnectorIT.java | 129 ++++++++++++++++++ .../resources/config/mongodb-connection.json | 4 + connectors/mysql-connector/pom.xml | 71 +++++++++- .../connector/mysql/MySQLConnectorIT.java | 79 +++++++++++ .../it/resources/config/mysql-connection.json | 8 ++ 8 files changed, 399 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/connector-it.yml create mode 100644 connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java create mode 100644 connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json create mode 100644 connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java create mode 100644 connectors/mysql-connector/src/it/resources/config/mysql-connection.json diff --git a/.github/workflows/connector-it.yml b/.github/workflows/connector-it.yml new file mode 100644 index 000000000..ab42e1abe --- /dev/null +++ b/.github/workflows/connector-it.yml @@ -0,0 +1,42 @@ +# Connector 集成测试（开源库触发入口） +# +# 触发时机：push 任意分支（含 PR 合并后的推送）+ PR 提交/更新 + 手工触发。 +# paths 过滤保证仅连接器相关路径变更时启动，文档/README 等改动不触发。 +# 实际测试逻辑在 tapdata-application 的 connector-it-reusable.yml 中统一维护。 +name: Connector Integration Tests + +on: + push: + branches: ['**'] + paths: + - 'connectors/**' + - 'connectors-common/**' + - 'connectors-javascript/**' + - 'pom.xml' + - '.github/**' + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'connectors/**' + - 'connectors-common/**' + - 'connectors-javascript/**' + - 'pom.xml' + - '.github/**' + workflow_dispatch: + inputs: + connectors: + description: '手工指定模块列表（逗号分隔，如 connectors/mysql-connector, connectors/mongodb-connector）' + required: false + type: string + +concurrency: + group: connector-it-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + connector-it: + uses: tapdata/tapdata-application/.github/workflows/connector-it-reusable.yml@main + with: + java_version: '17' + connectors_override: ${{ inputs.connectors || '' }} + secrets: inherit diff --git a/connectors-common/mysql-core/pom.xml b/connectors-common/mysql-core/pom.xml index 89f72f6bf..70eb25994 100644 --- a/connectors-common/mysql-core/pom.xml +++ b/connectors-common/mysql-core/pom.xml @@ -126,7 +126,7 @@ io.tapdata tapdata-common - 2.0-SNAPSHOT + 2.9-SNAPSHOT provided diff --git a/connectors/mongodb-connector/pom.xml b/connectors/mongodb-connector/pom.xml index a8193356a..cbcbeec01 100644 --- a/connectors/mongodb-connector/pom.xml +++ b/connectors/mongodb-connector/pom.xml @@ -15,6 +15,7 @@ 8 2.0.7-SNAPSHOT 1.84 + true @@ -116,6 +117,13 @@ + + + io.tapdata + tapdata-connector-it + 1.0-SNAPSHOT + test + org.mockito @@ -289,6 +297,65 @@ + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + add-integration-test-source + generate-test-sources + + add-test-source + add-test-resource + + + + src/it/java + + + + src/it/resources + + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.5.6 + + ${skipITs} + + + -Xmx4g + -Dapp_type=DAAS + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.security=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.time=ALL-UNNAMED + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED + + + ${project.build.directory}/failsafe-reports + + + + + integration-test + + integration-test + verify + + + + diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java new file mode 100644 index 000000000..5ae739c73 --- /dev/null +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -0,0 +1,129 @@ +package io.tapdata.mongodb; + +import io.tapdata.entity.codec.TapCodecsRegistry; +import io.tapdata.entity.logger.TapLog; +import io.tapdata.entity.utils.DataMap; +import io.tapdata.it.ConnectorIT; +import io.tapdata.it.ConnectorTestContext; +import io.tapdata.it.support.TestStateMap; +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.functions.ConnectorFunctions; +import org.bson.BsonRegularExpression; +import org.bson.BsonTimestamp; +import org.bson.types.Binary; +import org.bson.types.Code; +import org.bson.types.Decimal128; +import org.bson.types.ObjectId; +import org.bson.types.Symbol; +import org.junit.jupiter.api.TestInstance; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * MongoDB 连接器通用集成测试。 + *

+ * 继承 {@link ConnectorIT} 后自动运行框架内置的全部通用集成测试用例 + * （连接元数据、集合 DDL、数据读写、事务、流式读取、命令等）， + * 对 MongoDB 不支持的 ConnectorFunctions 能力自动跳过。 + * 必实现能力由 {@link #requiredCapabilities()} 主动声明（原则 3：声明式能力）， + * 框架据此校验：声明必实现但未实现、或已实现但无用例覆盖 → 测试失败。 + *

+ * 连接配置读取 src/it/resources/config/mongodb-connection.json + * （uri/database），支持系统属性/环境变量覆盖 + * （如 -Dconnector.it.uri=xxx 或 CONNECTOR_IT_URI=xxx）。 + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class MongoDBConnectorIT extends ConnectorIT { + + @Override + protected Set requiredCapabilities() { + // MongoDB 同时承担源与目标角色：声明对外承诺的全部能力（与 registerCapabilities 注册保持一致； + // memoryFetcher 为引擎诊断钩子，由基类 ignoredCapabilities() 排除，不参与校验） + return Stream.of("connectionTest", "discoverSchema", "getTableNames", + "createTableV2", "dropTable", "batchCount", "batchRead", "streamRead", + "timestampToStreamOffset", "queryByAdvanceFilter", "countByPartitionFilter", + "writeRecord", "createIndex", "queryIndexes", "errorHandle", + "executeCommand", "getTableInfo", "getReadPartitions", "queryFieldMinMaxValue", + "transactionBegin", "transactionCommit", "transactionRollback").collect(Collectors.toSet()); + } + + @Override + protected ConnectorTestContext createContext() throws Throwable { + // 1. 创建连接器 + MongodbConnector connector = new MongodbConnector(); + + // 2. 连接配置（URI 模式，与产品连接表单默认一致） + DataMap config = readConnectionConfig("config/mongodb-connection.json"); + + // 3. 构建 NodeContext：specification + connectionConfig + nodeConfig + log + // specification 提供 dataTypesMap，discoverSchema 的 tapType 自动填充依赖它 + TapConnectorContext nodeContext = new TapConnectorContext( + loadSpecification("spec.json"), config, DataMap.create(), new TapLog()); + nodeContext.setStateMap(new TestStateMap()); + + // 4. 注册能力（registerCapabilities 产物，基类据此动态检测并驱动用例） + ConnectorFunctions functions = new ConnectorFunctions(); + TapCodecsRegistry codecRegistry = TapCodecsRegistry.create(); + connector.registerCapabilities(functions, codecRegistry); + + // 5. 组装测试上下文（特性开关：MongoDB NoSQL 特性适配） + ConnectorTestContext ctx = ConnectorTestContext.builder() + .connector(connector) + .nodeContext(nodeContext) + .connectorFunctions(functions) + .codecRegistry(codecRegistry) + .config(config) + .log(new TapLog()) + // MongoDB createTableV2 幂等建集合固定返回 tableExists=false + .createTableReportsTableExists(false) + // schema-free：空集合无法推断字段，需先写入采样数据再 discoverSchema + .schemaDiscoveryRequiresSampleData(true) + // discoverSchema 会带出隐式 _id 字段 + .schemaAllowsExtraFields(true) + // 主键为库自动生成的 _id，业务字段不被标记为主键 + .schemaPrimaryKeyStrict(false) + // executeCommand 仅支持 execute/executeQuery/count/aggregate，不支持 ping + .executeCommandSupportsPing(false) + // queryFieldMinMaxValue 基于分区索引字段计算 min/max + .fieldMinMaxRequiresPartitionIndex(true) + .build(); + ctx.getLog().info("[IT] MongoDB connection: uri={}, database={}", + config.getString("uri"), config.getString("database")); + return ctx; + } + + @Override + protected List> beforeWrite(List> rows) { + // unique 索引用例（idx_c_int）要求 c_int 唯一：随机值冲突时改为超出随机范围的序列值 + Set seen = new HashSet<>(); + long seq = 1_000_001L; + for (Map row : rows) { + long v = ((Number) row.get("c_int")).longValue(); + if (!seen.add(v)) { + row.put("c_int", seq++); + } + } + return rows; + } + + @Override + protected Map specialValueSamples() { + // MongoDB 特有 BSON 类型样本：须与 registerCapabilities 注册的 ToTapValueCodec 一一对应 + // （U8 用例验证这些特殊值 wrap 后被连接器 codec 识别，而非 TapRawValue 兜底） + Map samples = new LinkedHashMap<>(); + samples.put("obj_objectid", new ObjectId()); + samples.put("obj_binary", new Binary((byte) 0x80, new byte[]{1, 2, 3})); + samples.put("obj_code", new Code("function() { return 1; }")); + samples.put("obj_decimal128", Decimal128.parse("12345.6789")); + samples.put("obj_symbol", new Symbol("sym")); + samples.put("obj_bson_timestamp", new BsonTimestamp(1_700_000_000, 1)); + samples.put("obj_regex", new BsonRegularExpression("^tap.*", "i")); + return samples; + } +} diff --git a/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json new file mode 100644 index 000000000..564ab69cf --- /dev/null +++ b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json @@ -0,0 +1,4 @@ +{ + "uri": "mongodb://127.0.0.1:27017/test", + "database": "test" +} diff --git a/connectors/mysql-connector/pom.xml b/connectors/mysql-connector/pom.xml index 4b36c0eb2..787a82546 100644 --- a/connectors/mysql-connector/pom.xml +++ b/connectors/mysql-connector/pom.xml @@ -22,8 +22,9 @@ - 8 + 17 2.0.5-SNAPSHOT + true @@ -54,11 +55,26 @@ 5.8.1 test + + + io.tapdata + tapdata-connector-it + 1.0-SNAPSHOT + test + io.tapdata mysql-core 1.0-SNAPSHOT + + + io.tapdata + tapdata-common + 2.9-SNAPSHOT + test + io.tapdata pdk-error-code @@ -209,6 +225,24 @@ + + add-integration-test-source + generate-test-sources + + add-test-source + add-test-resource + + + + src/it/java + + + + src/it/resources + + + + @@ -232,7 +266,40 @@ - + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.5.6 + + ${skipITs} + + + -Xmx8g + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.security=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.time=ALL-UNNAMED + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED + + + ${project.build.directory}/failsafe-reports + + + + + integration-test + + integration-test + verify + + + + + diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java new file mode 100644 index 000000000..9e8ed1551 --- /dev/null +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -0,0 +1,79 @@ +package io.tapdata.connector.mysql; + +import io.tapdata.entity.codec.TapCodecsRegistry; +import io.tapdata.entity.logger.TapLog; +import io.tapdata.entity.utils.DataMap; +import io.tapdata.it.ConnectorIT; +import io.tapdata.it.ConnectorTestContext; +import io.tapdata.it.support.TestStateMap; +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.functions.ConnectorFunctions; +import org.junit.jupiter.api.TestInstance; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * MySQL 连接器通用集成测试。 + *

+ * 继承 {@link ConnectorIT} 后自动运行框架内置的全部通用集成测试用例 + * （连接元数据、表 DDL、数据读写、事务、流式读取、命令等）， + * 对 MySQL 不支持的 ConnectorFunctions 能力自动跳过。 + * 必实现能力由 {@link #requiredCapabilities()} 主动声明（原则 3：声明式能力）， + * 框架据此校验：声明必实现但未实现、或已实现但无用例覆盖 → 测试失败。 + *

+ * 连接配置读取 src/it/resources/config/connection.json + * （host/port/database/user/password），支持系统属性/环境变量覆盖 + * （如 -Dconnector.it.host=xxx 或 CONNECTOR_IT_HOST=xxx）。 + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class MySQLConnectorIT extends ConnectorIT { + + @Override + protected Set requiredCapabilities() { + // MySQL 同时承担源与目标角色：声明对外承诺的全部能力。 + // 声明后若实现被移除或框架无对应用例，原则 3 校验用例将失败（不遗漏必实现接口、不漏测已实现能力） + return Stream.of("connectionTest", "discoverSchema", "getTableNames", + "createTableV2", "dropTable", "clearTable", "batchCount", "batchRead", + "streamRead", "streamReadMultiConnection", "timestampToStreamOffset", + "queryByAdvanceFilter", "countByPartitionFilter", "writeRecord", "afterInitialSync", + "createIndex", "queryIndexes", "createConstraint", "queryConstraints", "dropConstraint", + "newField", "alterFieldName", "alterFieldAttributes", "dropField", + "errorHandle", "executeCommand", "executeCommandV2", "getTableInfo", + "runRawCommand", "transactionBegin", "transactionCommit", "transactionRollback", + "queryHashByAdvanceFilter", "exportEventSql").collect(Collectors.toSet()); + } + + @Override + protected ConnectorTestContext createContext() throws Throwable { + // 1. 创建连接器 + MysqlConnector connector = new MysqlConnector(); + + // 2. 连接配置（与 src/test/resources 的占位 connection.json 隔离，避免资源覆盖冲突） + DataMap config = readConnectionConfig("config/mysql-connection.json"); + + // 3. 构建 NodeContext：specification + connectionConfig + nodeConfig + log + TapConnectorContext nodeContext = new TapConnectorContext( + null, config, DataMap.create().kv("enableTransaction", true), new TapLog()); + nodeContext.setStateMap(new TestStateMap()); + + // 4. 注册能力（registerCapabilities 产物，基类据此动态检测并驱动用例） + ConnectorFunctions functions = new ConnectorFunctions(); + TapCodecsRegistry codecRegistry = TapCodecsRegistry.create(); + connector.registerCapabilities(functions, codecRegistry); + + // 5. 组装测试上下文 + ConnectorTestContext ctx = ConnectorTestContext.builder() + .connector(connector) + .nodeContext(nodeContext) + .connectorFunctions(functions) + .codecRegistry(codecRegistry) + .config(config) + .log(new TapLog()) + .build(); + ctx.getLog().info("[IT] MySQL connection: host={}, port={}, database={}, user={}", + config.getString("host"), config.getInteger("port"), config.getString("database"), config.getString("user")); + return ctx; + } +} diff --git a/connectors/mysql-connector/src/it/resources/config/mysql-connection.json b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json new file mode 100644 index 000000000..790537efc --- /dev/null +++ b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json @@ -0,0 +1,8 @@ +{ + "host": "127.0.0.1", + "port": 13306, + "database": "test", + "user": "root", + "password": "root", + "highPerformance": true +} From 5716520c6b7453ab3e920bd5aad93cc475a4ae63 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 15:18:46 +0800 Subject: [PATCH 02/15] update connector it workflow --- .github/workflows/connector-it.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-it.yml b/.github/workflows/connector-it.yml index ab42e1abe..e50753101 100644 --- a/.github/workflows/connector-it.yml +++ b/.github/workflows/connector-it.yml @@ -35,7 +35,8 @@ concurrency: jobs: connector-it: - uses: tapdata/tapdata-application/.github/workflows/connector-it-reusable.yml@main + # TODO: 合入主分支后将 @feature-it 改回 @main + uses: tapdata/tapdata-application/.github/workflows/connector-it-reusable.yml@feature-it with: java_version: '17' connectors_override: ${{ inputs.connectors || '' }} From 7d30a0e6365b3b16da9202d72f5f42c00d5d1292 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 15:53:34 +0800 Subject: [PATCH 03/15] update connector it workflow --- .../java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index 9e8ed1551..ae7d0948f 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -8,6 +8,8 @@ import io.tapdata.it.support.TestStateMap; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.functions.ConnectorFunctions; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.TestInstance; import java.util.Set; @@ -76,4 +78,10 @@ protected ConnectorTestContext createContext() throws Throwable { config.getString("host"), config.getInteger("port"), config.getString("database"), config.getString("user")); return ctx; } + + @DisplayName("Test github action") + public void testGithubAction() { + int result = 10*2; + Assertions.assertEquals(20, result); + } } From 3fff0d6ee3a9a5d0db2cc7cbd8c72ed5de9432d3 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 16:06:47 +0800 Subject: [PATCH 04/15] update connector it workflow --- .github/workflows/connector-it.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/connector-it.yml b/.github/workflows/connector-it.yml index e50753101..2d7c05768 100644 --- a/.github/workflows/connector-it.yml +++ b/.github/workflows/connector-it.yml @@ -2,7 +2,7 @@ # # 触发时机：push 任意分支（含 PR 合并后的推送）+ PR 提交/更新 + 手工触发。 # paths 过滤保证仅连接器相关路径变更时启动，文档/README 等改动不触发。 -# 实际测试逻辑在 tapdata-application 的 connector-it-reusable.yml 中统一维护。 +# 实际测试逻辑在 tapdata-it 的 connector-it-reusable.yml 中统一维护。 name: Connector Integration Tests on: @@ -35,8 +35,7 @@ concurrency: jobs: connector-it: - # TODO: 合入主分支后将 @feature-it 改回 @main - uses: tapdata/tapdata-application/.github/workflows/connector-it-reusable.yml@feature-it + uses: tapdata/tapdata-it/.github/workflows/connector-it-reusable.yml@main with: java_version: '17' connectors_override: ${{ inputs.connectors || '' }} From 0ca99eaa9ea38f69adf0acebade2cb9047bf3565 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 16:08:12 +0800 Subject: [PATCH 05/15] update connector it workflow --- .../java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index ae7d0948f..42f508807 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -10,6 +10,7 @@ import io.tapdata.pdk.apis.functions.ConnectorFunctions; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import java.util.Set; @@ -80,8 +81,9 @@ protected ConnectorTestContext createContext() throws Throwable { } @DisplayName("Test github action") + @Test public void testGithubAction() { - int result = 10*2; - Assertions.assertEquals(20, result); + int result = 10*3; + Assertions.assertEquals(30, result); } } From 7268c6bc99509a038fb963e9bbf9cc6d6fde9a9e Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 18:01:17 +0800 Subject: [PATCH 06/15] update connector it workflow --- .../src/it/resources/config/mysql-connection.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/connectors/mysql-connector/src/it/resources/config/mysql-connection.json b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json index 790537efc..ac76ecd42 100644 --- a/connectors/mysql-connector/src/it/resources/config/mysql-connection.json +++ b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json @@ -1,8 +1,8 @@ { - "host": "127.0.0.1", - "port": 13306, - "database": "test", + "host": "192.168.1.184", + "port": 23306, + "database": "test_it", "user": "root", - "password": "root", + "password": "Gotapd8!", "highPerformance": true } From 8d18ed2c9962dc2d46e6c5e078f37540126328b4 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 18:06:39 +0800 Subject: [PATCH 07/15] update connector it workflow --- .github/workflows/connector-it.yml | 1 + .../src/it/resources/config/mongodb-connection.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-it.yml b/.github/workflows/connector-it.yml index 2d7c05768..84f0b0fd3 100644 --- a/.github/workflows/connector-it.yml +++ b/.github/workflows/connector-it.yml @@ -3,6 +3,7 @@ # 触发时机：push 任意分支（含 PR 合并后的推送）+ PR 提交/更新 + 手工触发。 # paths 过滤保证仅连接器相关路径变更时启动，文档/README 等改动不触发。 # 实际测试逻辑在 tapdata-it 的 connector-it-reusable.yml 中统一维护。 +# 运行环境：office-build 自托管 runner（runs-on 在 reusable workflow 中配置）。 name: Connector Integration Tests on: diff --git a/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json index 564ab69cf..4288ec431 100644 --- a/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json +++ b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json @@ -1,4 +1,4 @@ { - "uri": "mongodb://127.0.0.1:27017/test", + "uri": "mongodb://root:Gotapd8@192.168.1.184:14017/test?authSource=admin", "database": "test" } From 73e5d911a3010de6bd69c29b8398d2a0e401e1f2 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 18:14:29 +0800 Subject: [PATCH 08/15] update connector it workflow --- .../src/it/resources/config/mongodb-connection.json | 2 +- .../src/it/resources/config/mysql-connection.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json index 4288ec431..e2610ae85 100644 --- a/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json +++ b/connectors/mongodb-connector/src/it/resources/config/mongodb-connection.json @@ -1,4 +1,4 @@ { - "uri": "mongodb://root:Gotapd8@192.168.1.184:14017/test?authSource=admin", + "uri": "mongodb://root:Gotapd8@113.98.206.142:14017/test?authSource=admin", "database": "test" } diff --git a/connectors/mysql-connector/src/it/resources/config/mysql-connection.json b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json index ac76ecd42..d9061f96b 100644 --- a/connectors/mysql-connector/src/it/resources/config/mysql-connection.json +++ b/connectors/mysql-connector/src/it/resources/config/mysql-connection.json @@ -1,5 +1,5 @@ { - "host": "192.168.1.184", + "host": "113.98.206.142", "port": 23306, "database": "test_it", "user": "root", From ea8acf0085afc025ca0d7a53fd1226a9a7c4c186 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 18:18:21 +0800 Subject: [PATCH 09/15] update connector it workflow --- .../src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java | 7 +++++++ .../java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java index 5ae739c73..f93bad1c2 100644 --- a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -126,4 +126,11 @@ protected Map specialValueSamples() { samples.put("obj_regex", new BsonRegularExpression("^tap.*", "i")); return samples; } + + @DisplayName("Test github action") + @Test + public void testGithubAction() { + int result = 10*4; + Assertions.assertEquals(40, result); + } } diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index 42f508807..4efad09b9 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -83,7 +83,7 @@ protected ConnectorTestContext createContext() throws Throwable { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*3; - Assertions.assertEquals(30, result); + int result = 10*4; + Assertions.assertEquals(40, result); } } From 5f22097126929d543eede2604c2c47269c6e26b5 Mon Sep 17 00:00:00 2001 From: Leon Date: Mon, 24 Aug 2026 18:19:48 +0800 Subject: [PATCH 10/15] update connector it workflow --- .../src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java | 4 ++-- .../it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java index f93bad1c2..c7324b944 100644 --- a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -130,7 +130,7 @@ protected Map specialValueSamples() { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*4; - Assertions.assertEquals(40, result); + int result = 10*5; + Assertions.assertEquals(50, result); } } diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index 4efad09b9..a75fe3631 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -83,7 +83,7 @@ protected ConnectorTestContext createContext() throws Throwable { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*4; - Assertions.assertEquals(40, result); + int result = 10*5; + Assertions.assertEquals(50, result); } } From ee81e62e7515f86c0e96e9659fee083a3c9c748c Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 25 Aug 2026 09:42:25 +0800 Subject: [PATCH 11/15] update connector it workflow --- .../src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java index c7324b944..602af203d 100644 --- a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -15,6 +15,9 @@ import org.bson.types.Decimal128; import org.bson.types.ObjectId; import org.bson.types.Symbol; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import java.util.HashSet; From 9732576f8311089a7d372d0ff91c16e40ae626cd Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 26 Aug 2026 16:39:07 +0800 Subject: [PATCH 12/15] trigger integration test --- .../src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java | 4 ++-- .../it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java index 602af203d..9e2604e3b 100644 --- a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -133,7 +133,7 @@ protected Map specialValueSamples() { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*5; - Assertions.assertEquals(50, result); + int result = 10*6; + Assertions.assertEquals(60, result); } } diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index a75fe3631..76ad48c66 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -83,7 +83,7 @@ protected ConnectorTestContext createContext() throws Throwable { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*5; - Assertions.assertEquals(50, result); + int result = 10*6; + Assertions.assertEquals(60, result); } } From d49ac1f57302455fbe310855c4d5bb169a0721eb Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 26 Aug 2026 17:01:57 +0800 Subject: [PATCH 13/15] update workflow input parame --- .github/workflows/connector-it.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/connector-it.yml b/.github/workflows/connector-it.yml index 84f0b0fd3..0da9ea7f5 100644 --- a/.github/workflows/connector-it.yml +++ b/.github/workflows/connector-it.yml @@ -38,6 +38,5 @@ jobs: connector-it: uses: tapdata/tapdata-it/.github/workflows/connector-it-reusable.yml@main with: - java_version: '17' connectors_override: ${{ inputs.connectors || '' }} secrets: inherit From 93c7df4deaf0b8843e0dac8906078684d1024d6f Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 26 Aug 2026 17:02:55 +0800 Subject: [PATCH 14/15] trigger integration test --- .../src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java | 4 ++-- .../it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java index 9e2604e3b..9e4b1b7e6 100644 --- a/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java +++ b/connectors/mongodb-connector/src/it/java/io/tapdata/mongodb/MongoDBConnectorIT.java @@ -133,7 +133,7 @@ protected Map specialValueSamples() { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*6; - Assertions.assertEquals(60, result); + int result = 10*7; + Assertions.assertEquals(70, result); } } diff --git a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java index 76ad48c66..2a0142032 100644 --- a/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java +++ b/connectors/mysql-connector/src/it/java/io/tapdata/connector/mysql/MySQLConnectorIT.java @@ -83,7 +83,7 @@ protected ConnectorTestContext createContext() throws Throwable { @DisplayName("Test github action") @Test public void testGithubAction() { - int result = 10*6; - Assertions.assertEquals(60, result); + int result = 10*7; + Assertions.assertEquals(70, result); } } From 9b98ac38b98190590bbdae2fa372426cd3a549bf Mon Sep 17 00:00:00 2001 From: harsen Date: Fri, 28 Aug 2026 09:32:52 +0800 Subject: [PATCH 15/15] fix(mysql): add FROM for large batch reads --- .../io/tapdata/connector/mysql/MysqlConnector.java | 2 +- .../tapdata/connector/mysql/MysqlConnectorTest.java | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java index 734547e69..4ac26e89d 100644 --- a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java +++ b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java @@ -773,7 +773,7 @@ public void buildIllegalDateFieldName(TapRecordEvent event, List illegal @Override protected String getBatchReadSelectSql(TapTable tapTable) { if (tapTable.getNameFieldMap().size() > 50) { - return String.format("SELECT * %s", getSchemaAndTable(tapTable.getId())); + return String.format("SELECT * FROM %s", getSchemaAndTable(tapTable.getId())); } else { return super.getBatchReadSelectSql(tapTable); } diff --git a/connectors/mysql-connector/src/test/java/io/tapdata/connector/mysql/MysqlConnectorTest.java b/connectors/mysql-connector/src/test/java/io/tapdata/connector/mysql/MysqlConnectorTest.java index d2052169c..259cf5d36 100644 --- a/connectors/mysql-connector/src/test/java/io/tapdata/connector/mysql/MysqlConnectorTest.java +++ b/connectors/mysql-connector/src/test/java/io/tapdata/connector/mysql/MysqlConnectorTest.java @@ -235,15 +235,17 @@ void test7() throws SQLException { class BatchReadSqlTest { TapTable tapTable; MysqlConfig mysqlConfig; + CommonDbConfig commonDbConfig; MysqlConnector connector; @BeforeEach void setUp() { tapTable = mock(TapTable.class); mysqlConfig = mock(MysqlConfig.class); - connector = mock(MysqlConnector.class); - doCallRealMethod().when(connector).getBatchReadSelectSql(tapTable); + commonDbConfig = mock(CommonDbConfig.class); + connector = new MysqlConnector(); UnitTestUtils.injectField(MysqlConnector.class, connector, "mysqlConfig", mysqlConfig); + UnitTestUtils.injectField(CommonDbConnector.class, connector, "commonDbConfig", commonDbConfig); } private LinkedHashMap generateFieldMap(TapField... fields) { @@ -265,7 +267,11 @@ void testFieldSizeLargeThen50() { fields[i] = new TapField("f" + i, "INT"); } when(tapTable.getNameFieldMap()).thenReturn(generateFieldMap(fields)); - assertTrue(connector.getBatchReadSelectSql(tapTable).toLowerCase().startsWith("select *")); + when(tapTable.getId()).thenReturn("AA_0607"); + when(commonDbConfig.getEscapeChar()).thenReturn('`'); + when(commonDbConfig.getSchema()).thenReturn("COOLGJ"); + + assertEquals("SELECT * FROM `COOLGJ`.`AA_0607`", connector.getBatchReadSelectSql(tapTable)); } }