diff --git a/.github/workflows/collector-native-build.yml b/.github/workflows/collector-native-build.yml new file mode 100644 index 00000000000..d44877dba8f --- /dev/null +++ b/.github/workflows/collector-native-build.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Collector Native Release + +run-name: Native collector release build (${{ github.ref_name }}) + +on: + workflow_dispatch: + +jobs: + build-native-collector: + name: Native collector (${{ matrix.platform }}) + permissions: + contents: read + timeout-minutes: 120 + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: ubuntu-24.04 + archive_ext: tar.gz + - platform: linux-arm64 + runner: ubuntu-24.04-arm + archive_ext: tar.gz + - platform: macos-amd64 + runner: macos-13 + archive_ext: tar.gz + - platform: macos-arm64 + runner: macos-14 + archive_ext: tar.gz + - platform: windows-amd64 + runner: windows-latest + archive_ext: zip + + steps: + - uses: actions/checkout@v4 + + - name: Set up GraalVM JDK 25 + uses: graalvm/setup-graalvm@v1 + with: + distribution: graalvm-community + java-version: '25' + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: maven + native-image-job-reports: 'true' + + - name: Verify toolchain + shell: pwsh + run: | + java -version + native-image --version + mvn -version + + - name: Build native collector package + run: mvn -B -pl hertzbeat-collector/hertzbeat-collector-collector -am -Pnative -DskipTests package + + - name: Locate native collector package + id: package + shell: pwsh + run: | + $package = Get-ChildItem -Path "dist/apache-hertzbeat-collector-native-*-${{ matrix.platform }}-bin.${{ matrix.archive_ext }}" | Select-Object -First 1 + if (-not $package) { + throw "Native collector package not found for ${{ matrix.platform }}" + } + "archive=$($package.FullName)" >> $env:GITHUB_OUTPUT + + - name: Upload native collector package + uses: actions/upload-artifact@v4 + with: + name: apache-hertzbeat-collector-native-${{ matrix.platform }} + path: ${{ steps.package.outputs.archive }} + retention-days: 14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a1738267c9..9f329883578 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` @@ -172,7 +172,7 @@ Add WeChat account `ahertzbeat` to pull you into the WeChat group. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/README.md b/README.md index 0d8cff96a67..eb8cac645f8 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,12 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh ` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters (Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download](https://hertzbeat.apache.org/docs/download) + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml collector: @@ -148,7 +148,9 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh ` or `bin/startup.bat` + - If you do not provide JDBC drivers in `ext-lib`, MySQL, MariaDB, and OceanBase can use the built-in query engine and run on the native collector package as well. TiDB follows the same rule for its SQL query metric set. + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC after restart for MySQL, MariaDB, and OceanBase. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics are unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. + - Run `$ ./bin/startup.sh ` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh ` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. - Access `http://localhost:1157` and you will see the registered new collector in dashboard Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -156,7 +158,7 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `hertzbeat-startup` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, add VM options in IDE: ` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED `, then start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` diff --git a/README_CN.md b/README_CN.md index 7393ace7010..80058e8f7b4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -127,12 +127,12 @@ ##### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包 `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml` (可选) 3. 部署启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群（可选） - - 下载采集器安装包 `hertzbeat-collector-xx.tar.gz` 到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) + - 下载采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或与你目标平台匹配的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主 HertzBeat 服务的对外 IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml collector: @@ -145,7 +145,9 @@ manager-host: ${MANAGER_HOST:127.0.0.1} manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh ` 或 `bin/startup.bat` + - 如果没有在 `ext-lib` 中提供 JDBC 驱动，MySQL、MariaDB、OceanBase 可以直接使用内置查询引擎，也可以使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动。 + - JVM 采集器安装包使用 `$ ./bin/startup.sh ` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh ` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) @@ -153,7 +155,7 @@ ##### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程 `hertzbeat-startup` 和前端工程 `web-app` -2. 后端：需要 `maven3+`, `java21` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 +2. 后端：需要 `maven3+`, `java25` 和 `lombok` 环境，修改 `YML` 配置信息，添加JVM参数`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`后启动 `hertzbeat-startup` 服务即可。 3. 前端：需要 `nodejs npm angular-cli`环境，待本地后端启动后，在 `web-app` 目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` diff --git a/README_JP.md b/README_JP.md index 13c49de6a4f..4487344ce90 100644 --- a/README_JP.md +++ b/README_JP.md @@ -108,7 +108,7 @@ docker run -d -p 1157:1157 -p 1158:1158 --name hertzbeat apache/hertzbeat ``` -2. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 +2. スタート：`http://localhost:1157`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 3. コレクタークラスタのデプロイメント（オプション） @@ -126,13 +126,13 @@ ##### 方式２：インストールパッケージ -1. リリースパッケージをダウンロード `hertzbeat-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) -2. HertzBeatのymlファイルを設定 `hertzbeat/config/application.yml` （オプション） -3. コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行 -4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat` +1. リリースパッケージ `apache-hertzbeat-xx-bin.tar.gz` をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) +2. HertzBeat の設定ファイル `hertzbeat/config/application.yml` を編集します（任意） +3. コマンド `$ ./bin/startup.sh` または `bin/startup.bat` を実行します +4. ブラウザで `http://localhost:1157` にアクセスします。デフォルトのアカウントとパスワードは `admin/hertzbeat` です 5. コレクタークラスタのデプロイメント（オプション） - - コレクターパッケージを別のホストにダウンロード `hertzbeat-collector-xx.tar.gz` [Download](https://hertzbeat.apache.org/docs/download) - - コレクターのymlファイルを設定 `hertzbeat-collector/config/application.yml` + - 別ホストにコレクターのインストールパッケージ `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM コレクター）または対象プラットフォーム向けの Native コレクターパッケージ（例: `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`）をダウンロードします [Download](https://hertzbeat.apache.org/docs/download) + - コレクターの設定ファイル `hertzbeat-collector/config/application.yml` を編集します ```yaml collector: dispatch: @@ -148,15 +148,17 @@ - `mode: ${MODE:public}`：実行モード(パブリッククラスタまたはプライベートクラウドエッジ)。 - `manager-host: ${MANAGER_HOST:127.0.0.1}`：メインhertzbeatサーバーのIP。 - `manager-port: ${MANAGER_PORT:1158}`：メインhertzbeatサーバポート。 - - コマンド`$ ./bin/startup.sh`または`bin/startup.bat`を実行。 - - `http://localhost:1157`にアクセスし、登録された新しいコレクターを見ることがでます。 + - `ext-lib` に JDBC ドライバーを置かない場合、MySQL、MariaDB、OceanBase は組み込みのクエリエンジンを使って Native コレクターパッケージでも監視できます。TiDB も SQL クエリのメトリクスセットについては同じルールです。 + - `ext-lib` に `mysql-connector-j` を置いた場合は、再起動後に組み込みサーバーコレクターまたは JVM コレクターが MySQL、MariaDB、OceanBase で自動的に JDBC を優先します。TiDB も SQL クエリのメトリクスセットについては同じルールで、HTTP メトリクスは影響を受けません。Oracle と DB2 は引き続き外部 JDBC ドライバーに依存するため、JVM コレクターパッケージを使用してください。 + - JVM コレクターのインストールパッケージは `$ ./bin/startup.sh` または `bin/startup.bat`、Linux/macOS の Native コレクターパッケージは `$ ./bin/startup.sh`、Windows の Native コレクターパッケージは `bin\\startup.bat` で起動します。 + - メインの HertzBeat サービス `http://localhost:1157` にアクセスすると、登録された新しいコレクターを確認できます。 詳細ステップ [通过安装包安装HertzBeat](https://hertzbeat.apache.org/docs/start/package-deploy) ##### 方式３：ローカルの実行 1. ローカルの実行には、バックエンドのプロジェクト`hertzbeat-startup`とフロントエンドのプロジェクト`web-app`を起動する必要があります。 -2. バックエンド：`maven3+`、`Java21`と`lombok`の環境は必要です。`YML` 設定を修正し、Java仮想マシンパラメータに`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加し、`hertzbeat-startup` を起動します。 +2. バックエンド：`maven3+`、`Java25`、`lombok` の環境が必要です。`YML` 設定を修正し、Java 仮想マシンパラメータに `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` を追加して `hertzbeat-startup` を起動します。 3. フロントエンド：`nodejs npm angular-cli`の環境は必要です。ローカルのバックエンドが立ち上がったら、`web-app` ディレクトリで `ng serve --open` というコマンドを実行します。 4. スタート：`http://localhost:4200`にアクセスします。デフォルトのアカウントとパスワード：`admin/hertzbeat`。 diff --git a/hertzbeat-ai/pom.xml b/hertzbeat-ai/pom.xml index 2b9298b6b5e..235b2481dce 100644 --- a/hertzbeat-ai/pom.xml +++ b/hertzbeat-ai/pom.xml @@ -27,7 +27,7 @@ ${hertzbeat.version} 1.1.1 - 21 + 25 diff --git a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml index c43972cda0a..63c88e262df 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-basic/pom.xml @@ -30,8 +30,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 1.2.5 @@ -67,12 +67,6 @@ commons-net commons-net - - - com.mysql - mysql-connector-j - provided - com.clickhouse @@ -189,4 +183,4 @@ ${zookeeper.version} - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java index 42529c72842..76e9421a6a2 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java @@ -20,7 +20,6 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; @@ -34,6 +33,9 @@ import org.apache.hertzbeat.collector.collect.common.cache.CacheIdentifier; import org.apache.hertzbeat.collector.collect.common.cache.GlobalConnectionCache; import org.apache.hertzbeat.collector.collect.common.cache.JdbcConnect; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; import org.apache.hertzbeat.collector.constants.CollectorConstants; import org.apache.hertzbeat.collector.dispatch.DispatchConstants; @@ -205,31 +207,26 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException { public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { long startTime = System.currentTimeMillis(); JdbcProtocol jdbcProtocol = metrics.getJdbc(); - SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); - int timeout = CollectUtil.getTimeout(jdbcProtocol.getTimeout()); boolean reuseConnection = Boolean.parseBoolean(jdbcProtocol.getReuseConnection()); - Statement statement = null; - String databaseUrl; try { - if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { - int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - databaseUrl = constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); - } else { - databaseUrl = constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); - } - - statement = getConnection(jdbcProtocol.getUsername(), - jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); switch (jdbcProtocol.getQueryType()) { - case QUERY_TYPE_ONE_ROW -> queryOneRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_MULTI_ROW -> queryMultiRow(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case QUERY_TYPE_COLUMNS -> queryOneRowByMatchTwoColumns(statement, jdbcProtocol.getSql(), metrics.getAliasFields(), builder, startTime); - case RUN_SCRIPT -> { - Connection connection = statement.getConnection(); - FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); - ScriptUtils.executeSqlScript(connection, rc); + case QUERY_TYPE_ONE_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1)) { + queryOneRow(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case QUERY_TYPE_MULTI_ROW -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryMultiRow(rowSet, metrics.getAliasFields(), builder, startTime); + } } + case QUERY_TYPE_COLUMNS -> { + try (JdbcQueryRowSet rowSet = executeQuery(metrics, timeout, reuseConnection, 1000)) { + queryOneRowByMatchTwoColumns(rowSet, metrics.getAliasFields(), builder, startTime); + } + } + case RUN_SCRIPT -> runScript(metrics, timeout, reuseConnection); default -> { builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Not support database query type: " + jdbcProtocol.getQueryType()); @@ -261,23 +258,6 @@ public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { log.error("Jdbc error: {}.", errorMessage, e); builder.setCode(CollectRep.Code.FAIL); builder.setMsg("Query Error: " + errorMessage); - } finally { - if (statement != null) { - Connection connection = null; - try { - connection = statement.getConnection(); - statement.close(); - } catch (Exception e) { - log.error("Jdbc close statement error: {}", e.getMessage()); - } - try { - if (!reuseConnection && connection != null) { - connection.close(); - } - } catch (Exception e) { - log.error("Jdbc close connection error: {}", e.getMessage()); - } - } } } @@ -286,6 +266,52 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_JDBC; } + private JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, boolean reuseConnection, int maxRows) throws Exception { + Optional executor = JdbcQueryExecutorRegistry.resolve(metrics); + if (executor.isPresent()) { + return executor.get().executeQuery(metrics, timeout, maxRows); + } + return executeJdbcQuery(metrics.getJdbc(), timeout, reuseConnection, maxRows); + } + + private JdbcQueryRowSet executeJdbcQuery(JdbcProtocol jdbcProtocol, int timeout, boolean reuseConnection, + int maxRows) throws Exception { + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + statement.setMaxRows(maxRows); + return new ResultSetJdbcQueryRowSet(statement, statement.executeQuery(jdbcProtocol.getSql()), reuseConnection); + } catch (Exception exception) { + closeStatementAndConnection(statement, reuseConnection); + throw exception; + } + } + + private void runScript(Metrics metrics, int timeout, boolean reuseConnection) throws Exception { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + Statement statement = null; + try { + String databaseUrl = resolveDatabaseUrl(jdbcProtocol); + statement = getConnection(jdbcProtocol.getUsername(), + jdbcProtocol.getPassword(), databaseUrl, timeout, reuseConnection); + Connection connection = statement.getConnection(); + FileSystemResource rc = new FileSystemResource(jdbcProtocol.getSql()); + ScriptUtils.executeSqlScript(connection, rc); + } finally { + closeStatementAndConnection(statement, reuseConnection); + } + } + + private String resolveDatabaseUrl(JdbcProtocol jdbcProtocol) throws Exception { + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + return constructDatabaseUrl(jdbcProtocol, "localhost", String.valueOf(localPort)); + } + return constructDatabaseUrl(jdbcProtocol, jdbcProtocol.getHost(), jdbcProtocol.getPort()); + } private Statement getConnection(String username, String password, String url, Integer timeout, boolean reuseConnection) throws Exception { CacheIdentifier identifier = CacheIdentifier.builder() @@ -343,29 +369,25 @@ private Statement getConnection(String username, String password, String url, In * query metrics：one tow three four * query sql：select one, tow, three, four from book limit 1; * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRow(Statement statement, String sql, List columns, + private void queryOneRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - statement.setMaxRows(1); - try (ResultSet resultSet = statement.executeQuery(sql)) { - if (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + if (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } } @@ -380,33 +402,30 @@ private void queryOneRow(Statement statement, String sql, List columns, * three - value3 * four - value4 * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List columns, + private void queryOneRowByMatchTwoColumns(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - HashMap values = new HashMap<>(columns.size()); - while (resultSet.next()) { - if (resultSet.getString(1) != null) { - values.put(resultSet.getString(1).toLowerCase().trim(), resultSet.getString(2)); - } + HashMap values = new HashMap<>(columns.size()); + while (rowSet.next()) { + if (rowSet.getString(1) != null) { + values.put(rowSet.getString(1).toLowerCase().trim(), rowSet.getString(2)); } - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = values.get(column.toLowerCase()); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + } + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = values.get(column.toLowerCase()); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); } /** @@ -416,28 +435,45 @@ private void queryOneRowByMatchTwoColumns(Statement statement, String sql, List< * query sql：select one, tow, three, four from book; * and return multi row record mapping with the metrics * - * @param statement statement - * @param sql sql + * @param rowSet row set * @param columns query metrics field list * @throws Exception when error happen */ - private void queryMultiRow(Statement statement, String sql, List columns, + private void queryMultiRow(JdbcQueryRowSet rowSet, List columns, CollectRep.MetricsData.Builder builder, long startTime) throws Exception { - try (ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); - for (String column : columns) { - if (CollectorConstants.RESPONSE_TIME.equals(column)) { - long time = System.currentTimeMillis() - startTime; - valueRowBuilder.addColumn(String.valueOf(time)); - } else { - String value = resultSet.getString(column); - value = value == null ? CommonConstants.NULL_VALUE : value; - valueRowBuilder.addColumn(value); - } + while (rowSet.next()) { + CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder(); + for (String column : columns) { + if (CollectorConstants.RESPONSE_TIME.equals(column)) { + long time = System.currentTimeMillis() - startTime; + valueRowBuilder.addColumn(String.valueOf(time)); + } else { + String value = rowSet.getString(column); + value = value == null ? CommonConstants.NULL_VALUE : value; + valueRowBuilder.addColumn(value); } - builder.addValueRow(valueRowBuilder.build()); } + builder.addValueRow(valueRowBuilder.build()); + } + } + + private void closeStatementAndConnection(Statement statement, boolean reuseConnection) { + if (statement == null) { + return; + } + Connection connection = null; + try { + connection = statement.getConnection(); + statement.close(); + } catch (Exception exception) { + log.error("Jdbc close statement error: {}", exception.getMessage()); + } + try { + if (!reuseConnection && connection != null) { + connection.close(); + } + } catch (Exception exception) { + log.error("Jdbc close connection error: {}", exception.getMessage()); } } @@ -548,4 +584,53 @@ private String constructDatabaseUrl(JdbcProtocol jdbcProtocol, String host, Stri default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform()); }; } + + private static final class ResultSetJdbcQueryRowSet implements JdbcQueryRowSet { + + private final Statement statement; + private final java.sql.ResultSet resultSet; + private final boolean reuseConnection; + + private ResultSetJdbcQueryRowSet(Statement statement, java.sql.ResultSet resultSet, boolean reuseConnection) { + this.statement = statement; + this.resultSet = resultSet; + this.reuseConnection = reuseConnection; + } + + @Override + public boolean next() throws Exception { + return resultSet.next(); + } + + @Override + public String getString(String column) throws Exception { + return resultSet.getString(column); + } + + @Override + public String getString(int index) throws Exception { + return resultSet.getString(index); + } + + @Override + public void close() throws Exception { + Connection connection = null; + try { + connection = statement.getConnection(); + } catch (Exception ignored) { + // ignore + } + try { + resultSet.close(); + } finally { + try { + statement.close(); + } finally { + if (!reuseConnection && connection != null) { + connection.close(); + } + } + } + } + } } diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java new file mode 100644 index 00000000000..3d3c9076da3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Adapter point for replacing only the SQL query execution part of JdbcCommonCollect. + */ +public interface JdbcQueryExecutor { + + boolean supports(Metrics metrics); + + JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java new file mode 100644 index 00000000000..bb9f1c0baa6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryExecutorRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.hertzbeat.common.entity.job.Metrics; + +/** + * Static registry used by JdbcCommonCollect to discover optional query executors. + */ +public final class JdbcQueryExecutorRegistry { + + private static final List EXECUTORS = new CopyOnWriteArrayList<>(); + + private JdbcQueryExecutorRegistry() { + } + + public static void register(JdbcQueryExecutor executor) { + if (executor == null || EXECUTORS.contains(executor)) { + return; + } + EXECUTORS.add(executor); + } + + public static void unregister(JdbcQueryExecutor executor) { + if (executor == null) { + return; + } + EXECUTORS.remove(executor); + } + + public static Optional resolve(Metrics metrics) { + return EXECUTORS.stream() + .filter(executor -> executor.supports(metrics)) + .findFirst(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java new file mode 100644 index 00000000000..a93cef11d77 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/query/JdbcQueryRowSet.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.query; + +/** + * A minimal row cursor abstraction shared by JDBC and R2DBC-backed database queries. + */ +public interface JdbcQueryRowSet extends AutoCloseable { + + boolean next() throws Exception; + + String getString(String column) throws Exception; + + String getString(int index) throws Exception; + + @Override + void close() throws Exception; +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml index 8a108c99de0..5fcca5107d9 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-collector/pom.xml @@ -29,12 +29,18 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 + + org.apache.hertzbeat + hertzbeat-common-spring + ${hertzbeat.version} + + org.apache.hertzbeat @@ -42,6 +48,12 @@ ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + + org.apache.hertzbeat @@ -93,6 +105,23 @@ io.micrometer micrometer-registry-prometheus + + + org.springframework.boot + spring-boot-starter-test + test + + + com.mysql + mysql-connector-j + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + @@ -142,8 +171,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -216,8 +245,8 @@ org.apache.maven.plugins maven-compiler-plugin - 17 - 17 + ${java.version} + ${java.version} @@ -344,5 +373,193 @@ + + native + + hertzbeat-collector-collector + + ../../script/assembly/collector/bin-native + tar.gz + unsupported + ${native.target.platform}-bin + apache-hertzbeat-collector-native-${hzb.version}-${native.target.platform}-bin + target/${native.image.name}${native.binary.extension} + ${project.build.finalName}${native.binary.extension} + + + apache-hertzbeat-collector-native-${hzb.version} + + + org.apache.maven.plugins + maven-compiler-plugin + + full + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-supported-native-platform + validate + + enforce + + + + + native.target.platform + ^(linux-amd64|linux-arm64|macos-amd64|macos-arm64|windows-amd64)$ + Unsupported native collector platform. Build the native collector on a supported GitHub runner or matching local host: linux-amd64, linux-arm64, macos-amd64, macos-arm64, windows-amd64. + + + + + + + + org.graalvm.buildtools + native-maven-plugin + 0.11.4 + + + build-native-image + package + + compile-no-fork + + + + + ${native.image.name} + + -H:+UnlockExperimentalVMOptions + -H:-AddAllFileSystemProviders + -H:ServiceLoaderFeatureExcludeServiceProviders=org.apache.sshd.common.file.root.RootedFileSystemProvider,org.apache.sshd.sftp.client.fs.SftpFileSystemProvider + --initialize-at-build-time=net.i2p.crypto.eddsa.EdDSASecurityProvider,org.apache.arrow.memory.util.MemoryUtil + -J--add-opens=java.base/java.nio=ALL-UNNAMED + -J-Dorg.apache.sshd.security.registrars=org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + native-bin + package + + single + + + + ../../script/assembly/collector/assembly-native.xml + + ../../dist + + + + + + + + + native-platform-linux-amd64 + + + Linux + amd64 + + + + linux-amd64 + + + + native-platform-linux-x86_64 + + + Linux + x86_64 + + + + linux-amd64 + + + + native-platform-linux-arm64 + + + Linux + aarch64 + + + + linux-arm64 + + + + native-platform-macos-amd64 + + + mac + x86_64 + + + + macos-amd64 + + + + native-platform-macos-arm64 + + + mac + aarch64 + + + + macos-arm64 + + + + native-platform-windows-amd64 + + + windows + amd64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + + + native-platform-windows-x86_64 + + + windows + x86_64 + + + + .exe + ../../script/assembly/collector/bin-native-win + zip + windows-amd64 + + diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java index 3e41f228010..0eca3ce9e35 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/Collector.java @@ -18,10 +18,13 @@ package org.apache.hertzbeat.collector; import jakarta.annotation.PostConstruct; +import org.apache.hertzbeat.collector.nativex.CollectorRuntimeHintsRegistrar; +import org.apache.hertzbeat.collector.nativex.NativeCollectorDefaults; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ImportRuntimeHints; /** * collector startup @@ -29,13 +32,20 @@ @ComponentScan(basePackages = {"org.apache.hertzbeat"}) @ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"}) @SpringBootApplication +@ImportRuntimeHints(CollectorRuntimeHintsRegistrar.class) public class Collector { public static void main(String[] args) { - SpringApplication.run(Collector.class, args); + SpringApplication application = new SpringApplication(Collector.class); + NativeCollectorDefaults.applyTo(application); + application.run(args); } @PostConstruct public void init() { System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory"); + if (System.getProperty("arrow.allocation.manager.type") == null + && System.getenv("ARROW_ALLOCATION_MANAGER_TYPE") == null) { + System.setProperty("arrow.allocation.manager.type", "Netty"); + } } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java new file mode 100644 index 00000000000..ac34a37ab84 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlCollectorProperties.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Collector-side MySQL query engine routing. + */ +@Getter +@Setter +@ConfigurationProperties(prefix = "hertzbeat.collector.mysql") +public class MysqlCollectorProperties { + + private QueryEngine queryEngine = QueryEngine.AUTO; + + public QueryEngine resolveQueryEngine(boolean mysqlJdbcDriverAvailable) { + if (queryEngine == QueryEngine.AUTO) { + return mysqlJdbcDriverAvailable ? QueryEngine.JDBC : QueryEngine.R2DBC; + } + return queryEngine; + } + + public boolean useR2dbc(boolean mysqlJdbcDriverAvailable) { + return resolveQueryEngine(mysqlJdbcDriverAvailable) == QueryEngine.R2DBC; + } + + /** + * Supported collector-side query engines. + */ + public enum QueryEngine { + AUTO, + JDBC, + R2DBC + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java new file mode 100644 index 00000000000..8c1466aedeb --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailability.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URL; +import java.security.CodeSource; +import java.util.Locale; +import org.springframework.stereotype.Component; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * Detects whether a MySQL JDBC driver is available from the external ext-lib path. + */ +@Component +public class MysqlJdbcDriverAvailability { + + private static final String[] MYSQL_DRIVER_CLASSES = { + "com.mysql.cj.jdbc.Driver", + "com.mysql.jdbc.Driver" + }; + + public boolean hasMysqlJdbcDriver() { + ClassLoader classLoader = ClassUtils.getDefaultClassLoader(); + for (String driverClass : MYSQL_DRIVER_CLASSES) { + if (!ClassUtils.isPresent(driverClass, classLoader)) { + continue; + } + try { + if (isExternalExtLibDriver(ClassUtils.forName(driverClass, classLoader))) { + return true; + } + } catch (ClassNotFoundException ignored) { + // Race-free enough for runtime detection: keep probing other known driver class names. + } + } + return false; + } + + boolean isExternalExtLibDriver(Class driverClass) { + String location = resolveLocation(driverClass); + return isExtLibLocation(location); + } + + static boolean isExtLibLocation(String location) { + if (!StringUtils.hasText(location)) { + return false; + } + String normalized = location + .replace('\\', '/') + .toLowerCase(Locale.ROOT); + return normalized.contains("/ext-lib/"); + } + + private String resolveLocation(Class driverClass) { + CodeSource codeSource = driverClass.getProtectionDomain().getCodeSource(); + if (codeSource != null && codeSource.getLocation() != null) { + return codeSource.getLocation().toExternalForm(); + } + String resourceName = ClassUtils.convertClassNameToResourcePath(driverClass.getName()) + ".class"; + ClassLoader classLoader = driverClass.getClassLoader(); + URL resource = classLoader != null ? classLoader.getResource(resourceName) : ClassLoader.getSystemResource(resourceName); + return resource != null ? resource.toExternalForm() : null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java new file mode 100644 index 00000000000..55837095c60 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutor.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import java.net.URI; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.hertzbeat.collector.collect.common.ssh.SshTunnelHelper; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutor; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.SshTunnel; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +/** + * MySQL-compatible query-only adapter that lets JdbcCommonCollect execute read-only queries through the built-in + * R2DBC path when no MySQL JDBC driver is present. + */ +@Component +public class MysqlR2dbcJdbcQueryExecutor implements JdbcQueryExecutor, InitializingBean, DisposableBean { + + private static final String QUERY_TYPE_ONE_ROW = "oneRow"; + private static final String QUERY_TYPE_MULTI_ROW = "multiRow"; + private static final String QUERY_TYPE_COLUMNS = "columns"; + + private final MysqlCollectorProperties properties; + private final MysqlQueryExecutor mysqlQueryExecutor; + private final MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability; + + public MysqlR2dbcJdbcQueryExecutor(MysqlCollectorProperties properties, + MysqlQueryExecutor mysqlQueryExecutor, + MysqlJdbcDriverAvailability mysqlJdbcDriverAvailability) { + this.properties = properties; + this.mysqlQueryExecutor = mysqlQueryExecutor; + this.mysqlJdbcDriverAvailability = mysqlJdbcDriverAvailability; + } + + @Override + public boolean supports(Metrics metrics) { + if (metrics == null || metrics.getJdbc() == null) { + return false; + } + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + if (!isMysqlCompatiblePlatform(jdbcProtocol.getPlatform())) { + return false; + } + String queryType = jdbcProtocol.getQueryType(); + return properties.useR2dbc(mysqlJdbcDriverAvailability.hasMysqlJdbcDriver()) + && (QUERY_TYPE_ONE_ROW.equals(queryType) + || QUERY_TYPE_MULTI_ROW.equals(queryType) + || QUERY_TYPE_COLUMNS.equals(queryType)); + } + + private boolean isMysqlCompatiblePlatform(String platform) { + return "mysql".equalsIgnoreCase(platform) || "mariadb".equalsIgnoreCase(platform); + } + + @Override + public JdbcQueryRowSet executeQuery(Metrics metrics, int timeout, int maxRows) { + JdbcProtocol jdbcProtocol = metrics.getJdbc(); + QueryOptions options = buildQueryOptions(jdbcProtocol, timeout, maxRows); + QueryResult queryResult = mysqlQueryExecutor.execute(jdbcProtocol.getSql(), options); + if (queryResult.hasError()) { + throw new IllegalStateException("R2DBC MySQL query failed: " + queryResult.getError()); + } + return new QueryResultRowSet(queryResult); + } + + @Override + public void afterPropertiesSet() { + JdbcQueryExecutorRegistry.register(this); + } + + @Override + public void destroy() { + JdbcQueryExecutorRegistry.unregister(this); + } + + private QueryOptions buildQueryOptions(JdbcProtocol jdbcProtocol, int timeout, int maxRows) { + MysqlTarget target = resolveTarget(jdbcProtocol); + SshTunnel sshTunnel = jdbcProtocol.getSshTunnel(); + String host = target.host(); + int port = target.port(); + if (sshTunnel != null && Boolean.parseBoolean(sshTunnel.getEnable())) { + try { + int localPort = SshTunnelHelper.localPortForward(sshTunnel, host, String.valueOf(port)); + host = "127.0.0.1"; + port = localPort; + } catch (Exception exception) { + throw new IllegalStateException("R2DBC MySQL query adapter failed to establish SSH tunnel", exception); + } + } + return QueryOptions.builder() + .host(host) + .port(port) + .username(jdbcProtocol.getUsername()) + .password(jdbcProtocol.getPassword()) + .database(target.database()) + .schema(target.database()) + .timeout(Duration.ofMillis(timeout)) + .maxRows(maxRows) + .fetchSize(256) + .readOnly(true) + .build(); + } + + private MysqlTarget resolveTarget(JdbcProtocol jdbcProtocol) { + if (StringUtils.hasText(jdbcProtocol.getUrl())) { + return parseJdbcUrl(jdbcProtocol.getUrl(), jdbcProtocol.getDatabase()); + } + if (!StringUtils.hasText(jdbcProtocol.getHost()) || !StringUtils.hasText(jdbcProtocol.getPort())) { + throw new IllegalArgumentException("R2DBC MySQL query adapter requires host/port or a jdbc:mysql URL"); + } + return new MysqlTarget(jdbcProtocol.getHost(), Integer.parseInt(jdbcProtocol.getPort()), jdbcProtocol.getDatabase()); + } + + private MysqlTarget parseJdbcUrl(String url, String fallbackDatabase) { + String trimmed = url.trim(); + if (!(trimmed.startsWith("jdbc:mysql://") || trimmed.startsWith("jdbc:mariadb://"))) { + throw new IllegalArgumentException("R2DBC MySQL query adapter only supports jdbc:mysql:// or jdbc:mariadb:// URLs"); + } + URI uri = URI.create(trimmed.substring("jdbc:".length())); + String host = uri.getHost(); + int port = uri.getPort() > 0 ? uri.getPort() : 3306; + if (!StringUtils.hasText(host)) { + throw new IllegalArgumentException("R2DBC MySQL query adapter URL must include a host"); + } + String path = uri.getPath(); + String database = StringUtils.hasText(path) && path.length() > 1 ? path.substring(1) : fallbackDatabase; + return new MysqlTarget(host, port, database); + } + + private record MysqlTarget(String host, int port, String database) { + } + + private static final class QueryResultRowSet implements JdbcQueryRowSet { + + private final List> rows; + private final Map columnIndexMap; + private int currentIndex = -1; + + private QueryResultRowSet(QueryResult queryResult) { + this.rows = queryResult.getRows(); + this.columnIndexMap = buildColumnIndexMap(queryResult.getColumns()); + } + + @Override + public boolean next() { + currentIndex++; + return currentIndex < rows.size(); + } + + @Override + public String getString(String column) { + Integer index = columnIndexMap.get(column.toLowerCase(Locale.ROOT)); + if (index == null) { + throw new IllegalArgumentException("Column not found in R2DBC MySQL result: " + column); + } + return getString(index + 1); + } + + @Override + public String getString(int index) { + if (currentIndex < 0 || currentIndex >= rows.size()) { + throw new IllegalStateException("R2DBC MySQL result cursor is not positioned on a row"); + } + int zeroBased = index - 1; + List row = rows.get(currentIndex); + if (zeroBased < 0 || zeroBased >= row.size()) { + throw new IllegalArgumentException("Column index out of bounds in R2DBC MySQL result: " + index); + } + return row.get(zeroBased); + } + + @Override + public void close() { + // QueryResult is fully materialized, so there is nothing left to close here. + } + + private static Map buildColumnIndexMap(List columns) { + Map indexMap = new HashMap<>(columns.size()); + for (int index = 0; index < columns.size(); index++) { + indexMap.put(columns.get(index).toLowerCase(Locale.ROOT), index); + } + return indexMap; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java index 4c6517ab905..28b9d2db88e 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactory.java @@ -20,6 +20,7 @@ import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.springframework.boot.CommandLineRunner; import org.springframework.context.annotation.Configuration; @@ -29,6 +30,7 @@ /** * Specific metrics collection factory */ +@Slf4j @Configuration @Order(value = Ordered.HIGHEST_PRECEDENCE + 1) public class CollectStrategyFactory implements CommandLineRunner { @@ -49,10 +51,18 @@ public static AbstractCollect invoke(String protocol) { @Override public void run(String... args) throws Exception { + COLLECT_STRATEGY.clear(); // spi load and registry protocol and collect instance ServiceLoader loader = ServiceLoader.load(AbstractCollect.class, AbstractCollect.class.getClassLoader()); for (AbstractCollect collect : loader) { COLLECT_STRATEGY.put(collect.supportProtocol(), collect); } + if (COLLECT_STRATEGY.isEmpty()) { + throw new IllegalStateException( + "No collect strategies were registered. " + + "Verify META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect " + + "is present on the runtime classpath."); + } + log.info("Registered {} collect strategies: {}", COLLECT_STRATEGY.size(), COLLECT_STRATEGY.keySet()); } } diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java new file mode 100644 index 00000000000..3464caa1b1b --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/CollectorRuntimeHintsRegistrar.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.LinkedHashSet; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.arrow.memory.netty.NettyAllocationManager; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.IntervalUnit; +import org.apache.arrow.vector.types.MetadataVersion; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.UnionMode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.hertzbeat.common.entity.dto.ServerInfo; +import org.springframework.aot.hint.BindingReflectionHintsRegistrar; +import org.springframework.aot.hint.MemberCategory; +import org.springframework.aot.hint.RuntimeHints; +import org.springframework.aot.hint.RuntimeHintsRegistrar; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.lang.NonNull; +import org.springframework.util.ClassUtils; + +/** + * Registers native binding hints for collector-side message payloads. + */ +@Slf4j +public class CollectorRuntimeHintsRegistrar implements RuntimeHintsRegistrar { + + private static final String JOB_PACKAGE = "org.apache.hertzbeat.common.entity.job"; + private static final String JOB_PROTOCOL_PACKAGE = "org.apache.hertzbeat.common.entity.job.protocol"; + + @Override + public void registerHints(@NonNull RuntimeHints hints, ClassLoader classLoader) { + BindingReflectionHintsRegistrar bindingRegistrar = new BindingReflectionHintsRegistrar(); + registerType(bindingRegistrar, hints, ServerInfo.class); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PACKAGE); + scanBindingPackage(classLoader, bindingRegistrar, hints, JOB_PROTOCOL_PACKAGE); + hints.reflection().registerType(NettyAllocationManager.class, MemberCategory.DECLARED_FIELDS); + registerType(bindingRegistrar, hints, Schema.class); + registerType(bindingRegistrar, hints, Field.class); + registerType(bindingRegistrar, hints, FieldType.class); + registerType(bindingRegistrar, hints, DictionaryEncoding.class); + registerType(bindingRegistrar, hints, DateUnit.class); + registerType(bindingRegistrar, hints, FloatingPointPrecision.class); + registerType(bindingRegistrar, hints, IntervalUnit.class); + registerType(bindingRegistrar, hints, MetadataVersion.class); + registerType(bindingRegistrar, hints, TimeUnit.class); + registerType(bindingRegistrar, hints, UnionMode.class); + for (Class nestedClass : ArrowType.class.getDeclaredClasses()) { + if (!nestedClass.isAnnotation() && !nestedClass.isInterface()) { + registerType(bindingRegistrar, hints, nestedClass); + } + } + } + + private void scanBindingPackage(ClassLoader classLoader, BindingReflectionHintsRegistrar bindingRegistrar, + RuntimeHints hints, String basePackage) { + for (Class clazz : findBindingTypes(basePackage, classLoader)) { + registerType(bindingRegistrar, hints, clazz); + } + } + + private void registerType(BindingReflectionHintsRegistrar bindingRegistrar, RuntimeHints hints, Class clazz) { + bindingRegistrar.registerReflectionHints(hints.reflection(), clazz); + } + + private Set> findBindingTypes(String basePackage, ClassLoader classLoader) { + Set> bindingTypes = new LinkedHashSet<>(); + ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false); + TypeFilter includeAll = (metadataReader, metadataReaderFactory) -> true; + scanner.addIncludeFilter(includeAll); + for (BeanDefinition candidate : scanner.findCandidateComponents(basePackage)) { + String className = candidate.getBeanClassName(); + if (className == null) { + continue; + } + try { + Class clazz = ClassUtils.forName(className, classLoader); + if (!clazz.isAnnotation() && !clazz.isInterface()) { + bindingTypes.add(clazz); + } + } catch (Throwable ex) { + log.debug("Skip native binding hint registration for {}", className, ex); + } + } + return bindingTypes; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java new file mode 100644 index 00000000000..1926c48cfb8 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaults.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import java.util.Map; +import org.springframework.boot.SpringApplication; +import org.springframework.core.NativeDetector; + +/** + * Applies collector defaults without forking {@code application.yml}. + */ +public final class NativeCollectorDefaults { + + static final String AUTOCONFIGURE_EXCLUDE_PROPERTY = "spring.autoconfigure.exclude"; + static final String JVM_AUTOCONFIGURE_EXCLUDES = String.join(",", + "org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration", + "org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration", + "org.springframework.boot.hibernate.autoconfigure.HibernateJpaAutoConfiguration"); + static final String NATIVE_AUTOCONFIGURE_EXCLUDES = String.join(",", + JVM_AUTOCONFIGURE_EXCLUDES, + "org.springframework.boot.data.jpa.autoconfigure.DataJpaRepositoriesAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceInitializationAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.DataSourceTransactionManagerAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.health.DataSourceHealthContributorAutoConfiguration", + "org.springframework.boot.jdbc.autoconfigure.metrics.DataSourcePoolMetricsAutoConfiguration", + "org.springframework.boot.tomcat.autoconfigure.metrics.TomcatMetricsAutoConfiguration"); + + private NativeCollectorDefaults() { + } + + public static void applyTo(SpringApplication application) { + application.setDefaultProperties(defaultProperties(NativeDetector.inNativeImage())); + } + + static Map defaultProperties(boolean nativeImage) { + return Map.of( + AUTOCONFIGURE_EXCLUDE_PROPERTY, + nativeImage ? NATIVE_AUTOCONFIGURE_EXCLUDES : JVM_AUTOCONFIGURE_EXCLUDES); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..731da64e615 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/META-INF/spring.factories @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Intentionally left blank. +# Native collector defaults are applied from Collector.main. diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml index 9615e547b94..03702ef5627 100644 --- a/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/main/resources/application.yml @@ -27,9 +27,6 @@ spring: timeout-per-shutdown-phase: 10s jackson: default-property-inclusion: ALWAYS - # need to disable spring boot mongodb auto config, or default mongodb connection tried and failed... - autoconfigure: - exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration, org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration management: endpoints: web: @@ -77,6 +74,13 @@ common: type: netty hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json new file mode 100644 index 00000000000..227a978fe57 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/native-image/org.apache.hertzbeat/hertzbeat-collector-collector/reflect-config.json @@ -0,0 +1,54 @@ +[ + { + "name": "org.apache.hertzbeat.collector.Collector__ApplicationContextInitializer", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.collector.Collector__BeanFactoryRegistrations", + "allDeclaredConstructors": true, + "allDeclaredMethods": true + }, + { + "name": "org.apache.hertzbeat.common.entity.dto.ServerInfo", + "allDeclaredConstructors": true, + "allDeclaredFields": true, + "allDeclaredMethods": true + }, + { + "name": "io.netty.channel.kqueue.KQueueDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.kqueue.KQueueEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.epoll.EpollEventLoopGroup", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioDatagramChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.socket.nio.NioSocketChannel", + "allPublicConstructors": true + }, + { + "name": "io.netty.channel.nio.NioEventLoopGroup", + "allPublicConstructors": true + } +] diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect new file mode 100644 index 00000000000..81ed2990010 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/native/resources/META-INF/services/org.apache.hertzbeat.collector.collect.AbstractCollect @@ -0,0 +1,37 @@ +org.apache.hertzbeat.collector.collect.http.HttpCollectImpl +org.apache.hertzbeat.collector.collect.http.SslCertificateCollectImpl +org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect +org.apache.hertzbeat.collector.collect.icmp.IcmpCollectImpl +org.apache.hertzbeat.collector.collect.jmx.JmxCollectImpl +org.apache.hertzbeat.collector.collect.redis.RedisCommonCollectImpl +org.apache.hertzbeat.collector.collect.mongodb.MongodbSingleCollectImpl +org.apache.hertzbeat.collector.collect.rocketmq.RocketmqSingleCollectImpl +org.apache.hertzbeat.collector.collect.snmp.SnmpCollectImpl +org.apache.hertzbeat.collector.collect.ssh.SshCollectImpl +org.apache.hertzbeat.collector.collect.telnet.TelnetCollectImpl +org.apache.hertzbeat.collector.collect.smtp.SmtpCollectImpl +org.apache.hertzbeat.collector.collect.ntp.NtpCollectImpl +org.apache.hertzbeat.collector.collect.websocket.WebsocketCollectImpl +org.apache.hertzbeat.collector.collect.ftp.FtpCollectImpl +org.apache.hertzbeat.collector.collect.udp.UdpCollectImpl +org.apache.hertzbeat.collector.collect.push.PushCollectImpl +org.apache.hertzbeat.collector.collect.dns.DnsCollectImpl +org.apache.hertzbeat.collector.collect.nginx.NginxCollectImpl +org.apache.hertzbeat.collector.collect.memcached.MemcachedCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl +org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl +org.apache.hertzbeat.collector.collect.registry.RegistryImpl +org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl +org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl +org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl +org.apache.hertzbeat.collector.collect.script.ScriptCollectImpl +org.apache.hertzbeat.collector.collect.mqtt.MqttCollectImpl +org.apache.hertzbeat.collector.collect.ipmi2.IpmiCollectImpl +org.apache.hertzbeat.collector.collect.kafka.KafkaCollectImpl +org.apache.hertzbeat.collector.collect.sd.HttpSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.NacosSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.DnsSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.EurekaSdCollectImpl +org.apache.hertzbeat.collector.collect.sd.ConsulSdCollectImpl +org.apache.hertzbeat.collector.collect.modbus.ModbusCollectImpl +org.apache.hertzbeat.collector.collect.s7.S7CollectImpl diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..1ac654969d0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MariadbJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MariadbJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mariadbTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mariadb:11.4")) + .withExposedPorts(3306) + .withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mariadbTemplateMetrics = loadMariadbTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMariadbTemplateThroughJdbcQueryAdapter() { + return mariadbTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) || "slow_sql".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mariadb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMariadbTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mariadb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mariaDb) throws Exception { + execRoot(mariaDb, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mariaDb); + } + + private void generateSlowQuery(GenericContainer mariaDb) throws Exception { + execUser(mariaDb, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mariaDb) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mariaDb.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mariaDb, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MariaDB entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MariaDB TCP login to become ready"); + } + + private void execRoot(GenericContainer mariaDb, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mariaDb, String database, String sql) throws Exception { + var result = mariaDb.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java new file mode 100644 index 00000000000..9c5cabe8428 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcDriverAvailabilityTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class MysqlJdbcDriverAvailabilityTest { + + @Test + void shouldTreatOnlyExtLibLocationsAsAutoJdbcSignal() { + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("/opt/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertTrue(MysqlJdbcDriverAvailability.isExtLibLocation("file:/C:/hertzbeat/ext-lib/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation("/Users/dev/.m2/repository/com/mysql/mysql-connector-j/9.0.0/mysql-connector-j-9.0.0.jar")); + assertFalse(MysqlJdbcDriverAvailability.isExtLibLocation(null)); + } + + @Test + void shouldIgnoreTestClasspathMysqlDriverWhenItIsNotFromExtLib() { + MysqlJdbcDriverAvailability availability = new MysqlJdbcDriverAvailability(); + + assertFalse(availability.hasMysqlJdbcDriver()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java new file mode 100644 index 00000000000..38d7265ddb8 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterCompatibilityIntegrationTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterCompatibilityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final Set REPRESENTATIVE_TEMPLATE_METRICS = Set.of("basic", "process_state"); + + private List representativeTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + representativeTemplateMetrics = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> REPRESENTATIVE_TEMPLATE_METRICS.contains(metric.getName())) + .collect(Collectors.toList()); + } + + @TestFactory + Stream shouldCollectRepresentativeTemplateMetricsAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyRepresentativeMetrics(target))); + } + + private void verifyRepresentativeMetrics(DatabaseTarget target) throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try (GenericContainer container = createContainer(target)) { + jdbcQueryExecutor.afterPropertiesSet(); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + for (Metrics templateMetric : representativeTemplateMetrics) { + Metrics metric = materializeMetric(templateMetric, container); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> target.name() + " " + metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> target.name() + " " + metric.getName() + " should keep the original parser output shape"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> target.name() + " basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + () -> target.name() + " basic.version should be collected"); + } + if ("process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, + () -> target.name() + " process_state should return at least one grouped state row"); + } + } + } finally { + jdbcQueryExecutor.destroy(); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the database entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java new file mode 100644 index 00000000000..3d91c7107d6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryAdapterTemplateIntegrationTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryAdapterTemplateIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List mysqlTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initMonitoringData(container); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + mysqlTemplateMetrics = loadMysqlTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialMysqlTemplateThroughJdbcQueryAdapter() { + return mysqlTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(container); + } + if ("slow_sql".equals(metric.getName())) { + generateSlowQuery(container); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("columns".equals(metric.getJdbc().getQueryType())) { + assertEquals(1, metricsData.getValuesCount(), () -> metric.getName() + " should keep the original single-row shape"); + } + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("process_state".equals(metric.getName()) + || "slow_sql".equals(metric.getName()) + || "account_expiry".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void initMonitoringData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(mysql); + } + + private void generateSlowQuery(GenericContainer mysql) throws Exception { + execUser(mysql, TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery(GenericContainer mysql) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + mysql.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(GenericContainer mysql, String database, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java new file mode 100644 index 00000000000..650e6e92c5c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlJdbcQueryParityIntegrationTest.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryExecutorRegistry; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MysqlJdbcQueryParityIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + private static final String PARITY_TABLE = "collector_parity_metrics"; + + private Metrics basicTemplateMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + assertDoesNotThrow(() -> Class.forName("com.mysql.cj.jdbc.Driver")); + basicTemplateMetric = loadMysqlTemplate().getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-mysql.yml")); + } + + @AfterEach + void clearRegisteredExecutors() throws Exception { + Field executorsField = JdbcQueryExecutorRegistry.class.getDeclaredField("EXECUTORS"); + executorsField.setAccessible(true); + @SuppressWarnings("unchecked") + CopyOnWriteArrayList executors = (CopyOnWriteArrayList) executorsField.get(null); + executors.clear(); + } + + @TestFactory + Stream shouldMatchJdbcResultsForRepresentativeMysqlQueryShapes() { + return Stream.of( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44")), + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"))) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyParityAcrossTarget(target))); + } + + private void verifyParityAcrossTarget(DatabaseTarget target) throws Exception { + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container, TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE); + initParityData(container); + + List parityMetrics = List.of( + materializeMetric(basicTemplateMetric, container), + buildColumnsParityMetric(container), + buildOneRowParityMetric(container), + buildMultiRowParityMetric(container)); + + for (Metrics parityMetric : parityMetrics) { + CollectRep.MetricsData jdbcResult = collectWithJdbc(parityMetric); + CollectRep.MetricsData r2dbcResult = collectWithR2dbc(parityMetric); + + assertEquals(CollectRep.Code.SUCCESS, jdbcResult.getCode(), + () -> target.name() + " JDBC baseline failed for " + parityMetric.getName() + ": " + jdbcResult.getMsg()); + assertEquals(CollectRep.Code.SUCCESS, r2dbcResult.getCode(), + () -> target.name() + " R2DBC path failed for " + parityMetric.getName() + ": " + r2dbcResult.getMsg()); + assertEquals(jdbcResult.getFields(), r2dbcResult.getFields(), + () -> target.name() + " field set differs for " + parityMetric.getName()); + assertEquals(normalizeRows(jdbcResult), normalizeRows(r2dbcResult), + () -> target.name() + " row payload differs for " + parityMetric.getName()); + assertFalse(r2dbcResult.getValues().isEmpty(), + () -> target.name() + " " + parityMetric.getName() + " should return at least one row"); + } + } + } + + private CollectRep.MetricsData collectWithJdbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } + + private CollectRep.MetricsData collectWithR2dbc(Metrics metric) throws Exception { + clearRegisteredExecutors(); + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + try { + jdbcQueryExecutor.afterPropertiesSet(); + return collect(JsonUtil.fromJson(JsonUtil.toJson(metric), Metrics.class)); + } finally { + jdbcQueryExecutor.destroy(); + clearRegisteredExecutors(); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("mysql") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private Metrics buildColumnsParityMetric(GenericContainer container) { + return buildMetric( + "columns-parity", + List.of(field("version"), field("max_connections"), field("character_set_server")), + List.of("version", "max_connections", "character_set_server"), + "columns", + "SHOW VARIABLES WHERE Variable_name IN ('version', 'max_connections', 'character_set_server')", + container); + } + + private Metrics buildOneRowParityMetric(GenericContainer container) { + return buildMetric( + "one-row-parity", + List.of(field("answer"), field("label"), field("nullable_value")), + List.of("answer", "label", "nullable_value"), + "oneRow", + "SELECT 42 AS answer, 'adapter-parity' AS label, NULL AS nullable_value", + container); + } + + private Metrics buildMultiRowParityMetric(GenericContainer container) { + return buildMetric( + "multi-row-parity", + List.of(field("metric_name"), field("metric_value"), field("metric_note")), + List.of("metric_name", "metric_value", "metric_note"), + "multiRow", + "SELECT metric_name, metric_value, metric_note FROM " + PARITY_TABLE + " ORDER BY metric_name", + container); + } + + private Metrics buildMetric(String name, List fields, List aliasFields, + String queryType, String sql, GenericContainer container) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host(container.getHost()) + .port(String.valueOf(container.getMappedPort(3306))) + .platform("mysql") + .database(TEST_DATABASE) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .timeout(String.valueOf(Duration.ofSeconds(8).toMillis())) + .queryType(queryType) + .reuseConnection("false") + .url(buildJdbcUrl(container)) + .sql(sql) + .build(); + return Metrics.builder() + .name(name) + .protocol("jdbc") + .priority((byte) 1) + .fields(fields) + .aliasFields(aliasFields) + .jdbc(jdbcProtocol) + .build(); + } + + private Metrics materializeMetric(Metrics templateMetric, GenericContainer container) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(buildJdbcUrl(container)); + jdbcProtocol.setSshTunnel(null); + if (jdbcProtocol.getDatabase() == null || jdbcProtocol.getDatabase().contains("^_^")) { + jdbcProtocol.setDatabase(TEST_DATABASE); + } + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadMysqlTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + return new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + } + + private void initParityData(GenericContainer mysql) throws Exception { + execRoot(mysql, + "GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT SELECT ON " + TEST_DATABASE + ".* TO '" + TEST_USERNAME + "'@'%';" + + " DROP TABLE IF EXISTS " + TEST_DATABASE + "." + PARITY_TABLE + ";" + + " CREATE TABLE " + TEST_DATABASE + "." + PARITY_TABLE + " (" + + " metric_name VARCHAR(32) PRIMARY KEY," + + " metric_value VARCHAR(32) NOT NULL," + + " metric_note VARCHAR(32) NULL" + + " );" + + " INSERT INTO " + TEST_DATABASE + "." + PARITY_TABLE + + " (metric_name, metric_value, metric_note) VALUES" + + " ('alpha', '1', NULL)," + + " ('beta', '2', 'steady');" + + " FLUSH PRIVILEGES;"); + } + + private void awaitTcpLoginReady(GenericContainer mysql, String username, String password, String database) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand(username, password, database, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private void execRoot(GenericContainer mysql, String sql) throws Exception { + var result = mysql.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + private String buildJdbcUrl(GenericContainer container) { + return "jdbc:mysql://%s:%d/%s?allowPublicKeyRetrieval=true&useSSL=false" + .formatted(container.getHost(), container.getMappedPort(3306), TEST_DATABASE); + } + + private List> normalizeRows(CollectRep.MetricsData metricsData) { + return metricsData.getValues().stream() + .map(valueRow -> new ArrayList<>(valueRow.getColumnsList())) + .sorted(Comparator.comparing(row -> String.join("\u0001", row))) + .collect(Collectors.toList()); + } + + private Metrics.Field field(String name) { + return Metrics.Field.builder().field(name).type((byte) 1).build(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } + + private record DatabaseTarget(String name, DockerImageName image) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java new file mode 100644 index 00000000000..6162d5e193d --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/MysqlR2dbcJdbcQueryExecutorTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.hertzbeat.collector.collect.database.query.JdbcQueryRowSet; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.junit.jupiter.api.Test; + +class MysqlR2dbcJdbcQueryExecutorTest { + + @Test + void shouldAutoRouteToR2dbcOnlyWhenMysqlJdbcDriverIsAbsent() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(false); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertTrue(executor.supports(metrics("mysql", "columns"))); + assertTrue(executor.supports(metrics("mariadb", "columns"))); + assertTrue(executor.supports(metrics("mysql", "multiRow"))); + assertFalse(executor.supports(metrics("mysql", "runScript"))); + assertFalse(executor.supports(metrics("postgresql", "columns"))); + } + + @Test + void shouldPreferJdbcWhenMysqlJdbcDriverIsPresentInAutoMode() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldHonorExplicitQueryEngineOverrides() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + MysqlJdbcDriverAvailability driverAvailability = mock(MysqlJdbcDriverAvailability.class); + when(driverAvailability.hasMysqlJdbcDriver()).thenReturn(true); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mock(MysqlQueryExecutor.class), driverAvailability); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + assertTrue(executor.supports(metrics("mysql", "columns"))); + + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.JDBC); + assertFalse(executor.supports(metrics("mysql", "columns"))); + } + + @Test + void shouldExposeQueryResultsAsJdbcStyleRowSet() throws Exception { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder() + .columns(List.of("Variable_name", "Value")) + .rows(List.of( + List.of("Threads_connected", "5"), + List.of("Uptime", "10"))) + .elapsedMs(11) + .rowCount(2) + .build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + try (JdbcQueryRowSet rowSet = executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)) { + assertTrue(rowSet.next()); + assertEquals("Threads_connected", rowSet.getString(1)); + assertEquals("5", rowSet.getString(2)); + assertEquals("5", rowSet.getString("value")); + + assertTrue(rowSet.next()); + assertEquals("Uptime", rowSet.getString("variable_name")); + assertEquals("10", rowSet.getString(2)); + assertFalse(rowSet.next()); + } + } + + @Test + void shouldFailFastWhenR2dbcQueryReturnsError() { + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + MysqlQueryExecutor mysqlQueryExecutor = mock(MysqlQueryExecutor.class); + when(mysqlQueryExecutor.execute(anyString(), any())) + .thenReturn(QueryResult.builder().error("query timeout").build()); + MysqlR2dbcJdbcQueryExecutor executor = new MysqlR2dbcJdbcQueryExecutor( + properties, mysqlQueryExecutor, mock(MysqlJdbcDriverAvailability.class)); + + IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> executor.executeQuery(metrics("mysql", "columns"), 6000, 1000)); + assertTrue(exception.getMessage().contains("query timeout")); + } + + private Metrics metrics(String platform, String queryType) { + JdbcProtocol jdbcProtocol = JdbcProtocol.builder() + .host("127.0.0.1") + .port("3306") + .platform(platform) + .database("hzb") + .username("test") + .password("test123") + .queryType(queryType) + .sql("SHOW GLOBAL STATUS") + .timeout("6000") + .build(); + Metrics metrics = new Metrics(); + metrics.setProtocol("jdbc"); + metrics.setName("status"); + metrics.setJdbc(jdbcProtocol); + return metrics; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..e95cfa62052 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/OceanbaseJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class OceanbaseJdbcQueryAdapterIntegrationTest { + + private static final String OCEANBASE_IMAGE = "oceanbase/oceanbase-ce:latest"; + private static final String OCEANBASE_USERNAME = "root@sys"; + private static final String OCEANBASE_PASSWORD = ""; + private static final String OCEANBASE_DATABASE = "oceanbase"; + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private List oceanbaseTemplateMetrics; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse(OCEANBASE_IMAGE)) + .withExposedPorts(2881) + .withEnv("MODE", "MINI") + .withEnv("OB_MEMORY_LIMIT", "4096M") + .withEnv("OB_SYSTEM_MEMORY", "1024M") + .withEnv("OB_DATAFILE_SIZE", "2048M") + .withEnv("OB_LOG_DISK_SIZE", "2048M") + .withCommand("bash", "-lc", "/usr/sbin/sshd || true; /root/boot/start.sh || true; tail -f /dev/null") + .waitingFor(Wait.forListeningPort()) + .withStartupTimeout(Duration.ofMinutes(5)); + container.start(); + awaitSysLoginReady(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + oceanbaseTemplateMetrics = loadOceanbaseTemplate().getMetrics(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @TestFactory + Stream shouldCollectOfficialOceanbaseTemplateThroughJdbcQueryAdapter() { + return oceanbaseTemplateMetrics.stream() + .map(templateMetric -> DynamicTest.dynamicTest(templateMetric.getName(), + () -> verifyTemplateMetric(templateMetric))); + } + + private void verifyTemplateMetric(Metrics templateMetric) throws Exception { + Metrics metric = materializeMetric(templateMetric); + if ("process_state".equals(metric.getName())) { + startBackgroundSleepQuery(); + } + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), + () -> metric.getName() + " failed: " + metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount(), + () -> metric.getName() + " fields should still be produced by the original parser"); + if ("basic".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, "basic should return data"); + assertNotNull(metricsData.getValues().getFirst().getColumns(0)); + assertTrue(!Objects.equals(CommonConstants.NULL_VALUE, metricsData.getValues().getFirst().getColumns(0)), + "basic.version should be collected through the adapted query path"); + } + if ("tenant".equals(metric.getName()) || "sql".equals(metric.getName()) || "process_state".equals(metric.getName())) { + assertTrue(metricsData.getValuesCount() > 0, () -> metric.getName() + " should return at least one row"); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("oceanbase") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + assertNotNull(collectDataDispatch.metricsData, metric.getName() + " should dispatch metrics data"); + return collectDataDispatch.metricsData; + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(2881))); + jdbcProtocol.setUsername(OCEANBASE_USERNAME); + jdbcProtocol.setPassword(OCEANBASE_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(12).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + jdbcProtocol.setDatabase(OCEANBASE_DATABASE); + if (metric.getAliasFields() == null || metric.getAliasFields().isEmpty()) { + metric.setAliasFields(metric.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())); + } + return metric; + } + + private Job loadOceanbaseTemplate() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-oceanbase.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + return yaml.loadAs(reader, Job.class); + } + } + + private void awaitSysLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(3).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", obclientCommand("select 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for OceanBase observer bootstrap to finish and accept sys tenant logins. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for OceanBase sys tenant login to become ready"); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = "nohup sh -lc '" + obclientCommand("select sleep(15);").replace("'", "'\"'\"'") + " >/tmp/oceanbase-process-state.log 2>&1' >/dev/null 2>&1 &"; + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private String obclientCommand(String sql) { + StringBuilder command = new StringBuilder("obclient -h127.0.0.1 -P2881 -u") + .append(OCEANBASE_USERNAME) + .append(" -D") + .append(OCEANBASE_DATABASE) + .append(" -A "); + if (!OCEANBASE_PASSWORD.isEmpty()) { + command.append("-p").append(OCEANBASE_PASSWORD).append(' '); + } + command.append("-e \"").append(sql.replace("\"", "\\\"")).append('"'); + return command.toString(); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java new file mode 100644 index 00000000000..0d57526d95e --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/database/mysql/TidbJdbcQueryAdapterIntegrationTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.database.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; +import org.apache.hertzbeat.collector.collect.strategy.CollectStrategyFactory; +import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch; +import org.apache.hertzbeat.collector.dispatch.MetricsCollect; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.timer.WheelTimerTask; +import org.apache.hertzbeat.common.constants.CommonConstants; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.common.timer.Timeout; +import org.apache.hertzbeat.common.util.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.yaml.snakeyaml.Yaml; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TidbJdbcQueryAdapterIntegrationTest { + + private static final String TIDB_USERNAME = "root"; + private static final String TIDB_PASSWORD = ""; + private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); + + private GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + private Metrics tidbBasicMetric; + + @BeforeAll + void setUp() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + new CollectStrategyFactory().run(); + container = new GenericContainer<>(DockerImageName.parse("pingcap/tidb:v7.5.1")) + .withCommand("--store=unistore", "--path=") + .withExposedPorts(4000, 10080) + .waitingFor(Wait.forLogMessage(".*server is running MySQL protocol.*", 1)); + container.start(); + waitForStatusEndpoint(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + tidbBasicMetric = loadTidbBasicMetric(); + } + + @AfterAll + void tearDown() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + } + if (container != null) { + container.stop(); + } + } + + @Test + void shouldCollectTidbBasicMetricThroughMysqlCompatibleQueryAdapter() { + Metrics metric = materializeMetric(tidbBasicMetric); + CollectRep.MetricsData metricsData = collect(metric); + assertEquals(CollectRep.Code.SUCCESS, metricsData.getCode(), metricsData.getMsg()); + assertEquals(metric.getFields().size(), metricsData.getFieldsCount()); + assertTrue(metricsData.getValuesCount() > 0); + assertNotNull(metricsData.getValues().getFirst()); + assertTrue(metricsData.getValues().getFirst().getColumnsList().stream() + .anyMatch(value -> !Objects.equals(CommonConstants.NULL_VALUE, value) && !value.isEmpty()), + "TiDB basic should still return at least one concrete field value through the adapted query path"); + } + + private Metrics materializeMetric(Metrics templateMetric) { + Metrics metric = JsonUtil.fromJson(JsonUtil.toJson(templateMetric), Metrics.class); + JdbcProtocol jdbcProtocol = metric.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(4000))); + jdbcProtocol.setUsername(TIDB_USERNAME); + jdbcProtocol.setPassword(TIDB_PASSWORD); + jdbcProtocol.setTimeout(String.valueOf(Duration.ofSeconds(8).toMillis())); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setDatabase(null); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return metric; + } + + private Metrics loadTidbBasicMetric() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-tidb.yml") + .toAbsolutePath() + .normalize(); + Yaml yaml = new Yaml(); + try (Reader reader = Files.newBufferedReader(template)) { + Job job = yaml.loadAs(reader, Job.class); + return job.getMetrics().stream() + .filter(metric -> "basic".equals(metric.getName())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the basic metric in app-tidb.yml")); + } + } + + private CollectRep.MetricsData collect(Metrics metric) { + Job job = Job.builder() + .monitorId(1L) + .tenantId(1L) + .app("tidb") + .defaultInterval(600L) + .metadata(new HashMap<>(0)) + .labels(new HashMap<>(0)) + .annotations(new HashMap<>(0)) + .configmap(new ArrayList<>(0)) + .metrics(new ArrayList<>(List.of(metric))) + .build(); + WheelTimerTask timerTask = new WheelTimerTask(job, timeout -> { + }); + CapturingCollectDataDispatch collectDataDispatch = new CapturingCollectDataDispatch(); + MetricsCollect metricsCollect = new MetricsCollect( + metric, + new StubTimeout(timerTask), + collectDataDispatch, + "collector-test", + List.of()); + metricsCollect.run(); + return collectDataDispatch.metricsData; + } + + private void waitForStatusEndpoint() throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String statusUrl = "http://" + container.getHost() + ":" + container.getMappedPort(10080) + "/status"; + while (System.currentTimeMillis() < deadline) { + try { + HttpRequest request = HttpRequest.newBuilder(URI.create(statusUrl)) + .GET() + .timeout(Duration.ofSeconds(3)) + .build(); + HttpResponse response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) { + return; + } + } catch (Exception ignored) { + // Wait for the TiDB status endpoint to become available. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for TiDB status endpoint"); + } + + private static final class CapturingCollectDataDispatch implements CollectDataDispatch { + + private CollectRep.MetricsData metricsData; + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.MetricsData metricsData) { + this.metricsData = metricsData; + } + + @Override + public void dispatchCollectData(Timeout timeout, Metrics metrics, List metricsDataList) { + if (metricsDataList != null && !metricsDataList.isEmpty()) { + this.metricsData = metricsDataList.getFirst(); + } + } + } + + private record StubTimeout(WheelTimerTask wheelTimerTask) implements Timeout { + + @Override + public org.apache.hertzbeat.common.timer.Timer timer() { + return null; + } + + @Override + public org.apache.hertzbeat.common.timer.TimerTask task() { + return wheelTimerTask; + } + + @Override + public boolean isExpired() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean cancel() { + return false; + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java new file mode 100644 index 00000000000..45310f52abe --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/collect/strategy/CollectStrategyFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.strategy; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hertzbeat.collector.dispatch.DispatchConstants; +import org.junit.jupiter.api.Test; + +class CollectStrategyFactoryTest { + + @Test + void shouldRegisterCommonProtocolsFromServiceLoader() throws Exception { + new CollectStrategyFactory().run(); + + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_JDBC)); + assertNotNull(CollectStrategyFactory.invoke(DispatchConstants.PROTOCOL_HTTP)); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java new file mode 100644 index 00000000000..811259ecc16 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-collector/src/test/java/org/apache/hertzbeat/collector/nativex/NativeCollectorDefaultsTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.nativex; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.junit.jupiter.api.Test; + +class NativeCollectorDefaultsTest { + + @Test + void shouldProvideNativeAutoconfigureExcludesWhenNativeImage() { + Map properties = NativeCollectorDefaults.defaultProperties(true); + assertEquals(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void shouldProvideJvmAutoconfigureExcludesForJvmCollector() { + Map properties = NativeCollectorDefaults.defaultProperties(false); + assertEquals(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES, + properties.get(NativeCollectorDefaults.AUTOCONFIGURE_EXCLUDE_PROPERTY)); + } + + @Test + void nativeAutoconfigureExcludesShouldExtendJvmExcludes() { + assertTrue(NativeCollectorDefaults.NATIVE_AUTOCONFIGURE_EXCLUDES + .startsWith(NativeCollectorDefaults.JVM_AUTOCONFIGURE_EXCLUDES)); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-common/pom.xml b/hertzbeat-collector/hertzbeat-collector-common/pom.xml index d4cffe23fb7..7e0de3eda8a 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-common/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java index 16b3087e895..de65033abef 100644 --- a/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java +++ b/hertzbeat-collector/hertzbeat-collector-common/src/main/java/org/apache/hertzbeat/collector/collect/common/ssh/SshTunnelHelper.java @@ -17,10 +17,6 @@ package org.apache.hertzbeat.collector.collect.common.ssh; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.Scheduler; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -35,7 +31,6 @@ import java.io.IOException; import java.net.ServerSocket; import java.security.GeneralSecurityException; -import java.time.Duration; import java.util.Comparator; import java.util.ArrayList; import java.util.Iterator; @@ -53,34 +48,7 @@ public class SshTunnelHelper { private static final long DEFAULT_CACHE_TIMEOUT = 500 * 1000; - private static final Cache TRACKER_CACHE = - Caffeine.newBuilder() - .initialCapacity(1) - .maximumSize(1000) - .expireAfterAccess(Duration.ofMillis(DEFAULT_CACHE_TIMEOUT)) - .scheduler(Scheduler.systemScheduler()) - .removalListener((key, value, cause) -> { - if (cause == RemovalCause.REPLACED) { - return; - } - if (key != null && value != null) { - // 1. try close tunnel - SshClientSessionWrapper clientSessionWrapper = (SshClientSessionWrapper) key; - LocalPortForwardingWrapper wrapper = (LocalPortForwardingWrapper) value; - wrapper.remove(clientSessionWrapper.getClientSession()); - - // 2. try close session - if (!clientSessionWrapper.isShareConnection()) { - try { - clientSessionWrapper.close(); - log.info("[SSH Tunnel] close unshared ssh connection, {}", clientSessionWrapper); - } catch (IOException e) { - log.error("[SSH Tunnel] close unshared ssh connection error", e); - } - } - } - }) - .build(); + private static final Map TRACKER_CACHE = new ConcurrentHashMap<>(); /** @@ -122,15 +90,13 @@ public static int localPortForward(SshTunnel sshTunnel, String remoteHost, Strin // 2. get tunnel LocalPortForwardingWrapper forwardingWrapper = selectWrapper( - TRACKER_CACHE.getIfPresent(sessionWrapper), sessionWrapper, remoteHost, remotePort); + TRACKER_CACHE.get(sessionWrapper), sessionWrapper, remoteHost, remotePort); int localPort; if (forwardingWrapper == null) { localPort = getRandomPort(); LocalPortForwardingWrapper newForwardingWrapper = sessionWrapper .createLocalPortForwardingTracker(localPort, remoteHost, Integer.parseInt(remotePort)); - if (TRACKER_CACHE.getIfPresent(sessionWrapper) == null) { - TRACKER_CACHE.put(sessionWrapper, newForwardingWrapper); - } + TRACKER_CACHE.putIfAbsent(sessionWrapper, newForwardingWrapper); log.info("[SSH Tunnel] created ssh forwarding tracker ssh:{}, remote:{}, localPort:{}", sshTunnel.getHost() + ":" + sshTunnel.getPort(), remoteHost + ":" + remotePort, localPort); } else { @@ -184,6 +150,23 @@ private static int getRandomPort() throws IOException { } } + private static void removeSessionCacheEntry(ClientSession session) { + TRACKER_CACHE.entrySet().removeIf(entry -> { + if (!Objects.equals(entry.getKey().getClientSession(), session)) { + return false; + } + if (!entry.getKey().isShareConnection()) { + try { + entry.getKey().close(); + log.info("[SSH Tunnel] close unshared ssh connection, {}", entry.getKey()); + } catch (IOException e) { + log.error("[SSH Tunnel] close unshared ssh connection error", e); + } + } + return true; + }); + } + @Getter @Setter @EqualsAndHashCode @@ -267,6 +250,10 @@ public List select(ClientSession session, Predicate< list.add(wrapper); } } + if (trackerList.isEmpty()) { + map.remove(session); + removeSessionCacheEntry(session); + } return list; } @@ -290,6 +277,8 @@ public void remove(ClientSession session) { log.error("[SSH Tunnel] Remove ssh session local port forwarding error", e); } } + map.remove(session); + removeSessionCacheEntry(session); } public void close() throws IOException { diff --git a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml index 9acd5c98d94..7175050e818 100644 --- a/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-kafka/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -47,4 +47,4 @@ kafka-clients - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml index 1cdd2070f38..472dc55c3b9 100644 --- a/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-mongodb/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -46,4 +46,4 @@ mongodb-driver-sync - \ No newline at end of file + diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml new file mode 100644 index 00000000000..6c631b18978 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-collector + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc + ${project.artifactId} + + + ${java.version} + ${java.version} + UTF-8 + + + + + io.asyncer + r2dbc-mysql + ${r2dbc-mysql.version} + + + org.springframework.boot + spring-boot-autoconfigure + + + org.projectlombok + lombok + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + testcontainers-junit-jupiter + ${testcontainers.version} + test + + + io.netty + netty-tcnative-boringssl-static + 2.0.69.Final + test + + + diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java new file mode 100644 index 00000000000..0c476039e89 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlQueryExecutor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +/** + * Internal collector-side MySQL query executor. + */ +public interface MysqlQueryExecutor { + + /** + * Execute a single read-only SQL statement. + * + * @param sql SQL to execute + * @param options execution options and target connection settings + * @return normalized query result + */ + QueryResult execute(String sql, QueryOptions options); +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java new file mode 100644 index 00000000000..fcce4781bcd --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConfiguration.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring beans for the collector-side MySQL R2DBC route. + */ +@Configuration(proxyBeanMethods = false) +public class MysqlR2dbcConfiguration { + + @Bean + @ConditionalOnMissingBean + public SqlGuard mysqlR2dbcSqlGuard() { + return new SqlGuard(); + } + + @Bean + @ConditionalOnMissingBean + public ResultSetMapper mysqlR2dbcResultSetMapper() { + return new ResultSetMapper(); + } + + @Bean + @ConditionalOnMissingBean + public MysqlR2dbcConnectionFactoryProvider mysqlR2dbcConnectionFactoryProvider() { + return new MysqlR2dbcConnectionFactoryProvider(); + } + + @Bean + @ConditionalOnMissingBean(MysqlQueryExecutor.class) + public MysqlQueryExecutor mysqlQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + return new MysqlR2dbcQueryExecutor(connectionFactoryProvider, resultSetMapper, sqlGuard); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java new file mode 100644 index 00000000000..3121538c33c --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcConnectionFactoryProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionConfiguration; +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.asyncer.r2dbc.mysql.constant.TlsVersions; +import org.springframework.util.StringUtils; + +/** + * Builds R2DBC MySQL connection factories for the collector. + */ +public class MysqlR2dbcConnectionFactoryProvider { + + /** + * Create a connection factory for a target MySQL instance. + * + * @param options target connection settings + * @return connection factory + */ + public MySqlConnectionFactory create(QueryOptions options) { + return create(options, SslMode.PREFERRED); + } + + public MySqlConnectionFactory create(QueryOptions options, SslMode sslMode) { + if (!StringUtils.hasText(options.getHost())) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a target host"); + } + MySqlConnectionConfiguration.Builder builder = MySqlConnectionConfiguration.builder() + .host(options.getHost()) + .port(options.getPort()) + .connectTimeout(options.getTimeout()) + .sslMode(sslMode) + .tcpKeepAlive(true) + .tcpNoDelay(true); + if (sslMode.startSsl()) { + // Pin to TLSv1.2 because JDK 25 + the current server matrix is unreliable when TLSv1.3 is negotiated. + builder.tlsVersion(TlsVersions.TLS1_2); + } + if (StringUtils.hasText(options.getUsername())) { + builder.user(options.getUsername()); + } + if (options.getPassword() != null) { + builder.password(options.getPassword()); + } + if (StringUtils.hasText(options.resolvedDatabase())) { + builder.database(options.resolvedDatabase()); + } + return MySqlConnectionFactory.from(builder.build()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java new file mode 100644 index 00000000000..ab74d430204 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.asyncer.r2dbc.mysql.MySqlConnectionFactory; +import io.asyncer.r2dbc.mysql.constant.SslMode; +import io.r2dbc.spi.Connection; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.Locale; +import java.util.concurrent.TimeoutException; +import reactor.core.publisher.Mono; + +/** + * Collector-side MySQL query executor backed by R2DBC. + */ +public class MysqlR2dbcQueryExecutor implements MysqlQueryExecutor { + + private final MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider; + private final ResultSetMapper resultSetMapper; + private final SqlGuard sqlGuard; + + public MysqlR2dbcQueryExecutor(MysqlR2dbcConnectionFactoryProvider connectionFactoryProvider, + ResultSetMapper resultSetMapper, + SqlGuard sqlGuard) { + this.connectionFactoryProvider = connectionFactoryProvider; + this.resultSetMapper = resultSetMapper; + this.sqlGuard = sqlGuard; + } + + @Override + public QueryResult execute(String sql, QueryOptions options) { + String normalizedSql = sqlGuard.normalizeAndValidate(sql); + QueryResult firstAttempt = executeOnce(normalizedSql, options, SslMode.PREFERRED); + if (!firstAttempt.hasError() || !shouldRetryWithoutSsl(firstAttempt.getError())) { + return firstAttempt; + } + QueryResult fallbackAttempt = executeOnce(normalizedSql, options, SslMode.DISABLED); + if (!fallbackAttempt.hasError()) { + return fallbackAttempt; + } + if (requiresSslCompatibleAuth(fallbackAttempt.getError())) { + return QueryResult.builder() + .error(fallbackAttempt.getError() + + ". This route currently needs a TLS-compatible runtime or a mysql_native_password monitoring user.") + .build(); + } + return fallbackAttempt; + } + + private QueryResult executeOnce(String sql, QueryOptions options, SslMode sslMode) { + MySqlConnectionFactory connectionFactory = connectionFactoryProvider.create(options, sslMode); + Duration timeout = options.getTimeout().plusSeconds(1); + Connection connection = null; + try { + connection = Mono.from(connectionFactory.create()).block(timeout); + if (connection == null) { + return QueryResult.builder().error("R2DBC MySQL collector route could not create a connection").build(); + } + QueryResult result = map(connection, sql, options).block(timeout); + if (result == null) { + return QueryResult.builder().error("R2DBC MySQL collector route returned no result").build(); + } + return result; + } catch (IllegalArgumentException exception) { + throw exception; + } catch (Exception exception) { + String message = extractErrorMessage(exception, options.getTimeout()); + return QueryResult.builder() + .error(message) + .build(); + } finally { + if (connection != null) { + safelyClose(connection); + } + } + } + + private void safelyClose(Connection connection) { + try { + Mono.from(connection.close()) + .timeout(Duration.ofSeconds(1), Mono.empty()) + .onErrorResume(_ -> Mono.empty()) + .block(Duration.ofSeconds(2)); + } catch (Exception ignored) { + // Best-effort close only. Query completion must not be turned into a failure by cleanup. + } + } + + private String extractErrorMessage(Exception exception, Duration timeout) { + if (isTimeoutException(exception)) { + return "Query timed out after " + timeout.toMillis() + "ms"; + } + String message = exception.getMessage(); + if ((message == null || message.isBlank()) && exception.getCause() != null) { + message = exception.getCause().getMessage(); + } + return message == null || message.isBlank() ? exception.getClass().getSimpleName() : message; + } + + private boolean isTimeoutException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof TimeoutException) { + return true; + } + String message = current.getMessage(); + if (message != null && message.toLowerCase(Locale.ROOT).contains("timeout on blocking read")) { + return true; + } + current = current.getCause(); + } + return false; + } + + private boolean shouldRetryWithoutSsl(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("handshake_failure") + || normalized.contains("ssl/tls handshake") + || normalized.contains("closedchannelexception") + || normalized.contains("connection unexpectedly closed"); + } + + private boolean requiresSslCompatibleAuth(String error) { + if (error == null) { + return false; + } + String normalized = error.toLowerCase(Locale.ROOT); + return normalized.contains("caching_sha2_password") + || normalized.contains("must require ssl"); + } + + private Mono map(Connection connection, String sql, QueryOptions options) { + Statement statement = connection.createStatement(sql); + // Keep the query path read-only and deterministic, and let JdbcCommonCollect own the existing parser flow. + return resultSetMapper.map(statement, options.getTimeout(), options.getMaxRows()); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java new file mode 100644 index 00000000000..ffad2d25617 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryOptions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.time.Duration; + +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.springframework.util.StringUtils; + +/** + * Collector-only execution options for a target MySQL query. + */ +@Getter +@Builder(toBuilder = true) +@ToString(exclude = "password") +public class QueryOptions { + + private final String host; + @Builder.Default + private final int port = 3306; + private final String username; + private final String password; + private final String database; + private final String schema; + @Builder.Default + private final Duration timeout = Duration.ofSeconds(6); + @Builder.Default + private final int maxRows = 1000; + @Builder.Default + private final int fetchSize = 256; + @Builder.Default + private final boolean readOnly = true; + + public String resolvedDatabase() { + if (StringUtils.hasText(database)) { + return database; + } + if (StringUtils.hasText(schema)) { + return schema; + } + return null; + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java new file mode 100644 index 00000000000..93c5e580a04 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/QueryResult.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Collections; +import java.util.List; + +import lombok.Builder; +import lombok.Getter; + +/** + * Normalized query response. + */ +@Getter +@Builder(toBuilder = true) +public class QueryResult { + + @Builder.Default + private final List columns = Collections.emptyList(); + @Builder.Default + private final List> rows = Collections.emptyList(); + @Builder.Default + private final long elapsedMs = 0L; + @Builder.Default + private final int rowCount = 0; + private final String error; + + public boolean hasError() { + return error != null && !error.isBlank(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java new file mode 100644 index 00000000000..5ad8971c527 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapper.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Maps R2DBC results into the collector-neutral result model. + */ +public class ResultSetMapper { + + /** + * Execute and map a single statement. + * + * @param statement statement to execute + * @param timeout timeout for the query + * @param maxRows max rows to materialize + * @return normalized query result + */ + public Mono map(Statement statement, Duration timeout, int maxRows) { + long startNanos = System.nanoTime(); + ResultAccumulator accumulator = new ResultAccumulator(); + return Flux.from(statement.execute()) + .concatMap(result -> Flux.from(result.map(accumulator::mapRow))) + .take(maxRows) + .collectList() + .map(rows -> accumulator.toQueryResult(rows, elapsedMillis(startNanos))) + .timeout(timeout); + } + + private static long elapsedMillis(long startNanos) { + return Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + } + + private static ColumnLayout extractColumns(RowMetadata metadata) { + List columns = new ArrayList<>(); + metadata.getColumnMetadatas().forEach(columnMetadata -> columns.add(columnMetadata.getName())); + return new ColumnLayout(List.copyOf(columns), columns.size()); + } + + private static List extractRow(Row row, int columnCount) { + List values = new ArrayList<>(columnCount); + for (int index = 0; index < columnCount; index++) { + Object value = row.get(index); + values.add(value == null ? null : String.valueOf(value)); + } + return values; + } + + private record ColumnLayout(List columns, int columnCount) { + } + + private static final class ResultAccumulator { + + private ColumnLayout columnLayout; + + private List mapRow(Row row, RowMetadata metadata) { + if (columnLayout == null) { + columnLayout = extractColumns(metadata); + } + return extractRow(row, columnLayout.columnCount()); + } + + private QueryResult toQueryResult(List> rows, long elapsedMs) { + return QueryResult.builder() + .columns(columnLayout == null ? List.of() : columnLayout.columns()) + .rows(rows) + .rowCount(rows.size()) + .elapsedMs(elapsedMs) + .build(); + } + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java new file mode 100644 index 00000000000..47cb257a236 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/main/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuard.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Minimal SQL guard for the built-in read-only MySQL collector route. + */ +public class SqlGuard { + + private static final Pattern TRAILING_SEMICOLONS = Pattern.compile(";\\s*$"); + private static final Pattern COMMENTS = Pattern.compile("(/\\*|\\*/|--|#)"); + private static final Pattern FORBIDDEN = Pattern.compile( + "\\b(insert|update|delete|replace|merge|alter|drop|truncate|create|call)\\b"); + + /** + * Normalize a single read-only SQL statement and reject obvious unsafe statements. + * + * @param sql sql text + * @return normalized sql text + */ + public String normalizeAndValidate(String sql) { + if (sql == null || sql.isBlank()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + String normalized = TRAILING_SEMICOLONS.matcher(sql.trim()).replaceAll("").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("R2DBC MySQL collector route requires a non-empty SQL statement"); + } + if (normalized.indexOf(';') >= 0) { + throw new IllegalArgumentException("R2DBC MySQL collector route only allows a single SQL statement"); + } + if (COMMENTS.matcher(normalized).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route does not allow SQL comments"); + } + + String lower = normalized.toLowerCase(Locale.ROOT); + if (!(lower.startsWith("select") || lower.startsWith("show"))) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports SELECT or SHOW statements"); + } + + String stripped = stripQuotedContent(lower); + if (FORBIDDEN.matcher(stripped).find()) { + throw new IllegalArgumentException("R2DBC MySQL collector route only supports read-only statements"); + } + return normalized; + } + + private String stripQuotedContent(String sql) { + StringBuilder builder = new StringBuilder(sql.length()); + char quote = 0; + for (int i = 0; i < sql.length(); i++) { + char current = sql.charAt(i); + if (quote == 0 && (current == '\'' || current == '"' || current == '`')) { + quote = current; + builder.append(' '); + continue; + } + if (quote != 0 && current == quote) { + quote = 0; + builder.append(' '); + continue; + } + builder.append(quote == 0 ? current : ' '); + } + return builder.toString(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java new file mode 100644 index 00000000000..532c433c7f3 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlR2dbcQueryExecutorIntegrationTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +class MysqlR2dbcQueryExecutorIntegrationTest { + + private static final String TEST_DATABASE = "hzb"; + private static final String TEST_USERNAME = "test"; + private static final String TEST_PASSWORD = "test123"; + + private final MysqlQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), new ResultSetMapper(), new SqlGuard()); + + @TestFactory + Stream shouldRunReadOnlyQueriesAcrossCompatibilityMatrix() { + return Stream.of( + new DatabaseTarget("mysql-5.7", DockerImageName.parse("mysql:5.7.44"), false, false), + new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false), + new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true), + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true, false)) + .map(target -> DynamicTest.dynamicTest(target.name(), () -> verifyReadOnlyQueries(target))); + } + + @Test + void shouldRejectIllegalSqlBeforeConnecting() { + assertThrows(IllegalArgumentException.class, () -> executor.execute("DELETE FROM sample_metrics", QueryOptions.builder() + .host("127.0.0.1") + .port(3306) + .username("test") + .password("test123") + .database("hzb") + .build())); + } + + @Test + void shouldReturnTimeoutErrorOnSlowQuery() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-native-auth", DockerImageName.parse("mysql:8.0.36"), false, true); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + QueryResult result = executor.execute("SELECT SLEEP(3)", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(1))); + assertTrue(result.hasError()); + } + } + + @Test + void shouldSupportMysql8DefaultCachingSha2Users() throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + DatabaseTarget target = new DatabaseTarget("mysql-8.0-default-auth", DockerImageName.parse("mysql:8.0.36"), false, false); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + QueryResult result = executor.execute("SELECT 1 AS value", buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5))); + assertFalse(result.hasError(), () -> "mysql-8.0-default-auth failed: " + result.getError()); + assertEquals(List.of("value"), result.getColumns()); + assertEquals(List.of(List.of("1")), result.getRows()); + } + } + + private void verifyReadOnlyQueries(DatabaseTarget target) throws Exception { + Assumptions.assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for integration tests"); + try (GenericContainer container = createContainer(target)) { + container.start(); + awaitTcpLoginReady(container); + initSchema(container); + + QueryOptions options = buildOptions(container, TEST_DATABASE, Duration.ofSeconds(5)); + + QueryResult selectResult = executor.execute("SELECT 1 AS value", options); + assertFalse(selectResult.hasError(), + () -> target.name() + " SELECT 1 failed: " + selectResult.getError()); + assertEquals(List.of("value"), selectResult.getColumns()); + assertEquals(List.of(List.of("1")), selectResult.getRows()); + + QueryResult showResult = executor.execute("SHOW VARIABLES LIKE 'version%'", options); + assertFalse(showResult.hasError(), + () -> target.name() + " SHOW VARIABLES failed: " + showResult.getError()); + assertTrue(showResult.getRowCount() > 0); + + QueryResult businessResult = executor.execute("SELECT label FROM sample_metrics WHERE id = 1", options); + assertFalse(businessResult.hasError(), + () -> target.name() + " business SQL failed: " + businessResult.getError()); + assertEquals(List.of(List.of("alpha")), businessResult.getRows()); + } + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer container = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + container.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", "root123"); + return container; + } + container.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", "root123"); + if (target.mysqlNativePasswordUser()) { + container.withCommand("--default-authentication-plugin=mysql_native_password"); + } + return container; + } + + private QueryOptions buildOptions(GenericContainer container, String database, Duration timeout) { + return QueryOptions.builder() + .host(normalizeLoopbackHost(container.getHost())) + .port(container.getMappedPort(3306)) + .username(TEST_USERNAME) + .password(TEST_PASSWORD) + .database(database) + .timeout(timeout) + .maxRows(1000) + .fetchSize(128) + .readOnly(true) + .build(); + } + + private void awaitTcpLoginReady(GenericContainer container) throws Exception { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(30).toMillis(); + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT 1\""); + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", command); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // The entrypoint may still be switching from the temporary bootstrap server to the final one. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String normalizeLoopbackHost(String host) { + return host; + } + + private void initSchema(GenericContainer container) throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"CREATE TABLE IF NOT EXISTS sample_metrics (id INT PRIMARY KEY, label VARCHAR(32));", + "REPLACE INTO sample_metrics (id, label) VALUES (1, 'alpha');\""); + container.execInContainer("sh", "-lc", command); + } + + private record DatabaseTarget(String name, DockerImageName image, boolean mariaDb, boolean mysqlNativePasswordUser) { + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java new file mode 100644 index 00000000000..687cf0613c0 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/MysqlSqlTemplateCompatibilityTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +class MysqlSqlTemplateCompatibilityTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + @SuppressWarnings("unchecked") + void shouldAcceptCurrentOfficialMysqlTemplateSql() throws IOException { + Path template = Path.of("..", "..", "hertzbeat-manager", "src", "main", "resources", "define", "app-mysql.yml") + .toAbsolutePath() + .normalize(); + assertTrue(Files.isRegularFile(template), "MySQL monitor template must exist"); + + Yaml yaml = new Yaml(); + int sqlCount = 0; + try (Reader reader = Files.newBufferedReader(template)) { + Map root = yaml.load(reader); + List> metrics = (List>) root.get("metrics"); + for (Map metric : metrics) { + Map jdbc = (Map) metric.get("jdbc"); + if (jdbc == null) { + continue; + } + Object sql = jdbc.get("sql"); + if (!(sql instanceof String sqlText)) { + continue; + } + String normalized = sqlGuard.normalizeAndValidate(sqlText); + assertFalse(normalized.isBlank(), "Normalized SQL should not be blank"); + sqlCount++; + } + } + assertTrue(sqlCount > 0, "MySQL monitor template should contain SQL statements"); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java new file mode 100644 index 00000000000..2ec085b46a6 --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/ResultSetMapperTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.r2dbc.spi.ColumnMetadata; +import io.r2dbc.spi.Result; +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.time.Duration; +import java.util.List; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; + +class ResultSetMapperTest { + + private final ResultSetMapper mapper = new ResultSetMapper(); + + @Test + @SuppressWarnings("unchecked") + void shouldCacheColumnLayoutAcrossRows() { + Statement statement = mock(Statement.class); + Result result = mock(Result.class); + RowMetadata metadata = mock(RowMetadata.class); + Row firstRow = mock(Row.class); + Row secondRow = mock(Row.class); + ColumnMetadata firstColumn = mock(ColumnMetadata.class); + ColumnMetadata secondColumn = mock(ColumnMetadata.class); + + when(firstColumn.getName()).thenReturn("id"); + when(secondColumn.getName()).thenReturn("label"); + doReturn(List.of(firstColumn, secondColumn)).when(metadata).getColumnMetadatas(); + when(firstRow.get(0)).thenReturn(1); + when(firstRow.get(1)).thenReturn("alpha"); + when(secondRow.get(0)).thenReturn(2); + when(secondRow.get(1)).thenReturn("beta"); + when(result.map(any(BiFunction.class))).thenAnswer(invocation -> { + BiFunction> mapping = + (BiFunction>) invocation.getArgument(0); + return Flux.just( + mapping.apply(firstRow, metadata), + mapping.apply(secondRow, metadata)); + }); + doReturn(Flux.just(result)).when(statement).execute(); + + QueryResult queryResult = mapper.map(statement, Duration.ofSeconds(1), 10).block(); + + assertNotNull(queryResult); + assertEquals(List.of("id", "label"), queryResult.getColumns()); + assertEquals(List.of( + List.of("1", "alpha"), + List.of("2", "beta")), queryResult.getRows()); + assertEquals(2, queryResult.getRowCount()); + verify(metadata, times(1)).getColumnMetadatas(); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java new file mode 100644 index 00000000000..b46235c47cc --- /dev/null +++ b/hertzbeat-collector/hertzbeat-collector-mysql-r2dbc/src/test/java/org/apache/hertzbeat/collector/mysql/r2dbc/SqlGuardTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.mysql.r2dbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class SqlGuardTest { + + private final SqlGuard sqlGuard = new SqlGuard(); + + @Test + void shouldNormalizeTrailingSemicolon() { + assertEquals("SELECT 1", sqlGuard.normalizeAndValidate("SELECT 1;")); + } + + @Test + void shouldAllowShowStatement() { + assertEquals("SHOW VARIABLES LIKE 'version%'", sqlGuard.normalizeAndValidate("SHOW VARIABLES LIKE 'version%'")); + } + + @Test + void shouldRejectWriteStatement() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("DELETE FROM test")); + } + + @Test + void shouldRejectMultipleStatements() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1; SELECT 2")); + } + + @Test + void shouldRejectComments() { + assertThrows(IllegalArgumentException.class, () -> sqlGuard.normalizeAndValidate("SELECT 1 -- comment")); + } +} diff --git a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml index b21d4554633..9ea2e9d914e 100644 --- a/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml +++ b/hertzbeat-collector/hertzbeat-collector-nebulagraph/pom.xml @@ -29,8 +29,8 @@ ${project.artifactId} - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -57,4 +57,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-collector/pom.xml b/hertzbeat-collector/pom.xml index 48daddead48..f1e1454623c 100644 --- a/hertzbeat-collector/pom.xml +++ b/hertzbeat-collector/pom.xml @@ -28,15 +28,17 @@ ${project.artifactId} pom - 21 + 25 ${java.version} ${java.version} + 2024.0.3 hertzbeat-collector-basic hertzbeat-collector-common hertzbeat-collector-collector + hertzbeat-collector-mysql-r2dbc hertzbeat-collector-mongodb hertzbeat-collector-nebulagraph hertzbeat-collector-rocketmq @@ -45,6 +47,13 @@ + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + org.apache.hertzbeat hertzbeat-collector-common @@ -55,6 +64,11 @@ hertzbeat-collector-basic ${hertzbeat.version} + + org.apache.hertzbeat + hertzbeat-collector-mysql-r2dbc + ${hertzbeat.version} + org.apache.hertzbeat hertzbeat-collector-mongodb diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml index af0d937df12..316661344be 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-basic-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 42.5.5 diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml index 323e3c57b7e..1e2952bdf28 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-common-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 @@ -67,4 +67,4 @@ - \ No newline at end of file + diff --git a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml index d56825f6def..3c1824951d1 100644 --- a/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-kafka-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-collector-kafka-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml new file mode 100644 index 00000000000..49b2f517428 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + + org.apache.hertzbeat + hertzbeat-e2e + 2.0-SNAPSHOT + + + hertzbeat-collector-mysql-r2dbc-e2e + + + ${java.version} + ${java.version} + UTF-8 + 2024.0.3 + + + + + + io.projectreactor + reactor-bom + ${mysql.r2dbc.reactor.bom.version} + pom + import + + + + + + + org.apache.hertzbeat + hertzbeat-startup + ${hertzbeat.version} + test + + + com.mysql + mysql-connector-j + + + org.springframework.boot + spring-boot-starter-webflux + + + + + org.apache.hertzbeat + hertzbeat-collector-common-e2e + ${hertzbeat.version} + test + test-jar + + + org.apache.hertzbeat + hertzbeat-collector-basic + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-common + ${hertzbeat.version} + test + + + org.apache.hertzbeat + hertzbeat-collector-collector + ${hertzbeat.version} + test + + + io.projectreactor.netty + reactor-netty-core + test + + + + org.testcontainers + testcontainers-junit-jupiter + test + + + org.testcontainers + testcontainers-mysql + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + false + + + + + + diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..aff3ffb50cc --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/AbstractMysqlR2dbcCollectE2eTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; +import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlCollectorProperties; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlJdbcDriverAvailability; +import org.apache.hertzbeat.collector.collect.database.mysql.MysqlR2dbcJdbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.job.protocol.Protocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.Assertions; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Shared MySQL-compatible E2E support for the collector-side R2DBC adapter. + */ +abstract class AbstractMysqlR2dbcCollectE2eTest extends AbstractCollectE2eTest { + + protected static final String TEST_DATABASE = "hzb"; + protected static final String TEST_USERNAME = "test"; + protected static final String TEST_PASSWORD = "test123"; + protected static final String ROOT_PASSWORD = "root123"; + + protected GenericContainer container; + private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor; + + protected void setUpTarget(DatabaseTarget target) throws Exception { + super.setUp(); + collect = new JdbcCommonCollect(); + metrics = new Metrics(); + + container = createContainer(target); + container.start(); + awaitTcpLoginReady(); + initMonitoringData(); + + MysqlCollectorProperties properties = new MysqlCollectorProperties(); + properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC); + jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor( + properties, + new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()), + new MysqlJdbcDriverAvailability()); + jdbcQueryExecutor.afterPropertiesSet(); + } + + protected void tearDownTarget() throws Exception { + if (jdbcQueryExecutor != null) { + jdbcQueryExecutor.destroy(); + jdbcQueryExecutor = null; + } + if (container != null) { + container.stop(); + container = null; + } + } + + protected void assertMysqlJdbcDriverAbsent() { + Assertions.assertThrows(ClassNotFoundException.class, () -> Class.forName("com.mysql.cj.jdbc.Driver")); + } + + protected void collectMysqlTemplate(Set metricFilter) throws Exception { + Job mysqlJob = appService.getAppDefine("mysql"); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : mysqlJob.getMetrics()) { + if (metricFilter != null && !metricFilter.contains(metricsDef.getName())) { + continue; + } + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, + configmapFromPreCollectData.isEmpty() ? new HashMap<>() : configmapFromPreCollectData.getFirst()); + String metricName = metricsDef.getName(); + if ("process_state".equals(metricName)) { + startBackgroundSleepQuery(); + } + if ("slow_sql".equals(metricName)) { + generateSlowQuery(); + } + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true); + configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData); + } + } + + @Override + protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef); + metrics.setJdbc(jdbcProtocol); + CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder(); + metricsData.setApp("mysql"); + return collectMetricsData(metrics, metricsDef, metricsData); + } + + @Override + protected Protocol buildProtocol(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = metricsDef.getJdbc(); + jdbcProtocol.setHost(container.getHost()); + jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306))); + jdbcProtocol.setUsername(TEST_USERNAME); + jdbcProtocol.setPassword(TEST_PASSWORD); + jdbcProtocol.setDatabase(TEST_DATABASE); + jdbcProtocol.setTimeout("8000"); + jdbcProtocol.setReuseConnection("false"); + jdbcProtocol.setUrl(null); + jdbcProtocol.setSshTunnel(null); + return jdbcProtocol; + } + + private GenericContainer createContainer(DatabaseTarget target) { + GenericContainer mysql = new GenericContainer<>(target.image()) + .withExposedPorts(3306) + .waitingFor(Wait.forListeningPort()); + if (target.mariaDb()) { + return mysql.withEnv("MARIADB_DATABASE", TEST_DATABASE) + .withEnv("MARIADB_USER", TEST_USERNAME) + .withEnv("MARIADB_PASSWORD", TEST_PASSWORD) + .withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD); + } + return mysql.withEnv("MYSQL_DATABASE", TEST_DATABASE) + .withEnv("MYSQL_USER", TEST_USERNAME) + .withEnv("MYSQL_PASSWORD", TEST_PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD); + } + + private void initMonitoringData() throws Exception { + execRoot("GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';" + + " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';" + + " SET GLOBAL log_output='TABLE';" + + " SET GLOBAL slow_query_log='ON';" + + " SET GLOBAL long_query_time=0;" + + " FLUSH PRIVILEGES;"); + generateSlowQuery(); + } + + private void generateSlowQuery() throws Exception { + execUser(TEST_DATABASE, "SELECT SLEEP(0.2);"); + Thread.sleep(300); + } + + private void startBackgroundSleepQuery() throws Exception { + String command = String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "nohup sh -lc", + "'$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + TEST_USERNAME, + "-p" + TEST_PASSWORD, + TEST_DATABASE, + "-e", + "\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'", + ">/dev/null 2>&1 &"); + container.execInContainer("sh", "-lc", command); + Thread.sleep(500); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE, "SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready"); + } + + private void execRoot(String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("root mysql command failed: " + result.getStderr()); + } + } + + private void execUser(String database, String sql) throws Exception { + var result = container.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql)); + if (result.getExitCode() != 0) { + throw new IllegalStateException("user mysql command failed: " + result.getStderr()); + } + } + + private String mysqlCliCommand(String username, String password, String database, String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + username, + "-p" + password, + database, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } + + protected record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java new file mode 100644 index 00000000000..7c34f8a6947 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectCompatibilityE2eTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.utility.DockerImageName; + +/** + * Compatibility E2E coverage for the collector-side MySQL R2DBC adapter. + */ +class MysqlR2dbcCollectCompatibilityE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + private static final Set MARIADB_REPRESENTATIVE_METRICS = + Set.of("basic", "process_state", "slow_sql"); + + @TestFactory + Stream shouldCollectMysqlTemplateAcrossCompatibilityMatrixWithoutMysqlJdbcDriver() { + return Stream.of( + new MatrixTarget( + new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false), + null), + new MatrixTarget( + new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false), + null), + new MatrixTarget( + new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true), + MARIADB_REPRESENTATIVE_METRICS)) + .map(target -> DynamicTest.dynamicTest(target.databaseTarget().name(), () -> verifyTarget(target))); + } + + private void verifyTarget(MatrixTarget target) throws Exception { + setUpTarget(target.databaseTarget()); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(target.metricFilter()); + } finally { + tearDownTarget(); + } + } + + private record MatrixTarget(DatabaseTarget databaseTarget, Set metricFilter) { + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java new file mode 100644 index 00000000000..9bd4ebafe02 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e/src/test/java/org/apache/hertzbeat/collector/collect/mysql/MysqlR2dbcCollectE2eTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.collector.collect.mysql; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.testcontainers.utility.DockerImageName; + +/** + * E2E test for collector-side MySQL monitoring through the R2DBC query adapter. + */ +@Slf4j +class MysqlR2dbcCollectE2eTest extends AbstractMysqlR2dbcCollectE2eTest { + + @Test + void shouldCollectMysqlTemplateWithoutMysqlJdbcDriver() throws Exception { + DatabaseTarget target = new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false); + setUpTarget(target); + try { + assertMysqlJdbcDriverAbsent(); + collectMysqlTemplate(null); + } finally { + tearDownTarget(); + } + } +} diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml index c725c7bd35f..a5638e23a20 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml @@ -28,8 +28,8 @@ hertzbeat-log-e2e - 17 - 17 + ${java.version} + ${java.version} UTF-8 3.3.1 3.6.1 diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index dbcbad9c5df..6249cf0d447 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -31,13 +31,14 @@ hertzbeat-collector-common-e2e hertzbeat-collector-kafka-e2e hertzbeat-collector-basic-e2e + hertzbeat-collector-mysql-r2dbc-e2e hertzbeat-log-e2e true - 17 - 17 + ${java.version} + ${java.version} UTF-8 2.0.3 4.13.2 diff --git a/hertzbeat-startup/pom.xml b/hertzbeat-startup/pom.xml index b881e374d4c..90d3a3a663f 100644 --- a/hertzbeat-startup/pom.xml +++ b/hertzbeat-startup/pom.xml @@ -116,6 +116,18 @@ netty-all + + + io.projectreactor.netty + reactor-netty-core + 1.2.3 + + + io.projectreactor.netty + reactor-netty-http + 1.2.3 + + org.flywaydb @@ -150,6 +162,12 @@ spring-boot-starter-test test + + org.testcontainers + testcontainers + ${testcontainers.version} + test + diff --git a/hertzbeat-startup/src/main/resources/application.yml b/hertzbeat-startup/src/main/resources/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/hertzbeat-startup/src/main/resources/application.yml +++ b/hertzbeat-startup/src/main/resources/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java new file mode 100644 index 00000000000..b7e5c02e7c2 --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ReactorNettyCompatibilityTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.fail; + +import io.netty.channel.ChannelOption; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.netty.Connection; +import reactor.netty.tcp.TcpClient; + +/** + * Guards the startup runtime against Reactor Netty / Netty mismatches that only show up when a client loop + * is created for MySQL R2DBC collection. + */ +class ReactorNettyCompatibilityTest { + + @Test + void reactorNettyClientLoopCanBeCreated() { + Connection connection = null; + try { + connection = TcpClient.create() + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 1000) + .host("127.0.0.1") + .port(9) + .connect() + .onErrorResume(throwable -> Mono.empty()) + .block(Duration.ofSeconds(3)); + } catch (NoClassDefFoundError error) { + fail("Startup runtime is missing a Reactor Netty dependency needed by MySQL R2DBC collection", error); + } finally { + if (connection != null) { + connection.disposeNow(); + } + } + } +} diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java new file mode 100644 index 00000000000..b93b6c934dd --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/StartupMysqlR2dbcCompatibilityTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.time.Duration; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider; +import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryOptions; +import org.apache.hertzbeat.collector.mysql.r2dbc.QueryResult; +import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper; +import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * Verifies that the startup runtime can execute a real MySQL R2DBC query. + */ +class StartupMysqlR2dbcCompatibilityTest { + + private static final String DATABASE = "hzb"; + private static final String USERNAME = "test"; + private static final String PASSWORD = "test123"; + private static final String ROOT_PASSWORD = "root123"; + + private GenericContainer container; + + @AfterEach + void tearDown() { + if (container != null) { + container.stop(); + container = null; + } + } + + @Test + void shouldExecuteMysqlQueryOnStartupRuntimeClasspath() throws Exception { + container = new GenericContainer<>(DockerImageName.parse("mysql:8.0.36")) + .withExposedPorts(3306) + .withEnv("MYSQL_DATABASE", DATABASE) + .withEnv("MYSQL_USER", USERNAME) + .withEnv("MYSQL_PASSWORD", PASSWORD) + .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) + .waitingFor(Wait.forListeningPort()); + container.start(); + awaitTcpLoginReady(); + + MysqlR2dbcQueryExecutor executor = new MysqlR2dbcQueryExecutor( + new MysqlR2dbcConnectionFactoryProvider(), + new ResultSetMapper(), + new SqlGuard()); + QueryOptions options = QueryOptions.builder() + .host(container.getHost()) + .port(container.getMappedPort(3306)) + .database(DATABASE) + .username(USERNAME) + .password(PASSWORD) + .timeout(Duration.ofSeconds(8)) + .build(); + + QueryResult result = executor.execute("SELECT 1 AS ok", options); + + assertFalse(result.hasError(), () -> "Unexpected query error: " + result.getError()); + assertEquals(1, result.getRowCount()); + assertEquals("1", result.getRows().getFirst().getFirst()); + } + + private void awaitTcpLoginReady() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + while (System.currentTimeMillis() < deadline) { + try { + var result = container.execInContainer("sh", "-lc", + mysqlCliCommand("SELECT 1")); + if (result.getExitCode() == 0) { + return; + } + } catch (Exception ignored) { + // Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener. + } + Thread.sleep(1000); + } + throw new IllegalStateException("Timed out waiting for MySQL TCP login to become ready"); + } + + private String mysqlCliCommand(String sql) { + return String.join(" ", + "CLIENT=$(command -v mysql || command -v mariadb)", + "&&", + "$CLIENT --protocol=TCP -h127.0.0.1 -P3306", + "-u" + USERNAME, + "-p" + PASSWORD, + DATABASE, + "-e", + "\"" + sql.replace("\"", "\\\"") + "\""); + } +} diff --git a/home/docs/community/contribution.md b/home/docs/community/contribution.md index 72b823e716d..e1651205688 100644 --- a/home/docs/community/contribution.md +++ b/home/docs/community/contribution.md @@ -52,7 +52,7 @@ Even small corrections to typos are very welcome :) #### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` diff --git a/home/docs/community/development.md b/home/docs/community/development.md index fd2c83327cf..c52e5c075e1 100644 --- a/home/docs/community/development.md +++ b/home/docs/community/development.md @@ -12,7 +12,7 @@ sidebar_label: Development ### Backend start -1. Requires `maven3+`, `java21` and `lombok` environments +1. Requires `maven3+`, `java25` and `lombok` environments 2. (Optional) Modify the configuration file: `hertzbeat-startup/src/main/resources/application.yml` 3. Execute under the project root directory: `mvn clean install -DskipTests` 4. Add VM Options: `--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` @@ -34,7 +34,7 @@ sidebar_label: Development ## Build HertzBeat binary package -> Requires `maven3+`, `java21`, `node` and `pnpm` environments. +> Requires `maven3+`, `java25`, `node` and `pnpm` environments. ### Frontend build @@ -50,20 +50,23 @@ sidebar_label: Development ### Backend build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean package -Prelease` -The HertzBeat install package will at `dist/hertzbeat-{version}.tar.gz` +The HertzBeat install package will be generated at `dist/apache-hertzbeat-{version}-bin.tar.gz` ### Collector build -1. Requires `maven3+`, `java21` environments +1. Requires `maven3+`, `java25` environments 2. Execute under the project root directory: `mvn clean install` 3. Cd to the `hertzbeat-collector` directory: `cd hertzbeat-collector` -4. Execute under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +4. Build the JVM collector package under `hertzbeat-collector` directory: `mvn clean package -Pcluster` +5. Build the native collector package under `hertzbeat-collector` directory: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -The HertzBeat collector package will at `dist/hertzbeat-collector-{version}.tar.gz` +> Native collector packaging requires GraalVM for JDK 25 with the `native-image` tool available in `PATH`. + +The HertzBeat collector packages will be generated at `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` and a platform-specific native package such as `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` diff --git a/home/docs/community/how-to-release.md b/home/docs/community/how-to-release.md index 6deb97d01b2..9c92b940958 100644 --- a/home/docs/community/how-to-release.md +++ b/home/docs/community/how-to-release.md @@ -10,7 +10,7 @@ This tutorial describes in detail how to release Apache HertzBeat™, take the r This release process is operated in the UbuntuOS(Windows,Mac), and the following tools are required: -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,25 @@ mvn clean install mvn clean package -Pcluster ``` +> Build the native collector binary for the current host with GraalVM JDK 25 and `native-image` + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> The repository workflow `.github/workflows/collector-native-build.yml` is a release helper, not a regular PR or push CI workflow. +> +> It is intentionally manual-only because multi-platform native builds are relatively slow and consume scarce Linux ARM, macOS, and Windows runners. During release preparation, open the Actions page, select `Collector Native Release`, run it from the release branch or tag, and then download the uploaded artifacts for signing and publishing. + The release package are here: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 Package the source code diff --git a/home/docs/download.md b/home/docs/download.md index fde53959f4b..e167b88e3c1 100644 --- a/home/docs/download.md +++ b/home/docs/download.md @@ -22,9 +22,15 @@ Download the latest Apache HertzBeat™ release (v1.8.0) as server binary, colle |-------------|------|---------|----------| | **Server Binary** | ~200MB | Main monitoring server | Linux, macOS, Windows | | **Collector Binary** | ~50MB | Distributed collectors | Linux, macOS, Windows | -| **Source Code** | ~30MB | Build from source | Any with Java 21+ | +| **Source Code** | ~30MB | Build from source | Any with Java 25+ | | **Docker Compose** | ~5MB | Full stack deployment | Docker environments | +:::tip Native Collector Recommendation +If you do not need external JDBC drivers from `ext-lib`, you can choose the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase are included in this native-friendly path when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. + +Trade-offs: native packages are platform-specific and do not support runtime `ext-lib` JDBC loading. See [Native Collector Guide](start/native-collector). +::: + :::tip Security Verification Verify downloads using GPG signatures and SHA512 checksums. See [Apache Verification Guide](https://www.apache.org/dyn/closer.cgi#verify) and [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS). ::: @@ -57,6 +63,9 @@ For older releases, please check the [archive](https://archive.apache.org/dist/i **Server Binary** - For most users. Includes the main HertzBeat monitoring server with web UI. **Collector Binary** - For distributed deployments. Deploy collectors in remote networks to report to the main server. +Native collector downloads are platform-specific, for example `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`. + +If you are deciding between JVM and native collector packages, start with [Native Collector Guide](start/native-collector). **Source Code** - For developers who want to build, modify, or contribute to HertzBeat. @@ -73,17 +82,19 @@ Import Apache HertzBeat KEYS first: `wget https://downloads.apache.org/hertzbeat ### What are the system requirements? **Server Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 4GB RAM minimum (8GB recommended) - 2 CPU cores minimum - 20GB disk space **Collector Binary Requirements:** -- Java 21 or higher +- Java 25 or higher - 2GB RAM minimum - 1 CPU core minimum - 5GB disk space +Native collector packages are published per target platform, while the JVM collector package remains cross-platform. + ### Can I use Docker instead of binary packages? Yes. Docker is the recommended installation method: diff --git a/home/docs/help/db2.md b/home/docs/help/db2.md index 5652094fa92..9258d261634 100644 --- a/home/docs/help/db2.md +++ b/home/docs/help/db2.md @@ -16,6 +16,13 @@ keywords: [ open source monitoring tool, open source database monitoring tool, m - Copy the JAR package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +DB2 monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for DB2 monitoring +- Do not use the native collector package for DB2 monitoring +::: + ### Configuration Parameters The following are the required configuration parameters for DB2 monitoring: diff --git a/home/docs/help/mariadb.md b/home/docs/help/mariadb.md index 6bd75d08c48..1d8a155815c 100644 --- a/home/docs/help/mariadb.md +++ b/home/docs/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MariaDB database. Support MariaDB5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +MariaDB follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +MariaDB monitoring supports both JVM and native deployment now. + +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package +::: ### Configuration parameter diff --git a/home/docs/help/mysql.md b/home/docs/help/mysql.md index f21a1a1da87..51bcce08e17 100644 --- a/home/docs/help/mysql.md +++ b/home/docs/help/mysql.md @@ -7,12 +7,22 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of MySQL database. Support MYSQL5+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.4.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.4.0) -- It is recommended that you use the latest available mysql-connector-java version as there are regular security fixes to JDBC drivers. -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +HertzBeat now supports two MySQL query paths: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. +- The automatic decision only checks `ext-lib`. If you want to force one path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. + +:::important Collector package selection +MySQL monitoring supports both JVM and native deployment now. + +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package +::: ### Configuration parameter diff --git a/home/docs/help/oceanbase.md b/home/docs/help/oceanbase.md index 510934771f5..3bfcd3cf292 100644 --- a/home/docs/help/oceanbase.md +++ b/home/docs/help/oceanbase.md @@ -7,11 +7,21 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo > Collect and monitor the general performance Metrics of OceanBase database. Support OceanBase 4.0+. -### Attention, Need Add MYSQL jdbc driver jar +### Driver selection -- Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- Copy the jar package to the `hertzbeat/ext-lib` directory. -- Restart the HertzBeat service. +OceanBase now follows the same automatic routing as the MySQL-compatible query path: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +OceanBase monitoring now supports both JVM and native deployment. + +- Built-in server collector or JVM collector package: automatically prefers JDBC when `mysql-connector-j` exists in `ext-lib` +- Native collector package: supported when you do not rely on `ext-lib` and want the built-in MySQL-compatible query engine +- If you explicitly need runtime `ext-lib` JDBC loading, choose the JVM collector package +::: ### Configuration parameter diff --git a/home/docs/help/oracle.md b/home/docs/help/oracle.md index 83d7854030e..562e68cfcfd 100644 --- a/home/docs/help/oracle.md +++ b/home/docs/help/oracle.md @@ -13,6 +13,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo - Copy the jar package to the `hertzbeat/ext-lib` directory. - Restart the HertzBeat service. +:::important Collector package selection +Oracle monitoring requires external JDBC driver loading from `ext-lib`. + +- Use HertzBeat server built-in collector or the JVM collector package for Oracle monitoring +- Do not use the native collector package for Oracle monitoring +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/help/risc-v.md b/home/docs/help/risc-v.md index 41d605afe84..a7d2ee5d467 100644 --- a/home/docs/help/risc-v.md +++ b/home/docs/help/risc-v.md @@ -48,7 +48,7 @@ Reference: [Ubuntu Official Documentation](https://canonical-ubuntu-boards.readt ## Install and Configure RISC-V-Compatible JDK -> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 21. +> Configure a JDK that supports RISC-V architecture. Here we use Temurin JDK 25. **1. Download Temurin JDK** Download link: [https://adoptium.net/temurin/releases](https://adoptium.net/temurin/releases) @@ -66,11 +66,11 @@ sudo apt install -y tar wget sudo mkdir -p /usr/lib/jvm # Extract to system directory -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # Edit environment variables sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # Apply configuration @@ -91,6 +91,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > Notes: > -> 1. Replace `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` with your actual JDK filename. +> 1. Replace `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` with your actual JDK filename. > 2. Ensure the emulator has internet access to download HertzBeat. > 3. If issues arise, verify disk permissions and Java environment paths. diff --git a/home/docs/help/tidb.md b/home/docs/help/tidb.md index 73d8195aab4..78b741ea2c1 100644 --- a/home/docs/help/tidb.md +++ b/home/docs/help/tidb.md @@ -15,6 +15,22 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo **Protocol Use: HTTP and JDBC** +### Driver selection + +TiDB monitoring keeps the HTTP part unchanged, and the SQL query part now follows the same automatic routing as MySQL: + +- If `mysql-connector-j` is present in `ext-lib`, the JVM collector or built-in server collector automatically prefers JDBC for the SQL query metric set. +- If `mysql-connector-j` is absent, HertzBeat automatically uses the built-in MySQL-compatible query engine for the SQL query metric set. No extra JAR is required. +- Restart HertzBeat or the standalone JVM collector after adding or removing a JAR in `ext-lib`. + +:::important Collector package selection +The TiDB template mixes HTTP metrics and MySQL-compatible SQL queries. + +- HTTP metric sets are unaffected by JDBC driver selection +- The built-in SQL query engine can collect the default TiDB `basic` metric set without `mysql-connector-j` +- If you explicitly place `mysql-connector-j` in `ext-lib`, the JVM collector or built-in server collector will still prefer JDBC for the SQL query path +::: + ### Configuration parameter | Parameter name | Parameter help description | diff --git a/home/docs/start/docker-deploy.md b/home/docs/start/docker-deploy.md index 69c2310c023..db5bdd7a0ba 100644 --- a/home/docs/start/docker-deploy.md +++ b/home/docs/start/docker-deploy.md @@ -19,6 +19,7 @@ It is necessary to have Docker environment in your environment. If not installed ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ It is necessary to have Docker environment in your environment. If not installed - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (optional) Mount the configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (optional) Mount the account configuration file to the container (please ensure that the file exists locally). [Download](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR package [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (optional) Mount external third-party JAR packages when you need runtime JDBC extension. `mysql-jdbc` is only needed if you explicitly want the JDBC path for MySQL-compatible monitoring; [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) and [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) are still required for Oracle monitoring. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path used by the built-in collector. Supported values: `auto`, `jdbc`, `r2dbc`. - `--name hertzbeat` : (optional) Naming container name hertzbeat - `--restart=always` : (optional) Configure the container to restart automatically. - `apache/hertzbeat` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat` instead. @@ -71,6 +73,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -81,6 +84,7 @@ By deploying multiple HertzBeat Collectors, high availability, load balancing, a - `-e MODE=public` : set the running mode(public or private), public cluster or private - `-e MANAGER_HOST=127.0.0.1` : Important, Set the main hertzbeat server ip host, must use the server host instead of 127.0.0.1. - `-e MANAGER_PORT=1158` : (optional) Set the main hertzbeat server port, default 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (optional) Override the MySQL-compatible monitoring query path. Supported values: `auto`, `jdbc`, `r2dbc`. - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (optional) Mount the log file to the local host to facilitate viewing. - `--name hertzbeat-collector` : Naming container name hertzbeat-collector - `apache/hertzbeat-collector` : Use the [official application mirror](https://hub.docker.com/r/apache/hertzbeat-collector) to start the container, if the network times out, use `quay.io/tancloud/hertzbeat-collector` instead. diff --git a/home/docs/start/native-collector.md b/home/docs/start/native-collector.md new file mode 100644 index 00000000000..36527ce068f --- /dev/null +++ b/home/docs/start/native-collector.md @@ -0,0 +1,82 @@ +--- +id: native-collector +title: Native Collector Guide +sidebar_label: Native Collector +description: When to choose the HertzBeat native collector package, its benefits, limitations, and deployment guidance. +--- + +## When should I choose the native collector? + +Choose the native collector package when your monitoring workload does not depend on loading external JDBC drivers from `ext-lib`. + +Typical native-friendly workloads include: + +- HTTP, HTTPS, website availability, and API checks +- Port, ping, SSL certificate, and other network probes +- MySQL, MariaDB, and OceanBase when you do not rely on runtime `ext-lib` JDBC loading +- TiDB when you do not rely on runtime `ext-lib` JDBC loading for its SQL query metric set +- Redis, Zookeeper, Kafka, and other non-JDBC monitoring types + +## Why use it? + +Compared with the JVM collector package, the native collector package is usually a better fit when you want: + +- Faster startup +- Lower baseline memory usage +- A simpler runtime without a bundled or preinstalled JDK + +## What are the trade-offs? + +The native collector package is not a drop-in replacement for every JVM collector scenario. + +- Native packages are platform-specific. You must choose the package that matches your OS and CPU architecture. +- The native collector does not support loading external JDBC driver JARs from `ext-lib` at runtime. +- If your deployment depends on JVM-style runtime classpath extension, keep using the JVM collector package. + +## When should I stay on the JVM collector? + +Use the JVM collector package if your monitoring depends on external JDBC drivers, especially: + +- Oracle, which requires `ojdbc8` and sometimes `orai18n` +- DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path + +## Package naming + +The JVM collector package remains cross-platform: + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +The native collector package is platform-specific: + +- Linux or macOS: `apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows: `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +Examples: + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## Configuration consistency + +The native collector package uses the same `config/application.yml` layout as the JVM collector package. + +That means: + +- Collector connection settings are edited in the same place +- Virtual-thread related configuration is edited in the same place +- Native-only boot adjustments are applied by code at runtime instead of maintaining a second `application.yml` + +## Recommended decision + +- Choose the native collector package when you want lower memory usage and faster startup for non-JDBC monitoring, for MySQL, MariaDB, and OceanBase without `ext-lib`, or for TiDB when its SQL query metric set can use the built-in MySQL-compatible query engine. +- Choose the JVM collector package when you need `ext-lib`, external JDBC drivers, or JVM-style runtime extensibility. +- For MySQL-compatible monitoring on the JVM collector, `auto` only checks `ext-lib`. If you need to force a path, set `hertzbeat.collector.mysql.query-engine=jdbc`, `r2dbc`, or `auto`. + +## How are the official multi-platform packages built? + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` builds a native collector package for the current host only. +- The official Linux, macOS, and Windows native release packages are produced by manually running the `Collector Native Release` GitHub Actions workflow during release preparation, not on every push or pull request. + +For package deployment steps, refer to [Install HertzBeat via Package](package-deploy). diff --git a/home/docs/start/package-deploy.md b/home/docs/start/package-deploy.md index 0747d7821d9..c9486dbd053 100644 --- a/home/docs/start/package-deploy.md +++ b/home/docs/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: Install via Package :::tip You can install and run Apache HertzBeat™ on Linux Windows Mac system, and CPU supports X86/ARM64. -Since version 1.6.0 uses `Java 21` and the installation package no longer provides a built-in JDK version, use the new Hertzbeat according to the following situations: +The current branch uses `Java 25`, and the standard installation package no longer provides a built-in JDK. Use HertzBeat according to the following situations: -- When the default environment variable on your server is `Java 21`, you do not need to take any action for this step. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`, and if there are no other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and search the engine for how to set a new environment variable pointing to the new `Java 21`. -- When the default environment variable on your server is not `Java 21`, such as `Java 8` or `Java 11`,and you don't want to change the environment variable because if there are other applications on your server that require a lower version of Java, download the appropriate version from [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) according to your system, and rename the extracted folder to `java`, then copy it to the Hertzbeat extraction directory. +- When the default environment variable on your server is `Java 25`, you do not need to take any action for this step. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and if there are no other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, and set a new environment variable pointing to `Java 25`. +- When the default environment variable on your server is not `Java 25`, such as `Java 8`, `Java 11`, or `Java 21`, and you do not want to change the environment variable because there are other applications on your server that require a lower version of Java, download `Java 25` from [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) according to your system, rename the extracted folder to `java`, and then copy it to the HertzBeat extraction directory. ::: @@ -64,11 +64,20 @@ HertzBeat Collector is a lightweight data collector used to collect and send dat Deploying multiple HertzBeat Collectors can achieve high availability, load balancing, and cloud-edge collaboration of data. ::: +:::tip Native Collector Recommendation +If your monitoring workload does not depend on external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can also use the native collector package directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. + +Before choosing it, review the trade-offs in [Native Collector Guide](native-collector). +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. Download installation package - Download installation package `apache-hertzbeat-collector-xxx-bin.tar.gz` corresponding to your system environment + Download the collector package that matches your deployment mode: + - JVM collector package: `apache-hertzbeat-collector-xxx-bin.tar.gz` + - Native collector package for Linux or macOS: `apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Native collector package for Windows: `apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` - [Download Page](/docs/download) 2. Configure the collector configuration file @@ -77,6 +86,10 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # or + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # or + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` Configure the collector configuration yml file `config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` @@ -102,11 +115,30 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 3. Start the service - Run command `$ ./bin/startup.sh` or `bin/startup.bat` + Run `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. 4. Begin to explore HertzBeat Collector - Access `http://ip:1157` and you will see the registered new collector in dashboard + Open the HertzBeat server dashboard at `http://:1157` and confirm the new collector is registered. + +:::important Native Collector Limitations +The native collector package is suitable for monitoring types that do not rely on external JVM classpath extension. + +See [Native Collector Guide](native-collector) for package selection, package naming, and platform-specific trade-offs. + +`ext-lib`-based JDBC driver loading is a JVM collector capability. The native collector package does not support loading external JDBC driver JARs from `ext-lib` at runtime. + +If your monitoring depends on external JDBC drivers, use the JVM collector package instead of the native collector package. This currently includes: + +- Oracle, which requires `ojdbc8` and often `orai18n` +- DB2, which requires `jcc` +- Any MySQL, MariaDB, or OceanBase deployment where you explicitly place `mysql-connector-j` in `ext-lib` and want the JDBC path + +Recommended deployment: + +- Use the native collector package for HTTP, website, port, ping, similar non-JDBC monitoring types, and for MySQL, MariaDB, or OceanBase without `ext-lib` +- Use the JVM collector package when you need `ext-lib` driver extension +::: **HAVE FUN** @@ -117,15 +149,15 @@ Deploying multiple HertzBeat Collectors can achieve high availability, load bala 1. you need to prepare the JAVA environment in advance Install JAVA runtime environment-refer to [official website](https://www.oracle.com/java/technologies/downloads/) - requirement：JDK21 ENV + requirement：JDK25 ENV download JAVA installation package: [mirror website](https://mirrors.huaweicloud.com/openjdk/) After installation use command line to check whether you install it successfully. ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/docs/start/quickstart.md b/home/docs/start/quickstart.md index ddb24cab1e5..d9d061cb626 100644 --- a/home/docs/start/quickstart.md +++ b/home/docs/start/quickstart.md @@ -13,7 +13,7 @@ Install Apache HertzBeat™ in under 5 minutes using Docker with a single comman ## Installation Methods -HertzBeat provides four installation options: +HertzBeat provides multiple installation options: 1. **Docker** (Recommended) - Fastest setup, production-ready 2. **Binary Package** - Traditional deployment with manual configuration @@ -54,12 +54,13 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache ##### 2：Install via package -1. Download the release package `hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. Download the release package `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. Configure the HertzBeat configuration yml file `hertzbeat/config/application.yml` (optional) 3. Run command `$ ./bin/startup.sh` or `bin/startup.bat` 4. Access `http://localhost:1157` to start, default account: `admin/hertzbeat` 5. Deploy collector clusters(Optional) - - Download the release package `hertzbeat-collector-xx.tar.gz` to new machine [Download Page](https://hertzbeat.apache.org/docs/download) + - If you do not need external JDBC drivers from `ext-lib`, prefer the native collector package for faster startup and lower memory usage. MySQL, MariaDB, and OceanBase can use the built-in query engine directly when `mysql-connector-j` is not provided. TiDB follows the same rule for its SQL query metric set. See [Native Collector Guide](native-collector). + - Download the release package `apache-hertzbeat-collector-xx-bin.tar.gz` (JVM collector) or the native collector package for your target platform, such as `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz` or `apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`, to the new machine [Download Page](https://hertzbeat.apache.org/docs/download) - Configure the collector configuration yml file `hertzbeat-collector/config/application.yml`: unique `identity` name, running `mode` (public or private), hertzbeat `manager-host`, hertzbeat `manager-port` ```yaml @@ -74,15 +75,17 @@ Detailed config refer to [Install HertzBeat via Docker](https://hertzbeat.apache manager-port: ${MANAGER_PORT:1158} ``` - - Run command `$ ./bin/startup.sh` or `bin/startup.bat` - - Access `http://localhost:1157` and you will see the registered new collector in dashboard + - Native collector trade-offs: platform-specific packages, no runtime `ext-lib` JDBC loading, and less suitable for JVM-style runtime classpath extension. See [Native Collector Guide](native-collector). + - If `mysql-connector-j` is present in `ext-lib`, the built-in server collector or JVM collector automatically prefers JDBC for MySQL, MariaDB, and OceanBase after restart. TiDB follows the same rule for its SQL query metric set, while its HTTP metrics stay unchanged. Oracle and DB2 still require the JVM collector package because they depend on external JDBC drivers. + - Run command `$ ./bin/startup.sh` or `bin/startup.bat` for the JVM collector package. Run `$ ./bin/startup.sh` for Linux or macOS native collector packages, and `bin\\startup.bat` for the Windows native collector package. + - Access the HertzBeat server dashboard at `http://localhost:1157` and confirm the new collector is registered. Detailed config refer to [Install HertzBeat via Package](package-deploy) ##### 3：Start via source code 1. Local source code debugging needs to start the back-end project `manager` and the front-end project `web-app`. -2. Backend：need `maven3+`, `java21`, `lombok`, start the `hertzbeat-startup` service. +2. Backend：need `maven3+`, `java25`, `lombok`, start the `hertzbeat-startup` service. 3. Web：need `nodejs npm angular-cli` environment, Run `ng serve --open` in `web-app` directory after backend startup. 4. Access `http://localhost:4200` to start, default account: `admin/hertzbeat` @@ -108,7 +111,7 @@ Detailed steps refer to [Artifact Hub](https://artifacthub.io/packages/helm/hert - 2 CPU cores - 4GB RAM - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ **Operating Systems:** Linux, macOS, Windows (via Docker or WSL) diff --git a/home/docs/start/virtual-thread.md b/home/docs/start/virtual-thread.md index f9d36190316..06ee9518a8e 100644 --- a/home/docs/start/virtual-thread.md +++ b/home/docs/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: Virtual Threads description: Configure HertzBeat virtual-thread executors, defaults, rollback switches, and tuning guidance. --- -HertzBeat runs on JDK 21 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. +HertzBeat runs on JDK 25 and uses virtual threads for the blocking execution paths that benefit from them. All `hertzbeat.vthreads` keys are optional. If you upgrade HertzBeat but do not merge the new YAML block into your existing `application.yml`, HertzBeat still starts with built-in defaults. ## 1. Where to Configure It diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md index fe530b8bc01..3748879cf55 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/contribution.md @@ -52,7 +52,7 @@ limitations under the License. #### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md index 76b20f0d2ac..3df906d2f8e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/development.md @@ -11,7 +11,7 @@ sidebar_label: 运行编译 ### 后端启动 -1. 需要 `maven3+`, `java21` 和 `lombok` 环境 +1. 需要 `maven3+`, `java25` 和 `lombok` 环境 2. (可选)修改配置文件配置信息-`hertzbeat-startup/src/main/resources/application.yml` @@ -37,7 +37,7 @@ sidebar_label: 运行编译 ## 生成二进制包 -> 需要 `maven3+`, `java21`, `node` 和 `pnpm` 环境. +> 需要 `maven3+`, `java25`, `node` 和 `pnpm` 环境. ### 前端打包 @@ -53,20 +53,23 @@ sidebar_label: 运行编译 ### 后端打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean package -Prelease` -HertzBeat 包将生成为 `dist/hertzbeat-{version}.tar.gz` +HertzBeat 包将生成为 `dist/apache-hertzbeat-{version}-bin.tar.gz` ### 采样器打包 -1. 需要 `maven3+`, `java21` 环境 +1. 需要 `maven3+`, `java25` 环境 2. 在项目根目录运行: `mvn clean install` 3. 切换到 `hertzbeat-collector` 目录: `cd hertzbeat-collector` -4. 在 `hertzbeat-collector` 目录下执行: `mvn clean package -Pcluster` +4. 在 `hertzbeat-collector` 目录下打 JVM 采集器安装包: `mvn clean package -Pcluster` +5. 在 `hertzbeat-collector` 目录下打 Native 采集器安装包: `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` -HertzBeat 采样器包将生成为 `dist/hertzbeat-collector-{version}.tar.gz` +> Native 采集器打包需要带有 `native-image` 命令的 GraalVM JDK 25 环境。 + +HertzBeat 采集器安装包将生成为 `dist/apache-hertzbeat-collector-{version}-bin.tar.gz`，以及类似 `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 这样的 Native 平台安装包 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md index b0d5ae4239e..7415ca15f9e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/community/how-to-release.md @@ -10,7 +10,7 @@ sidebar_position: 4 此发布过程在 UbuntuOS(可在 Windows Mac) 中进行操作，并需要以下环境： -- JDK 21 +- JDK 25 - Node18 pnpm - Apache Maven 3.x - GnuPG 2.x @@ -211,10 +211,25 @@ mvn clean install mvn clean package -Pcluster ``` +> 使用带 `native-image` 的 GraalVM JDK 25，为当前宿主机构建 Native 采集器安装包 + +```shell +mvn clean package -pl hertzbeat-collector-collector -am -Pnative +``` + +> 仓库中的 `.github/workflows/collector-native-build.yml` 是 release 辅助工作流，不参与日常 PR 或 push 的常规 CI。 +> +> 之所以只保留手动触发，是因为跨平台 Native 构建耗时更长，也会占用相对稀缺的 Linux ARM、macOS 和 Windows Runner。准备发版时，请在 GitHub Actions 页面选择 `Collector Native Release`，基于 release 分支或 tag 手动触发，然后下载上传的产物用于签名和发布。 + 生成的二进制包在: - `dist/apache-hertzbeat-{version}-bin.tar.gz` - `dist/apache-hertzbeat-collector-{version}-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-linux-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-amd64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-macos-arm64-bin.tar.gz` +- `dist/apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` - `dist/apache-hertzbeat-{version}-docker-compose.tar.gz` #### 3.4 打包项目源代码 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md index 2325660f32d..0c226c424de 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/download.md @@ -22,9 +22,15 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 |--------|------|------|------| | **服务器二进制** | ~200MB | 主监控服务器 | Linux、macOS、Windows | | **采集器二进制** | ~50MB | 分布式采集器 | Linux、macOS、Windows | -| **源码** | ~30MB | 从源码构建 | 任何支持 Java 21+ 的平台 | +| **源码** | ~30MB | 从源码构建 | 任何支持 Java 25+ 的平台 | | **Docker Compose** | ~5MB | 全栈部署 | Docker 环境 | +:::tip Native 采集器推荐 +如果你不需要 `ext-lib` 外部 JDBC 驱动，可以优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时也属于这条 Native 友好路径；TiDB 的 SQL 查询指标也遵循同样规则。 + +它的代价是安装包按平台区分，且不支持运行时 `ext-lib` JDBC 加载。详见 [Native 采集器指南](start/native-collector)。 +::: + :::tip 安全验证 使用 GPG 签名和 SHA512 校验和验证下载。参见 [Apache 验证指南](https://www.apache.org/dyn/closer.cgi#verify) 和 [HertzBeat KEYS](https://downloads.apache.org/hertzbeat/KEYS)。 ::: @@ -57,6 +63,9 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 **服务器二进制** - 大多数用户使用。包含主 HertzBeat 监控服务器和 Web UI。 **采集器二进制** - 分布式部署使用。在远程网络部署采集器向主服务器上报。 +Native 采集器下载包按目标平台区分，例如 `apache-hertzbeat-collector-native-{version}-linux-amd64-bin.tar.gz` 或 `apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip`。 + +如果你正在 JVM 采集器和 Native 采集器之间做选择，建议先阅读 [Native 采集器指南](start/native-collector)。 **源码** - 开发者想要构建、修改或贡献 HertzBeat 时使用。 @@ -73,17 +82,19 @@ description: Apache HertzBeat 监控系统下载 - 服务器、采集器、源 ### 系统要求是什么？ **服务器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 4GB RAM 最低（推荐 8GB） - 2 CPU 核心 最低 - 20GB 磁盘空间 **采集器二进制要求：** -- Java 21 或更高版本 +- Java 25 或更高版本 - 2GB RAM 最低 - 1 CPU 核心 最低 - 5GB 磁盘空间 +Native 采集器安装包会按目标平台分别发布，JVM 采集器安装包仍然保持跨平台。 + ### 可以用 Docker 代替二进制包吗？ 可以。Docker 是推荐的安装方法： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md index 0c2f9bb7e32..0dcba300aee 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/db2.md @@ -15,6 +15,13 @@ keywords: [ 开源监控系统, 开源数据库监控, DB2数据库监控 ] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下 - 重启 HertzBeat 服务 +:::important 采集器包选择 +DB2 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- DB2 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 DB2 监控 +::: + ### 配置参数 以下是 DB2 监控所需的配置参数： diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md index dd8b3f7d868..6b2223a6396 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mariadb.md @@ -7,11 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, MariaDB数据库监控] > 对MariaDB数据库的通用性能指标进行采集监控。支持MariaDB5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +MariaDB 现在和 MySQL 一样支持自动分流： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +MariaDB 监控现在既支持 JVM 部署，也支持 Native 部署。 + +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 +::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md index 82b3f36b564..17bf69eff4d 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/mysql.md @@ -7,11 +7,22 @@ keywords: [开源监控系统, 开源数据库监控, Mysql数据库监控] > 对MYSQL数据库的通用性能指标进行采集监控。支持MYSQL5+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择说明 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +HertzBeat 现在支持两条 MySQL 查询链路： + +- 如果在 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置 MySQL 查询引擎，不需要额外复制 JAR。 +- 每次增删 `ext-lib` 里的驱动后，都需要重启 HertzBeat 或独立 JVM 采集器。 +- 自动分流只检查 `ext-lib`。如果你想显式指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 + +:::important 采集器包选择 +MySQL 监控现在既支持 JVM 部署，也支持 Native 部署。 + +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置 MySQL 查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，请选择 JVM 采集器安装包 +::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md index edeeaeaf611..415b6445a7a 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oceanbase.md @@ -7,11 +7,21 @@ keywords: [开源监控系统, 开源数据库监控, OceanBase 数据库监控] > 对 OceanBase 数据库的通用性能指标进行采集监控。支持 OceanBase 4.0+。 -### 注意，必须添加 MYSQL jdbc 驱动 jar +### 驱动选择 -- 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.1.0.jar. [https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0](https://mvnrepository.com/artifact/com.mysql/mysql-connector-j/8.1.0) -- 将此 jar 包拷贝放入 HertzBeat 的安装目录下的 `ext-lib` 目录下. -- 重启 HertzBeat 服务。 +OceanBase 现在和 MySQL 兼容查询路径一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +OceanBase 监控现在同样支持 JVM 和 Native 两种部署方式。 + +- 主程序内置采集器或 JVM 采集器安装包：当 `ext-lib` 中存在 `mysql-connector-j` 时会自动优先走 JDBC +- Native 采集器安装包：在不依赖 `ext-lib` 时可直接使用内置的 MySQL 兼容查询引擎 +- 如果你明确需要运行时 `ext-lib` JDBC 加载能力，仍然请选择 JVM 采集器安装包 +::: ### 配置参数 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md index f15d3dc62b5..153ef72fcb4 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/oracle.md @@ -13,6 +13,13 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控] - 将 jar 包复制到 `hertzbeat/ext-lib` 目录下。 - 重启 HertzBeat 服务。 +:::important 采集器包选择 +Oracle 监控依赖 `ext-lib` 目录下的外置 JDBC 驱动加载能力。 + +- Oracle 监控请使用 HertzBeat 主程序内置采集器，或 JVM 采集器安装包 +- 不要使用 Native 采集器安装包执行 Oracle 监控 +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md index f931b561024..50b7da7a250 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/risc-v.md @@ -45,7 +45,7 @@ qemu-system-riscv64 \ ## 安装并配置支持 RISC-V 架构的 JDK -> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 21。 +> 在启动的镜像中配置支持 RISC-V 架构的 JDK，这里选用 Temurin JDK 25。 **1. 下载 Temurin JDK** @@ -63,11 +63,11 @@ sudo apt install -y tar wget # 创建安装目录 sudo mkdir -p /usr/lib/jvm # 解压到系统目录 -sudo tar -xzf OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz -C /usr/lib/jvm +sudo tar -xzf OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz -C /usr/lib/jvm # 编辑环境变量，添加以下内容 sudo nano /etc/profile.d/java.sh -export JAVA_HOME=/usr/lib/jvm/jdk-21.0.2+13 +export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d -name 'jdk-25*' | head -n 1) export PATH=$JAVA_HOME/bin:$PATH # 使配置生效 @@ -88,6 +88,6 @@ cd apache-hertzbeat-1.7.2-incubating-bin/bin/ > 注意事项： > -> 1. 请将 `OpenJDK21U-jdk_riscv64_linux_hotspot_21.0.2_13.tar.gz` 替换为您实际下载的 JDK 文件名。 +> 1. 请将 `OpenJDK25U-jdk_riscv64_linux_hotspot_.tar.gz` 替换为您实际下载的 JDK 文件名。 > 2. 确保模拟器具备网络访问能力，以下载 HertzBeat。 > 3. 若遇到问题，请检查磁盘权限和 Java 环境路径配置。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md index 69bdd6fd40b..76280d8ebc9 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/tidb.md @@ -7,6 +7,22 @@ keywords: [开源监控系统, 开源数据库监控, TiDB数据库监控] > 使用 HTTP 和 JDBC 协议对 TiDB 的通用性能指标进行采集监控。 +### 驱动选择 + +TiDB 监控里的 HTTP 部分保持不变，SQL 查询部分现在和 MySQL 一样会自动分流： + +- 如果 `ext-lib` 中放入了 `mysql-connector-j`，JVM 采集器或主程序内置采集器会对 SQL 查询指标自动优先走 JDBC。 +- 如果没有放入 `mysql-connector-j`，HertzBeat 会对 SQL 查询指标自动切换到内置的 MySQL 兼容查询引擎，不需要额外复制 JAR。 +- 在 `ext-lib` 中新增或删除 JAR 后，请重启 HertzBeat 或独立 JVM 采集器。 + +:::important 采集器包选择 +TiDB 默认模板同时包含 HTTP 指标和 MySQL 兼容 SQL 查询。 + +- HTTP 指标集合不受 JDBC 驱动选择影响 +- 内置 SQL 查询引擎已经可以在不放 `mysql-connector-j` 的情况下采集默认 TiDB `basic` 指标集合 +- 如果你明确把 `mysql-connector-j` 放进 `ext-lib`，JVM 采集器或主程序内置采集器仍会对 SQL 查询路径优先走 JDBC +::: + ### 配置参数 | 参数名称 | 参数帮助描述 | diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md index 06fa83e496e..2265696f10c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/docker-deploy.md @@ -19,6 +19,7 @@ sidebar_label: Docker方式安装 ```shell $ docker run -d -p 1157:1157 -p 1158:1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ -v $(pwd)/data:/opt/hertzbeat/data \ -v $(pwd)/logs:/opt/hertzbeat/logs \ -v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml \ @@ -35,7 +36,8 @@ sidebar_label: Docker方式安装 - `-v $(pwd)/logs:/opt/hertzbeat/logs` : (可选) 挂载日志文件到本地主机方便查看 - `-v $(pwd)/application.yml:/opt/hertzbeat/config/application.yml` : (可选) 挂载配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/application.yml) - `-v $(pwd)/sureness.yml:/opt/hertzbeat/config/sureness.yml` : (可选) 挂载账户配置文件到容器中(请确保本地已有此文件)。[下载源](https://github.com/apache/hertzbeat/raw/master/script/sureness.yml) - - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 挂载外部的第三方 JAR 包 [mysql-jdbc](https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip) [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar) + - `-v $(pwd)/ext-lib:/opt/hertzbeat/ext-lib` : (可选) 在你需要运行时 JDBC 扩展时挂载外部第三方 JAR 包。`mysql-jdbc` 只在你明确希望 MySQL 兼容监控继续走 JDBC 时才需要；Oracle 监控仍然需要 [oracle-jdbc](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/23.4.0.24.05/ojdbc8-23.4.0.24.05.jar) 和 [oracle-i18n](https://repo.mavenlibs.com/maven/com/oracle/database/nls/orai18n/21.5.0.0/orai18n-21.5.0.0.jar)。 + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖主程序内置采集器的 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `--name hertzbeat` : (可选) 命名容器名称为 hertzbeat - `--restart=always` : (可选) 配置容器自动重启。 - `apache/hertzbeat` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat`代替。 @@ -69,6 +71,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 -e MODE=public \ -e MANAGER_HOST=127.0.0.1 \ -e MANAGER_PORT=1158 \ + -e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto \ --name hertzbeat-collector apache/hertzbeat-collector ``` @@ -79,6 +82,7 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 - `-e MODE=public` : 配置运行模式(public or private), 公共集群模式或私有云边模式。 - `-e MANAGER_HOST=127.0.0.1` : 重要, 配置连接的 HertzBeat Server 地址，127.0.0.1 需替换为 HertzBeat Server 对外 IP 地址。 - `-e MANAGER_PORT=1158` : (可选) 配置连接的 HertzBeat Server 端口，默认 1158. + - `-e HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE=auto` : (可选) 覆盖 MySQL 兼容监控查询链路。可选值：`auto`、`jdbc`、`r2dbc`。 - `-v $(pwd)/logs:/opt/hertzbeat-collector/logs` : (可选)挂载日志文件到本地主机方便查看 - `--name hertzbeat-collector` : 命名容器名称为 hertzbeat-collector - `apache/hertzbeat-collector` : 使用[官方应用镜像](https://hub.docker.com/r/apache/hertzbeat-collector)来启动容器, 若网络超时可用`quay.io/tancloud/hertzbeat-collector`代替。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md new file mode 100644 index 00000000000..9399673a846 --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/native-collector.md @@ -0,0 +1,82 @@ +--- +id: native-collector +title: Native 采集器指南 +sidebar_label: Native 采集器 +description: 说明 HertzBeat Native 采集器安装包适合什么场景、优缺点、限制和部署建议。 +--- + +## 什么场景适合使用 Native 采集器？ + +当你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动时，优先考虑 Native 采集器安装包。 + +比较适合 Native 采集器的场景包括： + +- HTTP、HTTPS、网站可用性、API 检查 +- 端口可用性、Ping、SSL 证书等网络探测 +- 不依赖运行时 `ext-lib` JDBC 加载的 MySQL、MariaDB、OceanBase +- SQL 查询指标不依赖运行时 `ext-lib` JDBC 加载的 TiDB +- Redis、Zookeeper、Kafka 等非 JDBC 监控类型 + +## 为什么选择它？ + +相较 JVM 采集器安装包，Native 采集器安装包通常更适合以下诉求： + +- 启动更快 +- 常驻内存更低 +- 运行时更轻，不需要额外准备 bundled 或预装 JDK + +## 它的缺点和限制是什么？ + +Native 采集器并不是所有 JVM 采集器场景的无损替代。 + +- Native 安装包是平台相关的，必须选择与你操作系统和 CPU 架构匹配的包。 +- Native 采集器不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 +- 如果你的部署依赖 JVM 风格的运行时 classpath 扩展能力，仍然应该使用 JVM 采集器安装包。 + +## 哪些场景应该继续使用 JVM 采集器？ + +如果你的监控依赖外部 JDBC 驱动，请继续使用 JVM 采集器安装包，尤其包括： + +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 + +## 安装包命名规则 + +JVM 采集器安装包仍然保持跨平台： + +- `apache-hertzbeat-collector-{version}-bin.tar.gz` + +Native 采集器安装包按平台区分： + +- Linux 或 macOS：`apache-hertzbeat-collector-native-{version}-{platform}-bin.tar.gz` +- Windows：`apache-hertzbeat-collector-native-{version}-windows-amd64-bin.zip` + +例如： + +- `apache-hertzbeat-collector-native-1.8.0-linux-amd64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-macos-arm64-bin.tar.gz` +- `apache-hertzbeat-collector-native-1.8.0-windows-amd64-bin.zip` + +## 配置文件是否和 JVM 采集器一致？ + +Native 采集器安装包和 JVM 采集器安装包使用同一套 `config/application.yml` 结构。 + +这意味着： + +- 采集器连接参数仍然在同一个位置修改 +- 虚拟线程相关配置仍然在同一个位置修改 +- Native 专用的启动调整通过代码在运行时生效，而不是长期维护第二份 `application.yml` + +## 推荐选择 + +- 想要更低内存、更快启动，并且监控类型不依赖 JDBC 驱动时，优先选择 Native 采集器安装包；MySQL、MariaDB、OceanBase 在不使用 `ext-lib` 时适合直接选择 Native 采集器安装包，TiDB 的 SQL 查询指标在不使用 `ext-lib` 时也可以走内置 MySQL 兼容查询引擎。 +- 需要 `ext-lib`、外置 JDBC 驱动，或者依赖 JVM 风格运行时扩展能力时，使用 JVM 采集器安装包。 +- 对 MySQL 兼容监控来说，`auto` 只检查 `ext-lib`。如果你想手动指定链路，可以配置 `hertzbeat.collector.mysql.query-engine=jdbc`、`r2dbc` 或 `auto`。 + +## 官方多平台安装包是怎么构建的？ + +- `mvn clean package -pl hertzbeat-collector-collector -am -Pnative` 只会为当前宿主机构建一个 Native 采集器安装包。 +- 官方发布使用的 Linux、macOS、Windows Native 安装包，会在发布准备阶段手动触发 `Collector Native Release` GitHub Actions 工作流来生成，而不是在每次 push 或 pull request 时自动构建。 + +具体安装步骤可参考 [通过安装包安装 HertzBeat](package-deploy)。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md index b34768ac984..c3d8120c564 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/package-deploy.md @@ -6,11 +6,11 @@ sidebar_label: 安装包方式安装 :::tip Apache HertzBeat™ 支持在Linux Windows Mac系统安装运行，CPU支持X86/ARM64。 -由于1.6.0及以后版本使用 `Java 21` ，且安装包不再提供内置jdk的版本，参考以下情况使用新版Hertzbeat。 +当前分支默认使用 `Java 25`，且标准安装包不再提供内置 JDK。可参考以下情况使用 HertzBeat： -- 当你的服务器中默认环境变量为 `Java 21` 时，这一步你无需任何操作。 -- 当你的服务器中默认环境变量不为 `Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**没有**其他应用需要低版本 `Java` ，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并在搜索引擎搜索如何设置新的环境变量指向新的`Java 21`。 -- 当你的服务器中默认环境变量不为`Java 21`时，如 `Java 8` 、 `Java 11` ，若你服务器中**有**其他应用需要低版本 `Java` ，你不想更改环境变量，根据你的系统，到 [https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html](https://www.oracle.com/java/technologies/javase/jdk21-archive-downloads.html) 选择相应的发行版下载，并将解压后的文件夹重命名为`java`，复制到Hertzbeat的解压目录下。 +- 当你的服务器中默认环境变量为 `Java 25` 时，这一步无需任何操作。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**没有**其他应用需要低版本 `Java`，根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将环境变量指向新的 `Java 25`。 +- 当你的服务器中默认环境变量不为 `Java 25` 时，如 `Java 8`、`Java 11`、`Java 21`，若你服务器中**有**其他应用需要低版本 `Java`，不希望修改全局环境变量，可根据你的系统到 [https://www.oracle.com/java/technologies/downloads/](https://www.oracle.com/java/technologies/downloads/) 下载 `Java 25`，并将解压后的文件夹重命名为 `java`，复制到 HertzBeat 的解压目录下。 ::: @@ -63,11 +63,21 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 通过部署多个 HertzBeat Collector 可以实现数据的高可用，负载均衡和云边协同。 ::: +:::tip Native 采集器推荐 +如果你的监控任务不依赖从 `ext-lib` 动态加载外部 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、常驻内存更低。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时，也可以直接使用 Native 采集器安装包；TiDB 的 SQL 查询指标也遵循同样规则。 + +在选择前，建议先阅读 [Native 采集器指南](native-collector) 了解它的限制和取舍。 +::: + ![HertzBeat](/img/docs/cluster-arch.png) 1. 下载安装包 - 从 [下载页面](/docs/download) 下载您系统环境对应的安装包版本 `apache-hertzbeat-collector-xxx-bin.tar.gz` + 按部署形态选择对应的采集器安装包： + - JVM 采集器安装包：`apache-hertzbeat-collector-xxx-bin.tar.gz` + - Linux 或 macOS 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-{platform}-bin.tar.gz` + - Windows 的 Native 采集器安装包：`apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip` + - 从 [下载页面](/docs/download) 下载 2. 设置配置文件 @@ -75,6 +85,10 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 ```shell tar zxvf apache-hertzbeat-collector-xxx-bin.tar.gz + # 或 + tar zxvf apache-hertzbeat-collector-native-xxx-linux-amd64-bin.tar.gz + # 或 + unzip apache-hertzbeat-collector-native-xxx-windows-amd64-bin.zip ``` 配置采集器的配置文件 `config/application.yml` 里面的 HertzBeat Server 连接 IP, 端口, 采集器名称(需保证唯一性)等参数。 @@ -100,14 +114,29 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 3. 启动 - 执行位于安装目录 hertzbeat-collector/bin/ 下的启动脚本 startup.sh, windows 环境下为 startup.bat - - ```shell - ./startup.sh - ``` + JVM 采集器安装包执行位于安装目录 `hertzbeat-collector/bin/` 下的启动脚本 `startup.sh`，Windows 环境下为 `startup.bat`；Linux 或 macOS 的 Native 采集器安装包执行 `./startup.sh`，Windows 的 Native 采集器安装包执行 `bin\\startup.bat` 4. 开始探索 HertzBeat Collector - 浏览器访问 [http://ip:1157/](http://ip:1157/) 即可开始探索使用，默认账户密码 admin/hertzbeat。 + 浏览器访问主 HertzBeat 服务 [http://manager-host:1157/](http://manager-host:1157/) 的概览页面，即可确认新采集器已注册。 + +:::important Native 采集器限制说明 +Native 采集器适合不依赖外部 JVM classpath 扩展的监控类型。 + +关于包选择、安装包命名和平台相关限制，详见 [Native 采集器指南](native-collector)。 + +基于 `ext-lib` 的 JDBC 驱动加载能力是 JVM 采集器的能力。Native 采集器当前不支持在运行时从 `ext-lib` 目录动态加载外部 JDBC 驱动 JAR。 + +因此，凡是依赖外置 JDBC 驱动的监控类型，请使用 JVM 采集器，不要使用 Native 采集器。当前至少包括： + +- Oracle，需要 `ojdbc8`，部分场景还需要 `orai18n` +- DB2，需要 `jcc` +- 任何明确把 `mysql-connector-j` 放进 `ext-lib` 并希望继续走 JDBC 的 MySQL、MariaDB、OceanBase 场景 + +建议部署方式： + +- `API`、`网站`、`端口可用性`、`Ping` 等非 JDBC 类型，以及不依赖 `ext-lib` 的 MySQL、MariaDB、OceanBase，优先使用 Native 采集器 +- 需要 `ext-lib` 扩展驱动时使用 JVM 采集器 +::: **HAVE FUN** @@ -120,15 +149,15 @@ HertzBeat Collector 是一个轻量级的数据采集器，用于采集并将数 1. 启动失败，需您提前准备JAVA运行环境 安装JAVA运行环境-可参考[官方网站](https://www.oracle.com/java/technologies/downloads/) - 要求：JAVA21环境 + 要求：JAVA25环境 下载JAVA安装包: [镜像站](https://mirrors.huaweicloud.com/openjdk/) 安装后命令行检查是否成功安装 ```shell $ java -version - openjdk version "21.0.9" 2025-10-21 LTS - OpenJDK Runtime Environment Corretto-21.0.9.10.1 (build 21.0.9+10-LTS) - OpenJDK 64-Bit Server VM Corretto-21.0.9.10.1 (build 21.0.9+10-LTS, mixed mode, sharing) + openjdk version "25.0.2" 2026-01-20 + OpenJDK Runtime Environment (build 25.0.2+8) + OpenJDK 64-Bit Server VM (build 25.0.2+8, mixed mode, sharing) ``` diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md index 9723ac7357a..d4d2b7c854c 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/quickstart.md @@ -13,7 +13,7 @@ description: Apache HertzBeat 监控系统快速安装指南 - Docker、安装 ## 安装方式 -HertzBeat 提供四种安装选项： +HertzBeat 提供多种安装选项： 1. **Docker**（推荐）- 最快设置，生产就绪 2. **二进制包** - 传统部署，手动配置 @@ -58,12 +58,13 @@ HertzBeat 提供四种安装选项： #### 方式二：通过安装包安装 -1. 下载您系统环境对应的安装包`hertzbeat-xx.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) +1. 下载您系统环境对应的安装包 `apache-hertzbeat-xx-bin.tar.gz` [Download Page](https://hertzbeat.apache.org/docs/download) 2. 配置 HertzBeat 的配置文件 `hertzbeat/config/application.yml`(可选) 3. 部署启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` 4. 浏览器访问 `http://localhost:1157` 即可开始，默认账号密码 `admin/hertzbeat` 5. 部署采集器集群(可选) - - 下载您系统环境对应采集器安装包`hertzbeat-collector-xx.tar.gz`到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) + - 如果你不需要 `ext-lib` 外置 JDBC 驱动，优先选择 Native 采集器安装包，通常启动更快、内存更省。MySQL、MariaDB、OceanBase 在没有提供 `mysql-connector-j` 时可以直接使用内置查询引擎；TiDB 的 SQL 查询指标也遵循同样规则。详见 [Native 采集器指南](native-collector)。 + - 下载您系统环境对应采集器安装包 `apache-hertzbeat-collector-xx-bin.tar.gz`（JVM 采集器）或匹配目标平台的 Native 采集器安装包，例如 `apache-hertzbeat-collector-native-xx-linux-amd64-bin.tar.gz`、`apache-hertzbeat-collector-native-xx-windows-amd64-bin.zip`，到规划的另一台部署主机上 [Download Page](https://hertzbeat.apache.org/docs/download) - 配置采集器的配置文件 `hertzbeat-collector/config/application.yml` 里面的连接主HertzBeat服务的对外IP，端口，当前采集器名称(需保证唯一性)等参数 `identity` `mode` (public or private) `manager-host` `manager-port` ```yaml @@ -78,15 +79,17 @@ HertzBeat 提供四种安装选项： manager-port: ${MANAGER_PORT:1158} ``` - - 启动 `$ ./bin/startup.sh` 或 `bin/startup.bat` - - 浏览器访问主HertzBeat服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 + - Native 采集器的代价是安装包按平台区分、不支持运行时 `ext-lib` JDBC 加载，也不适合依赖 JVM 风格运行时 classpath 扩展的场景。详见 [Native 采集器指南](native-collector)。 + - 如果在 `ext-lib` 中放入了 `mysql-connector-j`，主程序内置采集器或 JVM 采集器会在重启后自动优先走 JDBC；这一点现在适用于 MySQL、MariaDB、OceanBase，TiDB 的 SQL 查询指标也遵循同样规则，而它的 HTTP 指标不受影响。Oracle、DB2 仍然必须使用 JVM 采集器安装包，因为它们依赖外置 JDBC 驱动 + - JVM 采集器安装包使用 `$ ./bin/startup.sh` 或 `bin/startup.bat` 启动。Linux 或 macOS 的 Native 采集器安装包使用 `$ ./bin/startup.sh` 启动，Windows 的 Native 采集器安装包使用 `bin\\startup.bat` 启动 + - 浏览器访问主 HertzBeat 服务 `http://localhost:1157` 查看概览页面即可看到注册上来的新采集器 更多配置详细步骤参考 [通过安装包安装HertzBeat](package-deploy) #### 方式三：本地代码启动 1. 此为前后端分离项目，本地代码调试需要分别启动后端工程`hertzbeat-startup`和前端工程`web-app` -2. 后端：需要`maven3+`, `java21`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 +2. 后端：需要`maven3+`, `java25`和`lombok`环境，修改`YML`配置信息并启动`hertzbeat-startup`服务 3. 前端：需要`nodejs npm angular-cli`环境，待本地后端启动后，在`web-app`目录下启动 `ng serve --open` 4. 浏览器访问 `http://localhost:4200` 即可开始，默认账号密码 `admin/hertzbeat` @@ -112,7 +115,7 @@ HertzBeat 提供四种安装选项： - 2 CPU 核心 - 4GB RAM - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ **支持系统：** Linux、macOS、Windows（通过 Docker 或 WSL） diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md index ae69b63e0a9..020f5376b5f 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/virtual-thread.md @@ -5,7 +5,7 @@ sidebar_label: 虚拟线程 description: 说明 HertzBeat 虚拟线程执行器的默认值、回滚开关和调优方式。 --- -HertzBeat 基于 JDK 21 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 +HertzBeat 基于 JDK 25 运行，并把适合虚拟线程的阻塞型执行路径切到了虚拟线程模型。所有 `hertzbeat.vthreads` 配置项都是可选的。也就是说，即使升级 HertzBeat 后你没有把新的 YAML 配置块合并到原有 `application.yml`，系统也会使用内置默认值正常启动。 ## 1. 到哪里配置 diff --git a/home/sidebars.json b/home/sidebars.json index 24c235bc785..292e3ddff27 100755 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -15,6 +15,7 @@ "start/docker-deploy", "start/docker-compose-deploy", "start/package-deploy", + "start/native-collector", { "type": "link", "label": "Install via Helm", diff --git a/home/src/components/StructuredData.js b/home/src/components/StructuredData.js index 97472f05e6a..cbeeb493187 100644 --- a/home/src/components/StructuredData.js +++ b/home/src/components/StructuredData.js @@ -55,7 +55,7 @@ export default function StructuredData() { "Cloud-edge collaboration for isolated networks", "Status page builder for service communication" ], - "softwareRequirements": "Docker 20.10+ or Java 21+", + "softwareRequirements": "Docker 20.10+ or Java 25+", "memoryRequirements": "4GB minimum, 8GB recommended", "processorRequirements": "2 CPU cores minimum", "storageRequirements": "10GB minimum" diff --git a/home/src/pages/faq.js b/home/src/pages/faq.js index cc6d74af086..cc45b8c4d58 100644 --- a/home/src/pages/faq.js +++ b/home/src/pages/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "What are HertzBeat's system requirements?", - answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 21+. Supported on Linux, macOS, Windows." + answer: "Minimum: 2 CPU cores, 4GB RAM (8GB recommended), 10GB disk space, Docker 20.10+ or Java 25+. Supported on Linux, macOS, Windows." }, { question: "How do I upgrade HertzBeat?", diff --git a/home/src/pages/zh-cn/faq.js b/home/src/pages/zh-cn/faq.js index ae16a05b411..f422546dfca 100644 --- a/home/src/pages/zh-cn/faq.js +++ b/home/src/pages/zh-cn/faq.js @@ -46,7 +46,7 @@ const faqs = [ }, { question: "HertzBeat 的系统要求是什么？", - answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 21+。支持系统：Linux、macOS、Windows。" + answer: "最低：2 CPU 核心、4GB RAM（推荐 8GB）、10GB 磁盘空间、Docker 20.10+ 或 Java 25+。支持系统：Linux、macOS、Windows。" }, { question: "如何升级 HertzBeat？", diff --git a/home/static/llms-zh.txt b/home/static/llms-zh.txt index 0ae302d4691..8971f7a70f7 100644 --- a/home/static/llms-zh.txt +++ b/home/static/llms-zh.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU 核心 最低 - 4GB RAM 最低（推荐 8GB） - 10GB 磁盘空间 -- Docker 20.10+ 或 Java 21+ +- Docker 20.10+ 或 Java 25+ - 平台：Linux、macOS、Windows ## 支持的监控类型 diff --git a/home/static/llms.txt b/home/static/llms.txt index 18ab6d7bbf9..65ef12af761 100644 --- a/home/static/llms.txt +++ b/home/static/llms.txt @@ -57,7 +57,7 @@ cd apache-hertzbeat-1.8.0 - 2 CPU cores minimum - 4GB RAM minimum (8GB recommended) - 10GB disk space -- Docker 20.10+ or Java 21+ +- Docker 20.10+ or Java 25+ - Platforms: Linux, macOS, Windows ## Supported Monitoring Types diff --git a/material/licenses/NOTICE b/material/licenses/NOTICE index 66e5d1db5f7..3e8e5d05a62 100644 --- a/material/licenses/NOTICE +++ b/material/licenses/NOTICE @@ -1162,6 +1162,22 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== ======================================================================== Apache ECharts diff --git a/material/licenses/backend/LICENSE b/material/licenses/backend/LICENSE index be7c0505ed8..3440fc8c31d 100644 --- a/material/licenses/backend/LICENSE +++ b/material/licenses/backend/LICENSE @@ -265,6 +265,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-api/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-impl/0.11.2 Apache-2.0 https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt-jackson/0.11.2 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.12.3 Apache-2.0 @@ -305,6 +306,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.perfmark/perfmark-api/0.26.0 Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/LICENSE b/material/licenses/collector/LICENSE index b141b35c26a..ea272bc1567 100644 --- a/material/licenses/collector/LICENSE +++ b/material/licenses/collector/LICENSE @@ -222,6 +222,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6 Apache-2.0 https://mvnrepository.com/artifact/commons-net/commons-net/3.10.0 Apache-2.0 https://mvnrepository.com/artifact/commons-validator/commons-validator/1.7 Apache-2.0 + https://mvnrepository.com/artifact/io.asyncer/r2dbc-mysql/1.4.1 Apache-2.0 https://mvnrepository.com/artifact/io.lettuce/lettuce-core/6.3.1.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.3 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.12.3 Apache-2.0 @@ -259,6 +260,8 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.netty/netty-transport-sctp/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.netty/netty-transport-udt/4.1.100.Final Apache-2.0 https://mvnrepository.com/artifact/io.projectreactor/reactor-core/3.6.3 Apache-2.0 + https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty-core/1.3.3 Apache-2.0 + https://mvnrepository.com/artifact/io.r2dbc/r2dbc-spi/1.0.0.RELEASE Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0 Apache-2.0 https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0 Apache-2.0 diff --git a/material/licenses/collector/NOTICE b/material/licenses/collector/NOTICE index c56b217a965..1d686114d90 100644 --- a/material/licenses/collector/NOTICE +++ b/material/licenses/collector/NOTICE @@ -746,3 +746,19 @@ Copyright 2001-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================== +Reactive Relational Database Connectivity + +Copyright 2017-2022 the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +======================================================================== diff --git a/pom.xml b/pom.xml index fdbdcd0960a..a10511c3a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 1.8.0 - 21 + 25 ${java.version} ${java.version} 3.2.0 @@ -134,6 +134,7 @@ 3.7.1 4.1.117.Final 8.4.0 + 1.4.1 12.10.2.jre11 21.5.0.0 4.6.1 @@ -183,6 +184,7 @@ 2.25.0 2.25.0-alpha 1.3.1-alpha + 2.0.3 @@ -610,6 +612,10 @@ org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} + + + full + org.apache.maven.plugins @@ -621,6 +627,28 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + require-java-25 + validate + + enforce + + + + + [25,) + Apache HertzBeat now requires JDK 25 or newer. Please switch JAVA_HOME before building or testing. + + + + + + org.apache.maven.plugins diff --git a/script/application.yml b/script/application.yml index ecebc66681f..90c0ea4bb8e 100644 --- a/script/application.yml +++ b/script/application.yml @@ -336,6 +336,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/assembly/collector/assembly-native.xml b/script/assembly/collector/assembly-native.xml new file mode 100644 index 00000000000..c341eaffd05 --- /dev/null +++ b/script/assembly/collector/assembly-native.xml @@ -0,0 +1,82 @@ + + + + ${native.package.id} + true + ${native.package.baseDirectory} + + ${native.package.format} + + + + + ${native.launcher.dir} + true + bin + 0755 + + + + src/main/resources + + application.yml + logback-spring.xml + banner.txt + + true + ${file.separator}config + + + + ../../ + ${file.separator} + + README.md + + + + ../../material/licenses/collector + ${file.separator} + + LICENSE + NOTICE + + + + ../../material/licenses/collector + licenses + + LICENSE-* + + + + + + + ${native.binary.source} + ${file.separator} + ${native.executable.packageName} + 0755 + + + diff --git a/script/assembly/collector/assembly.xml b/script/assembly/collector/assembly.xml index f4af4618f28..d02ee97e0cc 100644 --- a/script/assembly/collector/assembly.xml +++ b/script/assembly/collector/assembly.xml @@ -58,6 +58,9 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> banner.txt META-INF/** + + META-INF/spring.factories + true ${file.separator}config @@ -68,7 +71,7 @@ http://maven.apache.org/ASSEMBLY/2.0.0 "> target / - *.jar + ${project.build.finalName}.jar diff --git a/script/assembly/collector/bin-native-win/restart.bat b/script/assembly/collector/bin-native-win/restart.bat new file mode 100644 index 00000000000..50521a8d85b --- /dev/null +++ b/script/assembly/collector/bin-native-win/restart.bat @@ -0,0 +1,25 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +echo Restarting Apache HertzBeat ${project.artifactId} ... +call "%~dp0shutdown.bat" +timeout /t 2 /nobreak >nul +call "%~dp0startup.bat" +exit /b %ERRORLEVEL% diff --git a/script/assembly/collector/bin-native-win/shutdown.bat b/script/assembly/collector/bin-native-win/shutdown.bat new file mode 100644 index 00000000000..eb1dc08e316 --- /dev/null +++ b/script/assembly/collector/bin-native-win/shutdown.bat @@ -0,0 +1,49 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set SERVER_PORT=1159 + +set PID= +if exist "%PID_FILE%" ( + set /p PID=<"%PID_FILE%" +) + +if not defined PID ( + for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$conn = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Select-Object -First 1; if ($conn) { $conn.OwningProcess }"') do set PID=%%i +) + +if not defined PID ( + echo Apache HertzBeat %SERVER_NAME% is already stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %PID% -ErrorAction SilentlyContinue) { Stop-Process -Id %PID% -Force; exit 0 } exit 1" +del /q "%PID_FILE%" >nul 2>&1 + +echo Shutdown Apache HertzBeat %SERVER_NAME% Success! +exit /b 0 diff --git a/script/assembly/collector/bin-native-win/startup.bat b/script/assembly/collector/bin-native-win/startup.bat new file mode 100644 index 00000000000..33bfec4f48c --- /dev/null +++ b/script/assembly/collector/bin-native-win/startup.bat @@ -0,0 +1,102 @@ +@rem +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@echo off +setlocal + +set SERVER_NAME=${project.artifactId} +set BINARY_NAME=${native.executable.packageName} + +cd /d %~dp0 +cd .. +set DEPLOY_DIR=%CD% +set CONF_DIR=%DEPLOY_DIR%\config +set LOGS_DIR=%DEPLOY_DIR%\logs +set PID_FILE=%LOGS_DIR%\%SERVER_NAME%.pid +set APP_PATH=%DEPLOY_DIR%\%BINARY_NAME% +set SERVER_PORT=1159 +set STDOUT_LOG=%LOGS_DIR%\startup.out.log +set STDERR_LOG=%LOGS_DIR%\startup.err.log + +if "%1"=="status" goto status + +if not exist "%APP_PATH%" ( + echo ERROR: native executable not found: %APP_PATH% + exit /b 1 +) + +if not exist "%LOGS_DIR%" ( + mkdir "%LOGS_DIR%" +) + +if exist "%PID_FILE%" ( + set /p RUNNING_PID=<"%PID_FILE%" + powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" + if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% already started! + echo PID: %RUNNING_PID% + exit /b 1 + ) + del /q "%PID_FILE%" >nul 2>&1 +) + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$portInUse = Get-NetTCPConnection -State Listen -LocalPort %SERVER_PORT% -ErrorAction SilentlyContinue; if ($portInUse) { exit 0 } exit 1" +if not errorlevel 1 ( + echo ERROR: The HertzBeat %SERVER_NAME% port %SERVER_PORT% is already used! + exit /b 1 +) + +echo You can review logs at hertzbeat\logs +echo Starting the HertzBeat %SERVER_NAME% ... + +for /f %%i in ('powershell -NoProfile -ExecutionPolicy Bypass -Command "$p = Start-Process -FilePath ''%APP_PATH%'' -ArgumentList ''--spring.config.location=%CONF_DIR%\'' -RedirectStandardOutput ''%STDOUT_LOG%'' -RedirectStandardError ''%STDERR_LOG%'' -PassThru; $p.Id"') do set APP_PID=%%i + +if not defined APP_PID ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + exit /b 1 +) + +>"%PID_FILE%" echo %APP_PID% + +powershell -NoProfile -ExecutionPolicy Bypass -Command "$deadline = (Get-Date).AddSeconds(30); do { Start-Sleep -Seconds 1; if (-not (Get-Process -Id %APP_PID% -ErrorAction SilentlyContinue)) { exit 1 } $listening = Get-NetTCPConnection -LocalPort %SERVER_PORT% -State Listen -ErrorAction SilentlyContinue | Where-Object { $_.OwningProcess -eq %APP_PID% }; if ($listening) { exit 0 } } while ((Get-Date) -lt $deadline); exit 0" +if errorlevel 1 ( + echo ERROR: Service start failed, check %STDOUT_LOG% and %STDERR_LOG% + del /q "%PID_FILE%" >nul 2>&1 + exit /b 1 +) + +echo Service Start Success! +echo Service PID: %APP_PID% +exit /b 0 + +:status +if not exist "%PID_FILE%" ( + echo The HertzBeat %SERVER_NAME% is stopped + exit /b 0 +) + +set /p RUNNING_PID=<"%PID_FILE%" +powershell -NoProfile -ExecutionPolicy Bypass -Command "if (Get-Process -Id %RUNNING_PID% -ErrorAction SilentlyContinue) { exit 0 } exit 1" +if errorlevel 1 ( + echo The HertzBeat %SERVER_NAME% is stopped + del /q "%PID_FILE%" >nul 2>&1 + exit /b 0 +) + +echo The HertzBeat %SERVER_NAME% is running...! +echo PID: %RUNNING_PID% +exit /b 0 diff --git a/script/assembly/collector/bin-native/restart.sh b/script/assembly/collector/bin-native/restart.sh new file mode 100644 index 00000000000..af640ba89f1 --- /dev/null +++ b/script/assembly/collector/bin-native/restart.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +startTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Now!\033[0m" +./shutdown.sh +echo +sleep 2 +echo +./startup.sh +endTime=$(date +%s) +echo -e "\033[0;31mCurrent Time is：$(date "+%Y-%m-%d %H:%M:%S") Restart Success！Spend $((endTime - startTime)) seconds \033[0m" diff --git a/script/assembly/collector/bin-native/shutdown.sh b/script/assembly/collector/bin-native/shutdown.sh new file mode 100644 index 00000000000..1c0478edbac --- /dev/null +++ b/script/assembly/collector/bin-native/shutdown.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +PID="$(find_running_pid)" +if [ -z "$PID" ]; then + echo "Apache HertzBeat ${SERVER_NAME} is already stopped" + rm -f "$PID_FILE" + exit 0 +fi + +kill "$PID" +for _ in $(seq 1 30); do + if ! kill -0 "$PID" 2>/dev/null; then + rm -f "$PID_FILE" + echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" + exit 0 + fi + sleep 1 +done + +kill -9 "$PID" 2>/dev/null +rm -f "$PID_FILE" +echo "Shutdown Apache HertzBeat ${SERVER_NAME} Success!" diff --git a/script/assembly/collector/bin-native/startup.sh b/script/assembly/collector/bin-native/startup.sh new file mode 100644 index 00000000000..bd57563fad8 --- /dev/null +++ b/script/assembly/collector/bin-native/startup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SERVER_NAME="${project.artifactId}" +BINARY_NAME="${project.build.finalName}" + +cd "$(dirname "$0")" +BIN_DIR="$(pwd)" +cd .. +DEPLOY_DIR="$(pwd)" + +CONF_DIR="$DEPLOY_DIR/config" +LOGS_DIR="$DEPLOY_DIR/logs" +PID_FILE="$LOGS_DIR/${project.artifactId}.pid" +APP_PATH="$DEPLOY_DIR/$BINARY_NAME" +SERVER_PORT=1159 + +find_running_pid() { + if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE" 2>/dev/null)" + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + echo "$PID" + return 0 + fi + fi + + ps -ef | grep "$APP_PATH" | grep "$CONF_DIR" | grep -v grep | awk '{print $2}' | head -n 1 +} + +RUNNING_PID="$(find_running_pid)" +if [ "$1" = "status" ]; then + if [ -n "$RUNNING_PID" ]; then + echo "The HertzBeat $SERVER_NAME is running...!" + echo "PID: $RUNNING_PID" + else + echo "The HertzBeat $SERVER_NAME is stopped" + fi + exit 0 +fi + +if [ ! -x "$APP_PATH" ]; then + echo "ERROR: native executable not found: $APP_PATH" + exit 1 +fi + +if [ -n "$RUNNING_PID" ]; then + echo "ERROR: The HertzBeat $SERVER_NAME already started!" + echo "PID: $RUNNING_PID" + exit 1 +fi + +mkdir -p "$LOGS_DIR" + +if command -v lsof >/dev/null 2>&1; then + SERVER_PORT_COUNT="$(lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | wc -l)" + if [ "$SERVER_PORT_COUNT" -gt 0 ]; then + echo "ERROR: The HertzBeat $SERVER_NAME port $SERVER_PORT is already used!" + exit 1 + fi +fi + +echo "You can review logs at hertzbeat/logs" +echo "Starting the HertzBeat $SERVER_NAME ..." +nohup "$APP_PATH" --spring.config.location="$CONF_DIR/" >"$LOGS_DIR/startup.log" 2>&1 & +APP_PID=$! +echo "$APP_PID" >"$PID_FILE" + +COUNT=0 +while [ $COUNT -lt 30 ]; do + sleep 1 + if ! kill -0 "$APP_PID" 2>/dev/null; then + echo "ERROR: Service start failed, check $LOGS_DIR/startup.log" + rm -f "$PID_FILE" + exit 1 + fi + if command -v lsof >/dev/null 2>&1 && lsof -nP -iTCP:$SERVER_PORT -sTCP:LISTEN | grep -q "$APP_PID"; then + break + fi + COUNT=$((COUNT + 1)) +done + +echo "Service Start Success!" +echo "Service PID: $APP_PID" diff --git a/script/assembly/collector/bin/startup.sh b/script/assembly/collector/bin/startup.sh index 1d2db5f9c27..dbdab8c581b 100644 --- a/script/assembly/collector/bin/startup.sh +++ b/script/assembly/collector/bin/startup.sh @@ -103,7 +103,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/assembly/server/bin/startup.sh b/script/assembly/server/bin/startup.sh index ebf25d7ae30..45bff0dc90f 100644 --- a/script/assembly/server/bin/startup.sh +++ b/script/assembly/server/bin/startup.sh @@ -107,7 +107,7 @@ if [ -f "./java/bin/java" ]; then else JAVA_EXIST=`which java | grep bin | wc -l` if [ $JAVA_EXIST -le 0 ]; then - echo -e "ERROR: there is no java21+ environment, please config java environment." + echo -e "ERROR: there is no java${java.version}+ environment, please config java environment." exit 1 fi echo -e "Use the system environment jdk to start" diff --git a/script/ci/github-actions/setup-deps/action.yml b/script/ci/github-actions/setup-deps/action.yml index 0e1698fe720..3ad4560a589 100644 --- a/script/ci/github-actions/setup-deps/action.yml +++ b/script/ci/github-actions/setup-deps/action.yml @@ -21,11 +21,11 @@ description: Install host system dependencies (with mvnd) runs: using: composite steps: - - name: Set up JDK 21 + - name: Set up JDK 25 uses: actions/setup-java@v4 with: distribution: "zulu" - java-version: 21 + java-version: 25 - name: Install mvnd shell: bash @@ -40,4 +40,4 @@ runs: - name: Verify mvnd installation shell: bash - run: mvnd --version \ No newline at end of file + run: mvnd --version diff --git a/script/docker-compose/README.md b/script/docker-compose/README.md index 99908b2cc4c..3a7b931bb9d 100644 --- a/script/docker-compose/README.md +++ b/script/docker-compose/README.md @@ -2,9 +2,14 @@ Suggest the [HertzBeat + GreptimeDB + Postgresql Solution](hertzbeat-postgresql-greptimedb) for the best performance and stability. +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If you place `mysql-connector-j` in `ext-lib`, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC driver jars in `ext-lib`. + - Use Postgresql + GreptimeDB as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+GreptimeDB Solution](hertzbeat-postgresql-greptimedb) - Use Postgresql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+PostgreSQL+VictoriaMetrics Solution](hertzbeat-postgresql-victoria-metrics) - Use Mysql + VictoriaMetrics as Hertzbeat dependent storage -> [HertzBeat+Mysql+VictoriaMetrics Solution](hertzbeat-mysql-victoria-metrics) - Use Mysql + IoTDB as Hertzbeat dependent storage -> [HertzBeat+Mysql+IoTDB Solution](hertzbeat-mysql-iotdb) - Use Mysql + Tdengine as Hertzbeat dependent storage -> [HertzBeat+Mysql+Tdengine Solution](hertzbeat-mysql-tdengine) - diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README.md b/script/docker-compose/hertzbeat-mysql-iotdb/README.md index b5b94efcec2..b6c8fc30504 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-iotdb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/ hertzbeat-mysql-iotdb) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md index 9f145fe769b..e310d72ce40 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-iotdb/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-iotdb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-iotdb) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index c4c9d80797e..33b0537468e 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml index 9a1af1589e8..472cca7f2c8 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/docker-compose.yaml @@ -68,6 +68,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-iotdb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README.md b/script/docker-compose/hertzbeat-mysql-tdengine/README.md index 27c1febaba5..0b7d988438b 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-tdengine` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md index 329344585d3..ad2d2758dad 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-tdengine/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-tdengine` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-tdengine) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 8a6dfe1528f..8d795ed6434 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml index 33afa63dadd..83e151bb550 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-tdengine/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md index 02a6829ea8f..8d5a2c52410 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README.md @@ -17,10 +17,11 @@ 1. Download the hertzbeat-docker-compose installation deployment script file The script file is located in `script/docker-compose/hertzbeat-mysql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. Add MYSQL jdbc driver jar +2. Optional: add external JDBC driver jars to `ext-lib` - Download the MYSQL jdbc driver jar package, such as mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - Copy the jar package to the ext-lib directory. + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. 3. Enter the deployment script docker-compose directory, execute diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md index 6602b34a318..30bf8f11e96 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/README_CN.md @@ -19,9 +19,10 @@ 1. 下载hertzbeat-docker-compose安装部署脚本文件 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-mysql-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-mysql-victoria-metrics) -2. 添加 MYSQL jdbc 驱动 jar - 下载 MYSQL jdbc driver jar, 例如 mysql-connector-java-8.0.25.jar. https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.25.zip - 将此 jar 包拷贝放入 ext-lib 目录下. +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 3. 进入部署脚本 docker-compose 目录, 执行 diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml index d9118c6c560..f92abb9f989 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/application.yml @@ -236,6 +236,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml index da4188bd442..acebf9889b6 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/docker-compose.yaml @@ -67,6 +67,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md index 5a84321adcb..8c2dd54d1e2 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-greptimedb` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md index 609ba130f64..2b375c99d2c 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgresql-greptimedb` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-greptimedb) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml index 0a1feb682dc..6cc14900303 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/application.yml @@ -233,6 +233,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml index be1603c409d..2c3782607ba 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/docker-compose.yaml @@ -87,6 +87,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md index ff4edb7525e..d791a30dd13 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README.md @@ -18,7 +18,13 @@ The script file is located in `script/docker-compose/hertzbeat-postgresql-victoria-metrics` link [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. Enter the deployment script docker-compose directory, execute +2. Optional: add external JDBC driver jars to `ext-lib` + + MySQL-compatible monitoring can use the built-in query engine directly, so `mysql-connector-j` is optional. + If you want HertzBeat to prefer JDBC after restart, place `mysql-connector-j` in `ext-lib`. + Oracle and DB2 still require external JDBC jars in `ext-lib`. + +3. Enter the deployment script docker-compose directory, execute `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md index f809587e05d..d6982c63092 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/README_CN.md @@ -20,7 +20,12 @@ 脚本文件位于代码仓库下`script/docker-compose/hertzbeat-postgre-victoria-metrics` 链接 [script/docker-compose](https://github.com/apache/hertzbeat/tree/master/script/docker-compose/hertzbeat-postgresql-victoria-metrics) -2. 进入部署脚本 docker-compose 目录, 执行 +2. 可选：向 `ext-lib` 添加外部 JDBC 驱动 jar + MySQL 兼容监控现在可以直接使用内置查询引擎，所以 `mysql-connector-j` 不是必需项。 + 如果你希望 HertzBeat 在重启后优先走 JDBC，可以把 `mysql-connector-j` 放到 `ext-lib`。 + Oracle、DB2 这类场景仍然需要把外部 JDBC 驱动放到 `ext-lib`。 + +3. 进入部署脚本 docker-compose 目录, 执行 `docker compose up -d` diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml index ed4c59bcbb4..3108f5b9944 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/application.yml @@ -235,6 +235,13 @@ grafana: password: admin hertzbeat: + collector: + mysql: + # MySQL-compatible query engine routing for MySQL, MariaDB, OceanBase, and TiDB SQL metrics. + # auto : prefer JDBC only when mysql-connector-j is available from ext-lib, otherwise use the built-in query engine + # jdbc : always use JDBC + # r2dbc : always use the built-in query engine + query-engine: ${HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE:auto} # Optional virtual-thread overrides. Remove this whole block to use built-in defaults. vthreads: enabled: true diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml index fa28d86ea79..fda39244719 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/docker-compose.yaml @@ -70,6 +70,7 @@ services: hostname: hertzbeat restart: always environment: + HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto TZ: Asia/Shanghai LANG: zh_CN.UTF-8 depends_on: diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README index 5898fde6b91..7e270434ca3 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/ext-lib/README @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need external JDBC jars, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +Notes: + +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, HertzBeat prefers JDBC after restart. +- Oracle and DB2 still require external JDBC jars in `ext-lib`. diff --git a/script/docker/collector/Dockerfile b/script/docker/collector/Dockerfile index a6f89d8c89d..5a33eb5b392 100644 --- a/script/docker/collector/Dockerfile +++ b/script/docker/collector/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" @@ -26,7 +26,7 @@ RUN sed -i 's#http://#https://#g' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && apt-get install -y openssh-server RUN mkdir /var/run/sshd -ADD apache-hertzbeat-collector-*-bin.tar.gz /opt/ +ADD apache-hertzbeat-collector-${VERSION}-bin.tar.gz /opt/ ENV JAVA_OPTS "" ENV TZ=Asia/Shanghai diff --git a/script/docker/collector/build.sh b/script/docker/collector/build.sh index 278df32c783..03f93312b0d 100644 --- a/script/docker/collector/build.sh +++ b/script/docker/collector/build.sh @@ -21,13 +21,20 @@ cd `dirname $0` CURRENT_DIR=`pwd` # cd dist dir cd ../../../dist -# auto detect hertzbeat version -VERSION=`ls apache-hertzbeat-collector-*-bin.tar.gz| awk -F"-" '{print $4}'` +# auto detect the JVM collector package version +PACKAGE_FILE=$(find . -maxdepth 1 -name 'apache-hertzbeat-collector-*-bin.tar.gz' \ + ! -name 'apache-hertzbeat-collector-native-*-bin.tar.gz' | head -n 1) +VERSION=$(basename "$PACKAGE_FILE" | sed -e 's/^apache-hertzbeat-collector-//' -e 's/-bin\.tar\.gz$//') # use the version param if [ -n "$1" ]; then VERSION="$1"; fi +if [ -z "$VERSION" ]; then + echo "Can not find the JVM collector package under dist/. Build apache-hertzbeat-collector-{version}-bin.tar.gz first." + exit 1 +fi + # docker compile context CONTEXT_DIR=`pwd` diff --git a/script/docker/server/Dockerfile b/script/docker/server/Dockerfile index 773c21c0d15..6427e6300b9 100644 --- a/script/docker/server/Dockerfile +++ b/script/docker/server/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM eclipse-temurin:21-jdk +FROM eclipse-temurin:25-jdk MAINTAINER Apache HertzBeat "dev@hertzbeat.apache.org" diff --git a/script/ext-lib/README b/script/ext-lib/README index 5898fde6b91..30800cd2328 100644 --- a/script/ext-lib/README +++ b/script/ext-lib/README @@ -13,9 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -Please move external libs to this folder like: +Please move external libs to this folder only when you need JVM runtime extension, for example: ojdbc8-21.5.0.0.jar orai18n-21.5.0.0.jar mysql-connector-java-8.0.30.jar +jcc-11.5.9.0.jar +Note: + +- `ext-lib` is loaded by the JVM server package and the JVM collector package. +- MySQL, MariaDB, OceanBase, and TiDB SQL query metrics can use the built-in MySQL-compatible query engine without `mysql-connector-j`. +- If `mysql-connector-j` is present here, the JVM server package or JVM collector package prefers JDBC after restart. +- The native collector package does not support loading external JDBC driver jars from `ext-lib` at runtime. +- If you need Oracle or DB2 monitoring, or you explicitly want the JDBC path for MySQL-compatible monitoring, use the JVM collector package.