diff --git a/.github/workflows/backend-build-test.yml b/.github/workflows/backend-build-test.yml index abb52e8960e..a71c0a0b19c 100644 --- a/.github/workflows/backend-build-test.yml +++ b/.github/workflows/backend-build-test.yml @@ -76,7 +76,7 @@ jobs: - name: Build backend Maven E2E modules run: | mvnd clean -B package \ - -pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-log-e2e \ + -pl hertzbeat-e2e/hertzbeat-collector-common-e2e,hertzbeat-e2e/hertzbeat-collector-kafka-e2e,hertzbeat-e2e/hertzbeat-collector-basic-e2e,hertzbeat-e2e/hertzbeat-collector-mysql-r2dbc-e2e,hertzbeat-e2e/hertzbeat-observability-e2e \ -am \ -Dmaven.test.skip=false \ --file pom.xml diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java index 47e8bacfe33..e3b19977748 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java @@ -69,6 +69,8 @@ interface FunctionModuleConstants { String GRAFANA = "grafana"; String LOG = "log"; + + String OBSERVABILITY = "observability"; } } diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java index 7f1f3647855..8e1210d43aa 100644 --- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java +++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/constants/NetworkConstants.java @@ -61,7 +61,7 @@ interface HttpClientConstants { Duration READ_TIMEOUT = Duration.ofSeconds(6); Duration WRITE_TIMEOUT = Duration.ofSeconds(6); Duration CONNECT_TIMEOUT = Duration.ofSeconds(6); - Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(5); + Duration GREPTIME_QUERY_READ_TIMEOUT = Duration.ofSeconds(15); Duration GREPTIME_QUERY_CONNECT_TIMEOUT = Duration.ofSeconds(2); Duration GREPTIME_WRITE_READ_TIMEOUT = Duration.ofSeconds(3); Duration GREPTIME_WRITE_CONNECT_TIMEOUT = Duration.ofSeconds(2); diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml similarity index 97% rename from hertzbeat-e2e/hertzbeat-log-e2e/pom.xml rename to hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml index a5638e23a20..dfffdb62c70 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/pom.xml @@ -25,7 +25,7 @@ 4.0.0 - hertzbeat-log-e2e + hertzbeat-observability-e2e ${java.version} @@ -52,7 +52,7 @@ org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability ${hertzbeat.version} test diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java index 9957ce417b0..cb5abb26f43 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogPeriodicAlertE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogPeriodicAlertE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.alert; +package org.apache.hertzbeat.observability.alert; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java index 4ae973b343c..1e93a4c77a4 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/alert/LogRealTimeAlertE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/alert/LogRealTimeAlertE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.alert; +package org.apache.hertzbeat.observability.alert; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java similarity index 98% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java index 57feada98ed..5fcb3e0668c 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/ingestion/LogIngestionE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/ingestion/LogIngestionE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.ingestion; +package org.apache.hertzbeat.observability.ingestion; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; diff --git a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java new file mode 100644 index 00000000000..c6ab50e64a6 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeEntityFreeSignalStorageE2eTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.storage; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.ByteString; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.resource.v1.Resource; +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeOtlpSignalStorage; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.junit.jupiter.api.Test; +import org.springframework.web.client.RestTemplate; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +/** Proves the warehouse-owned Entity-free OTLP log path against a real GreptimeDB. */ +@Testcontainers +class GreptimeEntityFreeSignalStorageE2eTest { + + private static final int GREPTIME_HTTP_PORT = 4000; + private static final int GREPTIME_GRPC_PORT = 4001; + private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql"; + private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml"; + private static final String PIPELINE_NAME = "hertzbeat_otlp_log_v1"; + private static final String TRACE_ID = "0123456789abcdef0123456789abcdef"; + private static final String SPAN_ID = "0123456789abcdef"; + private static final String BODY = "entity-free greptime proof"; + private static final long LOG_TIME_NANOS = 1_710_000_000_123_456_789L; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Container + @SuppressWarnings("resource") + private static final GenericContainer GREPTIME = new GenericContainer<>( + DockerImageName.parse("greptime/greptimedb:latest")) + .withExposedPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT) + .withCommand("standalone", "start", + "--http-addr", "0.0.0.0:" + GREPTIME_HTTP_PORT, + "--rpc-bind-addr", "0.0.0.0:" + GREPTIME_GRPC_PORT) + .waitingFor(Wait.forListeningPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT)) + .withStartupTimeout(Duration.ofSeconds(120)); + + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @Test + void warehouseStorageShouldPersistEntityFreeOtlpLogs() throws Exception { + executeSql(classpathResource(LOG_SCHEMA).strip().replaceFirst(";\\s*$", "")); + uploadPipeline(); + + GreptimeOtlpSignalStorage storage = new GreptimeOtlpSignalStorage( + new GreptimeProperties(true, GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_GRPC_PORT), + endpoint(), "public", "", ""), + new RestTemplate()); + + storage.writeProtobuf("logs", request().toByteArray()); + + await().atMost(Duration.ofSeconds(30)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> { + String sql = "SELECT COUNT(*) AS count FROM hertzbeat_logs WHERE trace_id = '" + TRACE_ID + + "' AND body = '" + BODY + "'"; + assertThat(queryCount(sql)).isEqualTo(1); + }); + } + + private void uploadPipeline() throws Exception { + String boundary = "----hertzbeat-entity-free-proof"; + String body = "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"pipeline.yaml\"\r\n" + + "Content-Type: application/x-yaml\r\n\r\n" + + classpathResource(LOG_PIPELINE) + "\r\n" + + "--" + boundary + "--\r\n"; + HttpResponse response = httpClient.send(HttpRequest.newBuilder() + .uri(URI.create(endpoint() + "/v1/pipelines/" + PIPELINE_NAME)) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).as(response.body()).isBetween(200, 299); + } + + private ExportLogsServiceRequest request() { + LogRecord record = LogRecord.newBuilder() + .setTimeUnixNano(LOG_TIME_NANOS) + .setObservedTimeUnixNano(LOG_TIME_NANOS) + .setSeverityNumberValue(9) + .setSeverityText("INFO") + .setBody(AnyValue.newBuilder().setStringValue(BODY).build()) + .setTraceId(ByteString.copyFrom(hexToBytes(TRACE_ID))) + .setSpanId(ByteString.copyFrom(hexToBytes(SPAN_ID))) + .build(); + return ExportLogsServiceRequest.newBuilder() + .addResourceLogs(ResourceLogs.newBuilder() + .setResource(Resource.newBuilder() + .addAttributes(stringAttribute("service.name", "checkout")) + .addAttributes(stringAttribute("deployment.environment.name", "test")) + .build()) + .addScopeLogs(ScopeLogs.newBuilder().addLogRecords(record).build()) + .build()) + .build(); + } + + private int queryCount(String sql) throws Exception { + JsonNode rows = OBJECT_MAPPER.readTree(executeSql(sql).body()) + .path("output").path(0).path("records").path("rows"); + assertThat(rows.isArray()).isTrue(); + assertThat(rows).isNotEmpty(); + return rows.get(0).get(0).asInt(); + } + + private HttpResponse executeSql(String sql) throws Exception { + HttpResponse response = httpClient.send(HttpRequest.newBuilder() + .uri(URI.create(endpoint() + "/v1/sql?db=public")) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString( + "sql=" + URLEncoder.encode(sql, StandardCharsets.UTF_8), StandardCharsets.UTF_8)) + .build(), HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).as(response.body()).isBetween(200, 299); + return response; + } + + private String classpathResource(String path) throws Exception { + try (InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(path)) { + assertThat(input).as(path).isNotNull(); + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static KeyValue stringAttribute(String key, String value) { + return KeyValue.newBuilder().setKey(key) + .setValue(AnyValue.newBuilder().setStringValue(value).build()).build(); + } + + private static byte[] hexToBytes(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int index = 0; index < value.length(); index += 2) { + bytes[index / 2] = (byte) Integer.parseInt(value.substring(index, index + 2), 16); + } + return bytes; + } + + private static String endpoint() { + return "http://" + GREPTIME.getHost() + ':' + GREPTIME.getMappedPort(GREPTIME_HTTP_PORT); + } +} diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java index d56a46ee6c6..7c62fb38462 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/java/org/apache/hertzbeat/log/storage/GreptimeLogStorageE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/java/org/apache/hertzbeat/observability/storage/GreptimeLogStorageE2eTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.storage; +package org.apache.hertzbeat.observability.storage; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml similarity index 94% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml index 1763868f1e1..55e0eae065e 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/sureness.yml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml @@ -80,12 +80,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # config the resource restful api that need bypass auth protection # rule: api===method @@ -95,7 +92,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml similarity index 99% rename from hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml rename to hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml index f86f2b01001..baea29f4c7d 100644 --- a/hertzbeat-e2e/hertzbeat-log-e2e/src/test/resources/vector.yml +++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml @@ -102,7 +102,7 @@ sinks: type: opentelemetry protocol: type: http - uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/logs/ingest/otlp" + uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/otlp/v1/logs" method: post encoding: codec: json diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index 6249cf0d447..2f261897528 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -32,7 +32,7 @@ hertzbeat-collector-kafka-e2e hertzbeat-collector-basic-e2e hertzbeat-collector-mysql-r2dbc-e2e - hertzbeat-log-e2e + hertzbeat-observability-e2e diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java deleted file mode 100644 index e32510e4d68..00000000000 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogIngestionController.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.extern.slf4j.Slf4j; -import java.util.List; - -import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.common.entity.dto.Message; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * Generic Log Ingestion Controller - * Provides a fallback endpoint for log protocols that don't have dedicated controllers. - * For OTLP protocol, use OtlpLogController instead. - */ -@Tag(name = "Log Ingestion Controller") -@RestController -@RequestMapping(path = "/api/logs", produces = MediaType.APPLICATION_JSON_VALUE) -@Slf4j -public class LogIngestionController { - - private final List protocolAdapters; - - public LogIngestionController(List protocolAdapters) { - this.protocolAdapters = protocolAdapters; - } - - /** - * Receive log payload pushed from external system specifying the log protocol. - * - * @param protocol log protocol identifier (e.g., "vector", "loki") - * @param content raw request body - */ - @Operation(summary = "Ingest logs by protocol name") - @PostMapping(value = "/ingest/{protocol}", consumes = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity> ingestLog(@PathVariable("protocol") String protocol, - @RequestBody String content) { - log.debug("Receive log from protocol: {}, content length: {}", protocol, content == null ? 0 : content.length()); - - for (LogProtocolAdapter adapter : protocolAdapters) { - if (adapter.supportProtocol().equalsIgnoreCase(protocol)) { - try { - adapter.ingest(content); - return ResponseEntity.ok(Message.success("Add extern log success")); - } catch (Exception e) { - log.error("Add log failed: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(Message.fail(CommonConstants.FAIL_CODE, "Add extern log failed: " + e.getMessage())); - } - } - } - log.warn("Not support extern log from protocol: {}", protocol); - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(Message.fail(CommonConstants.FAIL_CODE, "Not support the " + protocol + " protocol log")); - } -} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java deleted file mode 100644 index 3d010750810..00000000000 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpLogController.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import com.fasterxml.jackson.core.io.JsonStringEncoder; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.google.rpc.Status; -import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import lombok.extern.slf4j.Slf4j; -import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * OTLP Log Ingestion Controller - * Implements OTLP/HTTP specification for log ingestion. - * Supports both binary-encoded Protobuf (application/x-protobuf) and JSON-encoded Protobuf (application/json). - * - * @see OTLP/HTTP Specification - */ -@Tag(name = "OTLP Log Controller") -@RestController -@RequestMapping(path = "/api/logs/otlp") -@Slf4j -public class OtlpLogController { - - private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf"; - - private static final ExportLogsServiceResponse EMPTY_RESPONSE = ExportLogsServiceResponse.newBuilder().build(); - - private final OtlpLogProtocolAdapter otlpLogProtocolAdapter; - - public OtlpLogController(OtlpLogProtocolAdapter otlpLogProtocolAdapter) { - this.otlpLogProtocolAdapter = otlpLogProtocolAdapter; - } - - /** - * OTLP/HTTP standard endpoint for logs with JSON-encoded Protobuf payload. - * Content-Type: application/json - * - * Response follows OTLP specification: - * - Success: HTTP 200 with ExportLogsServiceResponse (JSON encoded) - * - Failure: HTTP 400 with google.rpc.Status (JSON encoded) - * - * @param content JSON-encoded ExportLogsServiceRequest - * @return ExportLogsServiceResponse on success, Status on failure - */ - @Operation(summary = "Ingest OTLP logs (JSON format)") - @PostMapping(value = "/v1/logs", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity ingestJsonLogs(@RequestBody String content) { - log.debug("Receive OTLP JSON logs, content length: {}", content == null ? 0 : content.length()); - try { - otlpLogProtocolAdapter.ingest(content); - return ResponseEntity.ok(toJsonResponse(EMPTY_RESPONSE)); - } catch (IllegalArgumentException e) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(toJsonErrorResponse(e.getMessage())); - } catch (Exception e) { - // Server-side errors - unexpected failure - log.error("Unexpected error ingesting OTLP JSON logs: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(toJsonErrorResponse(e.getMessage())); - } - } - - /** - * OTLP/HTTP standard endpoint for logs with binary-encoded Protobuf payload. - * Content-Type: application/x-protobuf - * - * Response follows OTLP specification: - * - Success: HTTP 200 with ExportLogsServiceResponse (binary encoded) - * - Failure: HTTP 400 with google.rpc.Status (binary encoded) - * - * @param content binary-encoded ExportLogsServiceRequest - * @return ExportLogsServiceResponse on success, Status on failure - */ - @Operation(summary = "Ingest OTLP logs (binary Protobuf format)") - @PostMapping(value = "/v1/logs", consumes = CONTENT_TYPE_PROTOBUF, produces = CONTENT_TYPE_PROTOBUF) - public ResponseEntity ingestBinaryLogs(@RequestBody byte[] content) { - log.debug("Receive OTLP binary logs, content length: {}", content == null ? 0 : content.length); - try { - otlpLogProtocolAdapter.ingestBinary(content); - return ResponseEntity.ok(EMPTY_RESPONSE.toByteArray()); - } catch (IllegalArgumentException e) { - // Client-side validation errors - malformed request - return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(createBinaryErrorResponse(e.getMessage())); - } catch (Exception e) { - // Server-side errors - unexpected failure - log.error("Unexpected error ingesting OTLP binary logs: {}", e.getMessage(), e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(createBinaryErrorResponse(e.getMessage())); - } - } - - private String toJsonResponse(ExportLogsServiceResponse response) { - try { - return JsonFormat.printer().print(response); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to convert ExportLogsServiceResponse to JSON: {}", e.getMessage(), e); - return "{}"; - } - } - - private String toJsonErrorResponse(String message) { - Status status = Status.newBuilder() - .setMessage(message != null ? message : "Unknown error") - .build(); - try { - return JsonFormat.printer().print(status); - } catch (InvalidProtocolBufferException e) { - return "{\"message\":\"" + escapeJson(message) + "\"}"; - } - } - - /** - * Escapes a string value for safe inclusion in JSON. - * - * @param message the string to escape - * @return the escaped string, or empty string if message is null - */ - private String escapeJson(String message) { - if (message == null) { - return ""; - } - - char[] escaped = JsonStringEncoder.getInstance().quoteAsString(message); - return new String(escaped); - } - - private byte[] createBinaryErrorResponse(String message) { - return Status.newBuilder() - .setMessage(message != null ? message : "Unknown error") - .build() - .toByteArray(); - } -} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java deleted file mode 100644 index cb6eb3678fc..00000000000 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogIngestionControllerTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; - -/** - * Unit test for {@link LogIngestionController} - */ -@ExtendWith(MockitoExtension.class) -class LogIngestionControllerTest { - - private MockMvc mockMvc; - - @Mock - private LogProtocolAdapter vectorAdapter; - - private LogIngestionController logIngestionController; - - @BeforeEach - void setUp() { - List adapters = Arrays.asList(vectorAdapter); - this.logIngestionController = new LogIngestionController(adapters); - this.mockMvc = MockMvcBuilders.standaloneSetup(logIngestionController).build(); - } - - @Test - void testIngestLogWithKnownProtocol() throws Exception { - String logContent = "{\"message\":\"Test log message\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - doNothing().when(vectorAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/vector") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) - .andExpect(jsonPath("$.msg").value("Add extern log success")) - .andReturn(); - } - - @Test - void testIngestLogWithUnsupportedProtocol() throws Exception { - String logContent = "{\"message\":\"Unsupported protocol log\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/unsupported") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) - .andExpect(jsonPath("$.msg").value("Not support the unsupported protocol log")); - } - - @Test - void testIngestLogWithAdapterException() throws Exception { - String logContent = "{\"message\":\"Log message that will cause exception\"}"; - - when(vectorAdapter.supportProtocol()).thenReturn("vector"); - doThrow(new IllegalArgumentException("Invalid log format")).when(vectorAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/ingest/vector") - .contentType(MediaType.APPLICATION_JSON) - .content(logContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) - .andExpect(jsonPath("$.msg").value("Add extern log failed: Invalid log format")); - } -} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java b/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java deleted file mode 100644 index d2e67bcb8ee..00000000000 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpLogControllerTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hertzbeat.log.controller; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; - -import org.apache.hertzbeat.log.service.impl.OtlpLogProtocolAdapter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; - -/** - * Unit test for {@link OtlpLogController} - */ -@ExtendWith(MockitoExtension.class) -class OtlpLogControllerTest { - - private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf"; - - private MockMvc mockMvc; - - @Mock - private OtlpLogProtocolAdapter otlpLogProtocolAdapter; - - private OtlpLogController otlpLogController; - - @BeforeEach - void setUp() { - this.otlpLogController = new OtlpLogController(otlpLogProtocolAdapter); - this.mockMvc = MockMvcBuilders.standaloneSetup(otlpLogController).build(); - } - - @Test - void testIngestJsonLogsSuccess() throws Exception { - String jsonContent = "{\"resourceLogs\":[]}"; - - doNothing().when(otlpLogProtocolAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(MediaType.APPLICATION_JSON) - .content(jsonContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - } - - @Test - void testIngestJsonLogsFailure() throws Exception { - String jsonContent = "{\"invalid\":\"content\"}"; - - doThrow(new IllegalArgumentException("Invalid OTLP JSON log content")) - .when(otlpLogProtocolAdapter).ingest(anyString()); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(MediaType.APPLICATION_JSON) - .content(jsonContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - } - - @Test - void testIngestBinaryLogsSuccess() throws Exception { - byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c}; - - doNothing().when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class)); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(CONTENT_TYPE_PROTOBUF) - .content(binaryContent) - ) - .andDo(print()) - .andExpect(status().isOk()) - .andExpect(content().contentType(CONTENT_TYPE_PROTOBUF)) - .andReturn(); - } - - @Test - void testIngestBinaryLogsFailure() throws Exception { - byte[] binaryContent = new byte[]{0x0a, 0x0b, 0x0c}; - - doThrow(new IllegalArgumentException("Invalid OTLP binary log content")) - .when(otlpLogProtocolAdapter).ingestBinary(any(byte[].class)); - - mockMvc.perform( - MockMvcRequestBuilders - .post("/api/logs/otlp/v1/logs") - .contentType(CONTENT_TYPE_PROTOBUF) - .content(binaryContent) - ) - .andExpect(status().isBadRequest()) - .andExpect(content().contentType(CONTENT_TYPE_PROTOBUF)) - .andReturn(); - } -} diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 57994eac2d9..f98eb08c521 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -91,10 +91,10 @@ org.apache.hertzbeat hertzbeat-otel - + org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java new file mode 100644 index 00000000000..8f1e40664b3 --- /dev/null +++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/RestTemplateConfigTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import org.springframework.web.client.RestTemplate; + +class RestTemplateConfigTest { + + @Test + void greptimeQueryRestTemplateAllowsColdQueryToComplete() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/query", exchange -> { + try { + Thread.sleep(5_500); + byte[] response = "ok".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, response.length); + exchange.getResponseBody().write(response); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + exchange.close(); + } + }); + server.start(); + + try { + RestTemplateConfig config = new RestTemplateConfig(); + RestTemplate restTemplate = config.greptimeQueryRestTemplate( + config.greptimeQueryClientHttpRequestFactory()); + + String response = restTemplate.getForObject( + "http://127.0.0.1:" + server.getAddress().getPort() + "/query", String.class); + + assertEquals("ok", response); + } finally { + server.stop(0); + } + } +} diff --git a/hertzbeat-manager/src/test/resources/sureness.yml b/hertzbeat-manager/src/test/resources/sureness.yml index 1763868f1e1..55e0eae065e 100644 --- a/hertzbeat-manager/src/test/resources/sureness.yml +++ b/hertzbeat-manager/src/test/resources/sureness.yml @@ -80,12 +80,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # config the resource restful api that need bypass auth protection # rule: api===method @@ -95,7 +92,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/hertzbeat-log/pom.xml b/hertzbeat-observability/pom.xml similarity index 98% rename from hertzbeat-log/pom.xml rename to hertzbeat-observability/pom.xml index c5c432aca5f..5d62967651d 100644 --- a/hertzbeat-log/pom.xml +++ b/hertzbeat-observability/pom.xml @@ -25,7 +25,7 @@ 4.0.0 - hertzbeat-log + hertzbeat-observability ${project.artifactId} diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java similarity index 72% rename from hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java index f58973b12dc..66e0a0c4dae 100644 --- a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/LogAutoConfiguration.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/ObservabilityAutoConfiguration.java @@ -15,21 +15,19 @@ * limitations under the License. */ -package org.apache.hertzbeat.otel.config; +package org.apache.hertzbeat.observability.config; import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.SignConstants; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; /** - * Log auto configuration. + * Entity-free observability module auto configuration. */ +@Configuration(proxyBeanMethods = false) @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG - + SignConstants.DOT - + ConfigConstants.FunctionModuleConstants.LOG -) -@EnableConfigurationProperties(GreptimeProperties.class) -public class LogAutoConfiguration { + + SignConstants.DOT + + ConfigConstants.FunctionModuleConstants.OBSERVABILITY) +public class ObservabilityAutoConfiguration { } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java similarity index 91% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java index 9e6d3bc067c..d98e4d8512d 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfig.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.observability.config; import io.grpc.ForwardingServerCallListener; import io.grpc.Metadata; @@ -41,10 +41,11 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -64,12 +65,14 @@ public OtlpGrpcServerRunner otlpGrpcServerRunner( @Value("${hertzbeat.otlp.grpc.host:0.0.0.0}") String host, @Value("${hertzbeat.otlp.grpc.port:4317}") int port, OtlpSignalForwarder signalForwarder, + OtlpLogIngestionService logIngestionService, SignalWorkloadGuard workloadGuard, ObjectProvider tokenValidatorProvider) { OtlpAccessTokenValidator validator = tokenValidatorProvider.getIfAvailable( () -> token -> "OTLP token validation is unavailable"); ServerInterceptor interceptor = new BearerTokenInterceptor(validator); - return new OtlpGrpcServerRunner(host, port, signalForwarder, workloadGuard, interceptor); + return new OtlpGrpcServerRunner(host, port, signalForwarder, logIngestionService, + workloadGuard, interceptor); } @Slf4j @@ -78,6 +81,7 @@ static final class OtlpGrpcServerRunner { private final String host; private final int port; private final OtlpSignalForwarder signalForwarder; + private final OtlpLogIngestionService logIngestionService; private final SignalWorkloadGuard workloadGuard; private final ServerInterceptor authInterceptor; private Server server; @@ -86,7 +90,7 @@ public void start() throws IOException { server = NettyServerBuilder.forAddress(new InetSocketAddress(host, port)) .addService(ServerInterceptors.intercept(new MetricsService(signalForwarder, workloadGuard), authInterceptor)) - .addService(ServerInterceptors.intercept(new LogsService(signalForwarder, workloadGuard), + .addService(ServerInterceptors.intercept(new LogsService(logIngestionService, workloadGuard), authInterceptor)) .addService(ServerInterceptors.intercept(new TracesService(signalForwarder, workloadGuard), authInterceptor)) @@ -123,14 +127,14 @@ public void export(ExportMetricsServiceRequest request, @RequiredArgsConstructor static final class LogsService extends LogsServiceGrpc.LogsServiceImplBase { - private final OtlpSignalForwarder signalForwarder; + private final OtlpLogIngestionService logIngestionService; private final SignalWorkloadGuard workloadGuard; @Override public void export(ExportLogsServiceRequest request, StreamObserver observer) { try { byte[] response = workloadGuard.execute(Workload.OTLP_WRITE, - () -> signalForwarder.forwardProtobuf("logs", request.toByteArray())); + () -> logIngestionService.ingestProtobuf(request.toByteArray())); observer.onNext(response.length == 0 ? ExportLogsServiceResponse.getDefaultInstance() : ExportLogsServiceResponse.parseFrom(response)); observer.onCompleted(); diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java index f552e5c20f1..8d9fbe506f7 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogManagerController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogManagerController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -37,7 +37,7 @@ * Controller for managing log entries in HertzBeat. */ @RestController -@RequestMapping(path = "/api/logs", produces = "application/json") +@RequestMapping(path = "/api/observability/logs", produces = "application/json") @Tag(name = "Log Management Controller") @Slf4j public class LogManagerController { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java similarity index 97% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java index 7cde958b437..a5e18c1c8cd 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogQueryController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogQueryController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -37,8 +37,8 @@ import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter; import org.apache.hertzbeat.common.entity.log.LogEntry; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -49,7 +49,7 @@ * Log query and statistics APIs for UI consumption */ @RestController -@RequestMapping(path = "/api/logs", produces = "application/json") +@RequestMapping(path = "/api/observability/logs", produces = "application/json") @Tag(name = "Log Query Controller") @Slf4j public class LogQueryController { @@ -67,7 +67,7 @@ public LogQueryController(HistoryDataReader historyDataReader, SignalWorkloadGua this.workloadGuard = workloadGuard; } - @GetMapping("/list") + @GetMapping @Operation(summary = "Query logs by time range with optional filters", description = "Query logs by [start,end] in ms and optional filters with pagination. Returns paginated log entries sorted by timestamp in descending order.") public ResponseEntity>> list( @@ -101,7 +101,7 @@ public ResponseEntity>> list( }); } - @GetMapping("/stats/overview") + @GetMapping("/overview") @Operation(summary = "Log overview statistics", description = "Overall counts and basic statistics with filters. Provides counts by severity levels according to OpenTelemetry standard.") public ResponseEntity>> overviewStats( @@ -132,7 +132,7 @@ private ResponseEntity>> doOverviewStats(LogQueryFil return ResponseEntity.ok(Message.success(historyDataReader.queryLogOverviewAggregate(filter))); } - @GetMapping("/stats/trace-coverage") + @GetMapping("/trace-coverage") @Operation(summary = "Trace coverage statistics", description = "Statistics about trace information availability. Shows how many logs have trace IDs, span IDs, or both for distributed tracing analysis.") public ResponseEntity>> traceCoverageStats( @@ -165,7 +165,7 @@ private ResponseEntity>> doTraceCoverageStats(LogQue return ResponseEntity.ok(Message.success(result)); } - @GetMapping("/stats/trend") + @GetMapping("/trend") @Operation(summary = "Log trend over time", description = "Count logs by hour intervals with filters. Groups logs by hour and provides time-series data for trend analysis.") public ResponseEntity>> trendStats( diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java similarity index 87% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java index e519ae06ee2..70956167616 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/LogSseController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/LogSseController.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.springframework.http.MediaType.TEXT_EVENT_STREAM_VALUE; import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator; -import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.RequestMapping; @@ -34,7 +34,7 @@ * SSE controller for log streaming with filtering support */ @RestController -@RequestMapping(path = "/api/logs/sse", produces = {TEXT_EVENT_STREAM_VALUE}) +@RequestMapping(path = "/api/observability/logs", produces = {TEXT_EVENT_STREAM_VALUE}) public class LogSseController { private final LogSseManager emitterManager; @@ -48,10 +48,10 @@ public LogSseController(LogSseManager emitterManager) { * @param filterCriteria Filter criteria for log events (all parameters are optional) * @return SSE emitter for streaming log events */ - @GetMapping(path = "/subscribe") + @GetMapping(path = "/stream") @Operation(summary = "Subscribe to log events with optional filtering", description = "Subscribe to log events with optional filtering") public SseEmitter subscribe(@ModelAttribute LogSseFilterCriteria filterCriteria) { Long clientId = SnowFlakeIdGenerator.generateId(); return emitterManager.createEmitter(clientId, filterCriteria); } -} \ No newline at end of file +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java similarity index 96% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java index 055167f4030..9943c2f69da 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityController.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.entity.dto.observability.ObservabilityCapability; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.springframework.beans.factory.ObjectProvider; import org.springframework.http.ResponseEntity; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java new file mode 100644 index 00000000000..8e031da9c8d --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.nio.charset.StandardCharsets; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** Canonical OTLP/HTTP log endpoint backed by the HertzBeat log fan-out. */ +@RestController +@RequestMapping("/api/otlp/v1") +@Tag(name = "OTLP Log Controller") +public class OtlpLogController { + + private final OtlpLogIngestionService logIngestionService; + private final SignalWorkloadGuard workloadGuard; + + public OtlpLogController(OtlpLogIngestionService logIngestionService, SignalWorkloadGuard workloadGuard) { + this.logIngestionService = logIngestionService; + this.workloadGuard = workloadGuard; + } + + @PostMapping("/logs") + @Operation(summary = "Ingest OTLP logs") + public ResponseEntity logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { + return workloadGuard.execute(Workload.OTLP_WRITE, + () -> logIngestionService.ingestHttp(content, headers)); + } + + @ExceptionHandler(IllegalArgumentException.class) + public ResponseEntity invalidPayload(IllegalArgumentException exception) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body(exception.getMessage().getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java similarity index 86% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java index a88e2d10eb5..fbca7708b1e 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/OtlpSignalController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/OtlpSignalController.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import java.nio.charset.StandardCharsets; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -55,12 +55,6 @@ public ResponseEntity metrics(@RequestBody byte[] content, @RequestHeade return forward("metrics", content, headers); } - @PostMapping("/logs") - @Operation(summary = "Ingest OTLP logs") - public ResponseEntity logs(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { - return forward("logs", content, headers); - } - @PostMapping("/traces") @Operation(summary = "Ingest OTLP traces") public ResponseEntity traces(@RequestBody byte[] content, @RequestHeader HttpHeaders headers) { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java similarity index 93% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java index 3cac628fc8a..80508a6fb21 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandler.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandler.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE; import org.apache.hertzbeat.common.entity.dto.Message; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -32,7 +32,7 @@ import org.springframework.web.client.RestClientException; /** Maps bounded signal workload rejection to a retryable HTTP response. */ -@RestControllerAdvice(basePackages = "org.apache.hertzbeat.log.controller") +@RestControllerAdvice(basePackages = "org.apache.hertzbeat.observability.controller") @Order(Ordered.HIGHEST_PRECEDENCE) public class SignalWorkloadExceptionHandler { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java similarity index 91% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java index de95425c17b..70be734dadd 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryController.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryController.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -26,19 +26,21 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceDetail; import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** Public entity-free query APIs for OTLP metrics and traces. */ @RestController @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") +@RequestMapping(path = "/api/observability", produces = "application/json") @Tag(name = "Three Signal Query Controller") public class ThreeSignalQueryController { @@ -50,7 +52,7 @@ public ThreeSignalQueryController(ThreeSignalQueryService queryService, SignalWo this.workloadGuard = workloadGuard; } - @GetMapping(path = "/api/ingestion/otlp/metrics/console", produces = "application/json") + @GetMapping("/metrics/query") @Operation(summary = "Query OTLP metrics") public ResponseEntity> metrics( @RequestParam(value = "query", required = false) String query, @@ -69,7 +71,7 @@ public ResponseEntity> metrics( filter, groupBy, aggregation, step, operationName)))); } - @GetMapping(path = "/api/ingestion/otlp/metrics/inventory", produces = "application/json") + @GetMapping("/metrics/inventory") @Operation(summary = "List OTLP metric names") public ResponseEntity> metricInventory( @RequestParam(value = "start", required = false) Long start, @@ -82,7 +84,7 @@ public ResponseEntity> metricInventory( queryService.metricInventory(start, end, serviceName, serviceNamespace, environment, limit)))); } - @GetMapping(path = "/api/traces/list", produces = "application/json") + @GetMapping("/traces") @Operation(summary = "Query traces") public ResponseEntity>> traces( @RequestParam(value = "start", required = false) Long start, @@ -102,7 +104,7 @@ public ResponseEntity>> traces( environment, operationName, minDurationMs, maxDurationMs, pageIndex, pageSize)))); } - @GetMapping(path = "/api/traces/stats/overview", produces = "application/json") + @GetMapping("/traces/overview") @Operation(summary = "Get trace overview") public ResponseEntity> traceOverview( @RequestParam(value = "start", required = false) Long start, @@ -120,14 +122,14 @@ public ResponseEntity> traceOverview( environment, operationName, minDurationMs, maxDurationMs)))); } - @GetMapping(path = "/api/traces/{traceId}", produces = "application/json") + @GetMapping("/traces/{traceId}") @Operation(summary = "Get trace detail") public ResponseEntity> traceDetail(@PathVariable("traceId") String traceId) { return workloadGuard.execute(Workload.TRACES, () -> ResponseEntity.ok(Message.success(queryService.traceDetail(traceId)))); } - @GetMapping(path = "/api/traces/{traceId}/spans", produces = "application/json") + @GetMapping("/traces/{traceId}/spans") @Operation(summary = "Get trace spans") public ResponseEntity> traceSpans(@PathVariable("traceId") String traceId) { return workloadGuard.execute(Workload.TRACES, diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java index b42f68aa8a8..6e476c1f3f7 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteria.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteria.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import lombok.Data; import lombok.AllArgsConstructor; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java similarity index 99% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java index 78d499cc45d..b8def56026f 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/notice/LogSseManager.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/notice/LogSseManager.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import jakarta.annotation.PreDestroy; import lombok.AllArgsConstructor; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java new file mode 100644 index 00000000000..c351fd34748 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpLogIngestionService.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +/** + * Canonical OTLP log ingestion boundary shared by the HTTP and gRPC transports. + */ +public interface OtlpLogIngestionService { + + /** + * Decode and ingest an OTLP/HTTP log request. + * + * @param content encoded OTLP request body + * @param headers request headers + * @return an OTLP response encoded for the request content type + */ + ResponseEntity ingestHttp(byte[] content, HttpHeaders headers); + + /** + * Decode and ingest an OTLP/gRPC protobuf request. + * + * @param content encoded {@code ExportLogsServiceRequest} + * @return encoded {@code ExportLogsServiceResponse} + */ + byte[] ingestProtobuf(byte[] content); +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java index b2305fafc3d..1cf555b8d05 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/OtlpSignalForwarder.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/OtlpSignalForwarder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java similarity index 95% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java index 40f43b1cbb3..7e1d9850078 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalQueryRejectedException.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalQueryRejectedException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; /** Raised when a signal workload exceeds its isolated concurrency budget. */ public class SignalQueryRejectedException extends RuntimeException { diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java index f60677ad151..0d2ada58dee 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/SignalWorkloadGuard.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuard.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import java.time.Duration; import java.util.EnumMap; diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java new file mode 100644 index 00000000000..4dbc8fef878 --- /dev/null +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionService.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service.impl; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.zip.GZIPInputStream; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +/** Default OTLP log ingestion service backed by the HertzBeat log fan-out. */ +@Service +public class DefaultOtlpLogIngestionService implements OtlpLogIngestionService { + + private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf"); + private static final byte[] JSON_RESPONSE = "{}".getBytes(StandardCharsets.UTF_8); + private static final byte[] PROTOBUF_RESPONSE = ExportLogsServiceResponse.getDefaultInstance().toByteArray(); + + private final OtlpLogProtocolAdapter protocolAdapter; + + public DefaultOtlpLogIngestionService(OtlpLogProtocolAdapter protocolAdapter) { + this.protocolAdapter = protocolAdapter; + } + + @Override + public ResponseEntity ingestHttp(byte[] content, HttpHeaders headers) { + HttpHeaders safeHeaders = headers == null ? HttpHeaders.EMPTY : headers; + byte[] normalizedContent = maybeDecompress(content, safeHeaders); + MediaType contentType = safeHeaders.getContentType(); + if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) { + protocolAdapter.ingest(new String(normalizedContent, StandardCharsets.UTF_8)); + return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(JSON_RESPONSE); + } + protocolAdapter.ingestBinary(normalizedContent); + return ResponseEntity.ok().contentType(PROTOBUF).body(PROTOBUF_RESPONSE); + } + + @Override + public byte[] ingestProtobuf(byte[] content) { + protocolAdapter.ingestBinary(content); + return PROTOBUF_RESPONSE; + } + + private byte[] maybeDecompress(byte[] content, HttpHeaders headers) { + byte[] safeContent = content == null ? new byte[0] : content; + List encodings = headers.get(HttpHeaders.CONTENT_ENCODING); + if (encodings == null || encodings.stream().noneMatch(this::containsGzipEncoding)) { + return safeContent; + } + try (GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(safeContent)); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + input.transferTo(output); + return output.toByteArray(); + } catch (IOException exception) { + throw new IllegalArgumentException("Malformed gzip OTLP log payload", exception); + } + } + + private boolean containsGzipEncoding(String value) { + if (value == null) { + return false; + } + for (String encoding : value.split(",")) { + if ("gzip".equalsIgnoreCase(encoding.trim())) { + return true; + } + } + return false; + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java similarity index 65% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java index 766c58f1eca..89ea162ab19 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarder.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; @@ -34,20 +34,15 @@ import java.util.Map; import java.util.Set; import java.util.zip.GZIPInputStream; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; import org.apache.hertzbeat.common.util.JsonUtil; -import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.springframework.web.client.RestTemplate; /** Greptime native OTLP forwarder for metrics, logs, and traces. */ @Service @@ -55,25 +50,11 @@ public class GreptimeOtlpSignalForwarder implements OtlpSignalForwarder { private static final String PROTOBUF = "application/x-protobuf"; - private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name"; - private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name"; - private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name"; - private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name"; - private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name"; - private static final String GREPTIME_PROMOTE_RESOURCE_HEADER = - "X-Greptime-OTLP-Metric-Promote-Resource-Attrs"; - private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of( - "service.name", "service.namespace", "service.version", "deployment.environment.name", - "host.name", "k8s.namespace.name", "k8s.pod.name")); private static final Set SIGNALS = Set.of("metrics", "logs", "traces"); - private final GreptimeProperties greptimeProperties; - private final RestTemplate restTemplate; + private final OtlpSignalStorage signalStorage; - public GreptimeOtlpSignalForwarder(GreptimeProperties greptimeProperties, - @Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE) - RestTemplate restTemplate) { - this.greptimeProperties = greptimeProperties; - this.restTemplate = restTemplate; + public GreptimeOtlpSignalForwarder(OtlpSignalStorage signalStorage) { + this.signalStorage = signalStorage; } @Override @@ -93,40 +74,8 @@ public ResponseEntity forwardHttp(String signal, byte[] content, HttpHea @Override public byte[] forwardProtobuf(String signal, byte[] content) { - String normalizedSignal = normalizeSignal(signal); - byte[] validContent = validateProtobuf(normalizedSignal, content); - HttpHeaders headers = greptimeHeaders(normalizedSignal); - ResponseEntity response = restTemplate.exchange( - endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal), - HttpMethod.POST, - new HttpEntity<>(validContent, headers), - byte[].class); - if (!response.getStatusCode().is2xxSuccessful()) { - throw new IllegalStateException("GreptimeDB rejected OTLP " + normalizedSignal); - } - return response.getBody() == null ? new byte[0] : response.getBody(); - } - - private HttpHeaders greptimeHeaders(String signal) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.parseMediaType(PROTOBUF)); - headers.setAccept(List.of(MediaType.parseMediaType(PROTOBUF))); - headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database()) - ? greptimeProperties.database() : "public"); - if ("metrics".equals(signal)) { - headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES); - } else if ("traces".equals(signal)) { - headers.set(GREPTIME_TRACE_TABLE_HEADER, "hzb_traces"); - headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1"); - } else { - headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME); - headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1"); - } - if (StringUtils.hasText(greptimeProperties.username()) && StringUtils.hasText(greptimeProperties.password())) { - String credentials = greptimeProperties.username() + ":" + greptimeProperties.password(); - headers.setBasicAuth(Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8))); - } - return headers; + byte[] validContent = validateProtobuf(signal, content); + return signalStorage.writeProtobuf(signal, validContent); } private byte[] jsonToProtobuf(String signal, byte[] content) { @@ -213,7 +162,4 @@ private String normalizeSignal(String signal) { return normalized; } - private String endpoint(String base, String path) { - return StringUtils.trimTrailingCharacter(base, '/') + path; - } } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java similarity index 79% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java rename to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java index 46978af182e..93db9e7e0b5 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapter.java +++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapter.java @@ -15,8 +15,11 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; @@ -29,14 +32,16 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.queue.CommonDataQueue; -import org.apache.hertzbeat.log.notice.LogSseManager; -import org.apache.hertzbeat.log.service.LogProtocolAdapter; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; +import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.Set; /** * Adapter for OpenTelemetry OTLP/HTTP log ingestion. @@ -46,9 +51,10 @@ */ @Slf4j @Service -public class OtlpLogProtocolAdapter implements LogProtocolAdapter { +public class OtlpLogProtocolAdapter { - private static final String PROTOCOL_NAME = "otlp"; + private static final Set OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId"); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final CommonDataQueue commonDataQueue; private final LogSseManager logSseManager; @@ -58,7 +64,6 @@ public OtlpLogProtocolAdapter(CommonDataQueue commonDataQueue, LogSseManager log this.logSseManager = logSseManager; } - @Override public void ingest(String content) { if (content == null || content.isEmpty()) { log.warn("Received empty OTLP JSON log payload - skip processing."); @@ -66,7 +71,7 @@ public void ingest(String content) { } ExportLogsServiceRequest.Builder builder = ExportLogsServiceRequest.newBuilder(); try { - JsonFormat.parser().ignoringUnknownFields().merge(content, builder); + JsonFormat.parser().ignoringUnknownFields().merge(normalizeOtlpJson(content), builder); ExportLogsServiceRequest request = builder.build(); processLogsRequest(request, "JSON"); } catch (InvalidProtocolBufferException e) { @@ -274,8 +279,58 @@ private String bytesToHex(byte[] bytes) { return hexString.toString(); } - @Override - public String supportProtocol() { - return PROTOCOL_NAME; + /** + * Accept the hexadecimal IDs emitted by the HertzBeat onboarding probe while preserving + * standard base64-encoded OTLP/JSON IDs. + */ + private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException { + try { + JsonNode root = OBJECT_MAPPER.readTree(content); + normalizeOtlpHexEncodedIds(root); + return OBJECT_MAPPER.writeValueAsString(root); + } catch (Exception exception) { + throw new InvalidProtocolBufferException("Failed to normalize OTLP JSON: " + exception.getMessage()); + } + } + + private void normalizeOtlpHexEncodedIds(JsonNode node) { + if (node == null) { + return; + } + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + objectNode.fieldNames().forEachRemaining(fieldName -> { + JsonNode child = objectNode.get(fieldName); + if (OTLP_HEX_ID_FIELDS.contains(fieldName) && child != null && child.isTextual()) { + String normalized = tryConvertHexToBase64(child.asText()); + if (normalized != null) { + objectNode.put(fieldName, normalized); + } + } else { + normalizeOtlpHexEncodedIds(child); + } + }); + return; + } + if (node.isArray()) { + node.forEach(this::normalizeOtlpHexEncodedIds); + } } -} \ No newline at end of file + + private String tryConvertHexToBase64(String value) { + if (value == null || value.isBlank() || (value.length() & 1) != 0) { + return null; + } + for (int i = 0; i < value.length(); i++) { + if (Character.digit(value.charAt(i), 16) < 0) { + return null; + } + } + try { + return Base64.getEncoder().encodeToString(HexFormat.of().parseHex(value)); + } catch (IllegalArgumentException exception) { + return null; + } + } + +} diff --git a/hertzbeat-observability/src/main/resources/META-INF/spring.factories b/hertzbeat-observability/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..076463ce82b --- /dev/null +++ b/hertzbeat-observability/src/main/resources/META-INF/spring.factories @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.apache.hertzbeat.observability.config.ObservabilityAutoConfiguration diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java similarity index 74% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java index 41245fe3914..d762cd37873 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/OtlpGrpcServerConfigTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfigTest.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.observability.config; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.grpc.ManagedChannel; @@ -33,15 +34,19 @@ import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.stub.MetadataUtils; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.hertzbeat.common.security.OtlpAccessTokenValidator; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,10 +55,12 @@ class OtlpGrpcServerConfigTest { private Server server; private ManagedChannel channel; + private OtlpLogIngestionService logIngestionService; @BeforeEach void setUp() throws Exception { OtlpSignalForwarder forwarder = mock(OtlpSignalForwarder.class); + logIngestionService = mock(OtlpLogIngestionService.class); SignalWorkloadGuard guard = mock(SignalWorkloadGuard.class); OtlpAccessTokenValidator validator = token -> "probe-token".equals(token) ? null : "Invalid token"; when(guard.execute(eq(SignalWorkloadGuard.Workload.OTLP_WRITE), any())).thenAnswer(invocation -> { @@ -61,10 +68,14 @@ void setUp() throws Exception { return action.get(); }); when(forwarder.forwardProtobuf(eq("metrics"), any())).thenReturn(new byte[0]); + when(logIngestionService.ingestProtobuf(any())).thenReturn(new byte[0]); server = NettyServerBuilder.forPort(0) .addService(ServerInterceptors.intercept( new OtlpGrpcServerConfig.MetricsService(forwarder, guard), new OtlpGrpcServerConfig.BearerTokenInterceptor(validator))) + .addService(ServerInterceptors.intercept( + new OtlpGrpcServerConfig.LogsService(logIngestionService, guard), + new OtlpGrpcServerConfig.BearerTokenInterceptor(validator))) .build().start(); channel = NettyChannelBuilder.forAddress("127.0.0.1", server.getPort()).usePlaintext().build(); } @@ -86,6 +97,18 @@ void shouldAcceptAuthorizedOtlpGrpcMetricsRequest() { assertThat(response).isEqualTo(ExportMetricsServiceResponse.getDefaultInstance()); } + @Test + void shouldRouteAuthorizedOtlpGrpcLogsThroughLogIngestionService() { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER), "Bearer probe-token"); + ExportLogsServiceResponse response = LogsServiceGrpc.newBlockingStub(channel) + .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers)) + .export(ExportLogsServiceRequest.getDefaultInstance()); + + assertThat(response).isEqualTo(ExportLogsServiceResponse.getDefaultInstance()); + verify(logIngestionService).ingestProtobuf(any()); + } + @Test void shouldRejectMissingOtlpGrpcBearerToken() { assertThatThrownBy(() -> MetricsServiceGrpc.newBlockingStub(channel) diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java similarity index 94% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java index 8bd4939e0b6..29e73e0db37 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogManagerControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogManagerControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.when; @@ -61,7 +61,7 @@ void testBatchDeleteLogsSuccess() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000") ) .andDo(print()) @@ -77,7 +77,7 @@ void testBatchDeleteLogsFailure() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000", "1734005477640000000") ) .andExpect(status().isOk()) @@ -91,7 +91,7 @@ void testBatchDeleteLogsWithSingleTimestamp() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .delete("/api/logs") + .delete("/api/observability/logs") .param("timeUnixNanos", "1734005477630000000") ) .andExpect(status().isOk()) diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java similarity index 91% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java index 8dd39de96d6..902ca5142ef 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogQueryControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogQueryControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -32,7 +32,7 @@ import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.log.LogEntry; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -69,8 +69,8 @@ void setUp() { void shouldReturnFriendlyFailureWhenLogQueryUnsupported() throws Exception { when(historyDataReader.supportsLogQuery()).thenReturn(false); - for (String path : List.of("/api/logs/list", "/api/logs/stats/overview", - "/api/logs/stats/trace-coverage", "/api/logs/stats/trend")) { + for (String path : List.of("/api/observability/logs", "/api/observability/logs/overview", + "/api/observability/logs/trace-coverage", "/api/observability/logs/trend")) { mockMvc.perform(MockMvcRequestBuilders.get(path)) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)) @@ -108,7 +108,7 @@ void testListLogsWithAllFilters() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/list") + .get("/api/observability/logs") .param("start", "1734005477000") .param("end", "1734005478000") .param("traceId", "trace123") @@ -144,7 +144,7 @@ void testListLogsWithoutFilters() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/list") + .get("/api/observability/logs") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -157,7 +157,7 @@ void shouldBoundLogPaginationBeforeReadingStorage() throws Exception { when(historyDataReader.countObservabilityLogs(any())).thenReturn(0L); when(historyDataReader.queryObservabilityLogs(any(), any(), any())).thenReturn(List.of()); - mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs") .param("pageIndex", "-1") .param("pageSize", "1000")) .andExpect(status().isOk()) @@ -172,7 +172,7 @@ void shouldReturnBadRequestForInvalidResourceExpression() throws Exception { when(historyDataReader.countObservabilityLogs(any())) .thenThrow(new IllegalArgumentException("Invalid resource filter expression")); - mockMvc.perform(MockMvcRequestBuilders.get("/api/logs/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/logs") .param("resource", "bad-expression")) .andExpect(status().isBadRequest()) .andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE)); @@ -186,7 +186,7 @@ void testOverviewStatsWithMixedSeverityLogs() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/overview") + .get("/api/observability/logs/overview") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -205,7 +205,7 @@ void testOverviewStatsWithTimeRange() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/overview") + .get("/api/observability/logs/overview") .param("start", "1734005477000") .param("end", "1734005478000") ) @@ -221,7 +221,7 @@ void testTraceCoverageStats() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trace-coverage") + .get("/api/observability/logs/trace-coverage") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -238,7 +238,7 @@ void testTrendStats() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trend") + .get("/api/observability/logs/trend") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) @@ -251,7 +251,7 @@ void testTrendStatsWithNullTimestamp() throws Exception { mockMvc.perform( MockMvcRequestBuilders - .get("/api/logs/stats/trend") + .get("/api/observability/logs/trend") ) .andExpect(status().isOk()) .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE)) diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java similarity index 92% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java index 5528ec4f92a..302e0e925ac 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/LogSseControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/LogSseControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; @@ -24,8 +24,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.hertzbeat.log.notice.LogSseFilterCriteria; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseFilterCriteria; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -64,7 +64,7 @@ void setUp() { @Test void testSubscribeWithoutFilters() throws Exception { // When: A request is made to the subscribe endpoint without any parameters - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().isOk()); @@ -86,7 +86,7 @@ void testSubscribeCommitsInitialEvent() throws Exception { .standaloneSetup(new LogSseController(realEmitterManager)) .build(); - MvcResult result = realMockMvc.perform(get("/api/logs/sse/subscribe") + MvcResult result = realMockMvc.perform(get("/api/observability/logs/stream") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().isOk()) .andExpect(request().asyncStarted()) @@ -109,7 +109,7 @@ void testSubscribeWithMultipleFilters() throws Exception { String spanId = "abcdef1234567890"; // When: A request is made with all filter parameters - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .param("severityText", severityText) .param("severityNumber", severityNumber) .param("traceId", traceId) @@ -130,7 +130,7 @@ void testSubscribeWithMultipleFilters() throws Exception { @Test void testSubscribeWithInvalidSeverityNumber() throws Exception { // When: A request is made with a non-integer value for severityNumber - mockMvc.perform(get("/api/logs/sse/subscribe") + mockMvc.perform(get("/api/observability/logs/stream") .param("severityNumber", "not-a-number") .accept(MediaType.TEXT_EVENT_STREAM_VALUE)) .andExpect(status().is4xxClientError()); diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java similarity index 96% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java index e406213d116..dd6b54cb85f 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ObservabilityCapabilityControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ObservabilityCapabilityControllerTest.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.apache.hertzbeat.common.constants.CommonConstants; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java new file mode 100644 index 00000000000..2d6aec81260 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogControllerTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.apache.hertzbeat.observability.service.OtlpLogIngestionService; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +/** OTLP/HTTP log route contract tests. */ +@ExtendWith(MockitoExtension.class) +class OtlpLogControllerTest { + + @Mock + private OtlpLogIngestionService logIngestionService; + + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + mockMvc = MockMvcBuilders.standaloneSetup( + new OtlpLogController(logIngestionService, new SignalWorkloadGuard())).build(); + } + + @Test + void shouldRouteLogsToTheLogFanOut() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isOk()); + + verify(logIngestionService).ingestHttp(any(), any(HttpHeaders.class)); + } + + @Test + void shouldReturnBadRequestForMalformedOtlpPayload() throws Exception { + when(logIngestionService.ingestHttp(any(), any())) + .thenThrow(new IllegalArgumentException("Malformed OTLP logs JSON payload")); + + mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/logs") + .contentType(MediaType.APPLICATION_JSON) + .content("{")) + .andExpect(status().isBadRequest()) + .andExpect(content().string("Malformed OTLP logs JSON payload")); + } +} diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java new file mode 100644 index 00000000000..1039b463127 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpLogIngestionOwnershipTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.controller; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +class OtlpLogIngestionOwnershipTest { + + @Test + void httpAndGrpcLogsShouldUseTheLogFanOutInsteadOfTheRawGreptimeForwarder() throws Exception { + String httpController = Files.readString(Path.of( + "src/main/java/org/apache/hertzbeat/observability/controller/OtlpLogController.java")); + String grpcConfig = Files.readString(Path.of( + "src/main/java/org/apache/hertzbeat/observability/config/OtlpGrpcServerConfig.java")); + + assertTrue(httpController.contains("OtlpLogIngestionService")); + assertTrue(httpController.contains("logIngestionService.ingestHttp(content, headers)")); + assertFalse(httpController.contains("forward(\"logs\"")); + + assertTrue(grpcConfig.contains("OtlpLogIngestionService")); + assertTrue(grpcConfig.contains("logIngestionService.ingestProtobuf(request.toByteArray())")); + assertFalse(grpcConfig.contains("forwardProtobuf(\"logs\"")); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java similarity index 88% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java index a3e48a14e04..e9936c9212c 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/OtlpSignalControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/OtlpSignalControllerTest.java @@ -15,17 +15,18 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.hertzbeat.log.service.OtlpSignalForwarder; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.OtlpSignalForwarder; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -54,17 +55,19 @@ void setUp() { } @Test - void shouldRouteAllThreeSignals() throws Exception { + void shouldRouteGreptimeSignals() throws Exception { when(signalForwarder.forwardHttp(any(), any(), any())) .thenReturn(ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body("{}".getBytes())); - for (String signal : new String[] {"metrics", "logs", "traces"}) { + for (String signal : new String[] {"metrics", "traces"}) { mockMvc.perform(MockMvcRequestBuilders.post("/api/otlp/v1/" + signal) .contentType(MediaType.APPLICATION_JSON) .content("{}")) .andExpect(status().isOk()); verify(signalForwarder).forwardHttp(eq(signal), any(), any(HttpHeaders.class)); } + + verifyNoMoreInteractions(signalForwarder); } @Test diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java similarity index 94% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java index 9e3a32fa6da..cd4c49bcf4a 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/SignalWorkloadExceptionHandlerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/SignalWorkloadExceptionHandlerTest.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import org.apache.hertzbeat.common.entity.dto.Message; -import org.apache.hertzbeat.log.service.SignalQueryRejectedException; +import org.apache.hertzbeat.observability.service.SignalQueryRejectedException; import org.junit.jupiter.api.Test; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java similarity index 91% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java index a98c9fdd2e3..260e6714741 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/controller/ThreeSignalQueryControllerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/controller/ThreeSignalQueryControllerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.controller; +package org.apache.hertzbeat.observability.controller; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -30,8 +30,8 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceListItem; import org.apache.hertzbeat.common.entity.dto.observability.TraceOverview; import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -62,7 +62,7 @@ void shouldQueryMetricsWithoutEntityContext() throws Exception { "prod", null, null, null, 60, null)) .thenReturn(new OtlpMetricsConsole("http_server_duration", 1000L, 2000L, 60, List.of())); - mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/console") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/query") .param("query", "http_server_duration") .param("start", "1000") .param("end", "2000") @@ -80,7 +80,7 @@ void shouldExposeMetricInventory() throws Exception { when(queryService.metricInventory(1000L, 2000L, "checkout", "payments", "prod", 50)) .thenReturn(new OtlpMetricsInventory(List.of("http_server_duration"))); - mockMvc.perform(MockMvcRequestBuilders.get("/api/ingestion/otlp/metrics/inventory") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/metrics/inventory") .param("start", "1000") .param("end", "2000") .param("serviceName", "checkout") @@ -102,7 +102,7 @@ void shouldQueryTraceListAndDetail() throws Exception { "SERVER", "failed", 10_000_000L, 1000L, Map.of(), Map.of(), List.of()); when(queryService.traceDetail("trace-1")).thenReturn(new TraceDetail(item, List.of(span))); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/list") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces") .param("start", "1000") .param("end", "2000") .param("traceId", "trace-1") @@ -116,7 +116,7 @@ void shouldQueryTraceListAndDetail() throws Exception { .andExpect(status().isOk()) .andExpect(jsonPath("$.data.content[0].traceId").value("trace-1")); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/trace-1")) + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/trace-1")) .andExpect(status().isOk()) .andExpect(jsonPath("$.data.spans[0].spanId").value("span-1")); } @@ -127,7 +127,7 @@ void shouldExposeTraceOverview() throws Exception { "GET /cart", 1L, 100L)) .thenReturn(new TraceOverview(4, 1, 0.25, 12.5, 24.0)); - mockMvc.perform(MockMvcRequestBuilders.get("/api/traces/stats/overview") + mockMvc.perform(MockMvcRequestBuilders.get("/api/observability/traces/overview") .param("start", "1000") .param("end", "2000") .param("traceId", "trace-1") diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java index b06284d9bae..1845e6bc5b1 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseFilterCriteriaTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseFilterCriteriaTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.junit.jupiter.api.BeforeEach; diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java index 231e8af8772..2616507a307 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/notice/LogSseManagerTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/notice/LogSseManagerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.hertzbeat.log.notice; +package org.apache.hertzbeat.observability.notice; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.junit.jupiter.api.AfterEach; diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java similarity index 92% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java index d8c92d7625b..d0c496af05d 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/SignalWorkloadGuardTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/SignalWorkloadGuardTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.observability.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -25,8 +25,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Limits; -import org.apache.hertzbeat.log.service.SignalWorkloadGuard.Workload; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Limits; +import org.apache.hertzbeat.observability.service.SignalWorkloadGuard.Workload; import org.junit.jupiter.api.Test; /** Tests bounded signal workload isolation and recovery. */ diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java new file mode 100644 index 00000000000..791bc616560 --- /dev/null +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/DefaultOtlpLogIngestionServiceTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.observability.service.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.verify; + +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +@ExtendWith(MockitoExtension.class) +class DefaultOtlpLogIngestionServiceTest { + + @Mock + private OtlpLogProtocolAdapter protocolAdapter; + + private DefaultOtlpLogIngestionService service; + + @BeforeEach + void setUp() { + service = new DefaultOtlpLogIngestionService(protocolAdapter); + } + + @Test + void shouldIngestJsonAndReturnJsonOtlpResponse() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + var response = service.ingestHttp("{}".getBytes(StandardCharsets.UTF_8), headers); + + verify(protocolAdapter).ingest("{}"); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); + assertThat(response.getBody()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void shouldIngestProtobufAndReturnProtobufOtlpResponse() { + byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + + var response = service.ingestHttp(request, headers); + + verify(protocolAdapter).ingestBinary(request); + assertThat(response.getHeaders().getContentType()) + .isEqualTo(MediaType.parseMediaType("application/x-protobuf")); + assertThat(response.getBody()).isEmpty(); + } + + @Test + void shouldDecompressCommaSeparatedGzipContentEncoding() throws Exception { + byte[] request = ExportLogsServiceRequest.getDefaultInstance().toByteArray(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + headers.add(HttpHeaders.CONTENT_ENCODING, "identity, gzip"); + + service.ingestHttp(gzip(request), headers); + + verify(protocolAdapter).ingestBinary(request); + } + + @Test + void shouldRejectMalformedGzipBeforeFanOut() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); + headers.set(HttpHeaders.CONTENT_ENCODING, "gzip"); + + assertThatThrownBy(() -> service.ingestHttp(new byte[] {1, 2, 3}, headers)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Malformed gzip OTLP log payload"); + } + + private byte[] gzip(byte[] content) throws Exception { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(output)) { + gzip.write(content); + gzip.finish(); + return output.toByteArray(); + } + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java similarity index 63% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java index 07b4cd18151..c5a03ed6ff9 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeOtlpSignalForwarderTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarderTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -28,33 +28,27 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import java.nio.charset.StandardCharsets; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestTemplate; /** Greptime native OTLP forwarding tests. */ @ExtendWith(MockitoExtension.class) class GreptimeOtlpSignalForwarderTest { @Mock - private RestTemplate restTemplate; + private OtlpSignalStorage signalStorage; @Test - void shouldConvertJsonAndForwardMetricsWithResourcePromotion() { - when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), - any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + void shouldConvertJsonBeforeWritingMetrics() { + when(signalStorage.writeProtobuf(eq("metrics"), any())).thenReturn(new byte[0]); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); @@ -62,21 +56,15 @@ void shouldConvertJsonAndForwardMetricsWithResourcePromotion() { headers); assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); - verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), - request.capture(), eq(byte[].class)); - assertThat(request.getValue().getHeaders().getFirst("X-Greptime-OTLP-Metric-Promote-Resource-Attrs")) - .contains("service.name", "deployment.environment.name"); - assertThat(request.getValue().getBody()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray()); + ArgumentCaptor content = ArgumentCaptor.forClass(byte[].class); + verify(signalStorage).writeProtobuf(eq("metrics"), content.capture()); + assertThat(content.getValue()).isEqualTo(ExportMetricsServiceRequest.getDefaultInstance().toByteArray()); } @Test void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception { - when(restTemplate.exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), - any(HttpEntity.class), eq(byte[].class))).thenReturn(ResponseEntity.ok(new byte[0])); - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + when(signalStorage.writeProtobuf(eq("traces"), any())).thenReturn(new byte[0]); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); String json = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{" @@ -86,10 +74,9 @@ void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception { forwarder.forwardHttp("traces", json.getBytes(StandardCharsets.UTF_8), headers); - ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); - verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), - request.capture(), eq(byte[].class)); - ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(request.getValue().getBody()); + ArgumentCaptor content = ArgumentCaptor.forClass(byte[].class); + verify(signalStorage).writeProtobuf(eq("traces"), content.capture()); + ExportTraceServiceRequest parsed = ExportTraceServiceRequest.parseFrom(content.getValue()); assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getTraceId().toByteArray()) .containsExactly(java.util.HexFormat.of().parseHex("0123456789abcdef0123456789abcdef")); assertThat(parsed.getResourceSpans(0).getScopeSpans(0).getSpans(0).getParentSpanId().toByteArray()) @@ -98,29 +85,25 @@ void shouldAcceptStandardHexTraceIdsInOtlpJson() throws Exception { @Test void shouldRejectMalformedJsonBeforeCallingGreptime() { - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); assertThatThrownBy(() -> forwarder.forwardHttp("metrics", "{".getBytes(StandardCharsets.UTF_8), headers)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Malformed OTLP JSON payload"); - verifyNoInteractions(restTemplate); + verifyNoInteractions(signalStorage); } @Test void shouldRejectMalformedProtobufBeforeCallingGreptime() { - GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder( - new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000", - "public", "greptime", "secret"), restTemplate); + GreptimeOtlpSignalForwarder forwarder = new GreptimeOtlpSignalForwarder(signalStorage); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.parseMediaType("application/x-protobuf")); assertThatThrownBy(() -> forwarder.forwardHttp("traces", new byte[] {(byte) 0xff}, headers)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Malformed OTLP traces protobuf payload"); - verifyNoInteractions(restTemplate); + verifyNoInteractions(signalStorage); } } diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java similarity index 94% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java rename to hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java index 0a570155bfd..bcd9b150e14 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/OtlpLogProtocolAdapterTest.java +++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/service/impl/OtlpLogProtocolAdapterTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.observability.service.impl; import com.google.protobuf.ByteString; import com.google.protobuf.util.JsonFormat; @@ -29,7 +29,7 @@ import io.opentelemetry.proto.common.v1.InstrumentationScope; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.queue.CommonDataQueue; -import org.apache.hertzbeat.log.notice.LogSseManager; +import org.apache.hertzbeat.observability.notice.LogSseManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -211,6 +211,31 @@ void testIngestWithTraceAndSpanIds() throws Exception { assertEquals(1, capturedEntry.getTraceFlags()); } + @Test + void testIngestWithHexEncodedTraceAndSpanIds() { + String otlpPayload = """ + { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "body": {"stringValue": "hex id probe"}, + "traceId": "1234567890abcdef1234567890abcdef", + "spanId": "1234567890abcdef" + }] + }] + }] + } + """; + + adapter.ingest(otlpPayload); + + ArgumentCaptor> listCaptor = ArgumentCaptor.forClass(List.class); + verify(commonDataQueue).sendLogEntryToStorageBatch(listCaptor.capture()); + LogEntry capturedEntry = listCaptor.getValue().getFirst(); + assertEquals("1234567890abcdef1234567890abcdef", capturedEntry.getTraceId()); + assertEquals("1234567890abcdef", capturedEntry.getSpanId()); + } + @Test void testIngestWithInvalidJsonContent() { String invalidJson = "{ invalid json content }"; diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java new file mode 100644 index 00000000000..61f145a7ff2 --- /dev/null +++ b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryAutoConfiguration.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.otel.config; + +import org.springframework.context.annotation.ComponentScan; + +/** + * HertzBeat self-telemetry auto configuration. + */ +@ComponentScan(basePackageClasses = OpenTelemetryConfig.class) +public class OpenTelemetryAutoConfiguration { +} diff --git a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java index 134de326215..1cebd09e56c 100644 --- a/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java +++ b/hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java @@ -51,8 +51,8 @@ public class OpenTelemetryConfig { private static final String HERTZBEAT_SERVICE_NAME = "HertzBeat"; private static final String DEFAULT_GREPTIME_DB_NAME = "public"; - private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_logs"; - private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_traces"; + private static final String DEFAULT_LOGS_TABLE_NAME = "hzb_internal_logs"; + private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_internal_traces"; private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name"; private static final String GREPTIME_LOG_TABLE_NAME_HEADER = "X-Greptime-Log-Table-Name"; private static final String GREPTIME_TRACE_TABLE_NAME_HEADER = "X-Greptime-Trace-Table-Name"; @@ -144,4 +144,4 @@ public AutoConfigurationCustomizerProvider greptimeOtelCustomizer(GreptimeProper return sdkLoggerProviderBuilder.addLogRecordProcessor(batchLogProcessor); }); } -} \ No newline at end of file +} diff --git a/hertzbeat-otel/src/main/resources/META-INF/spring.factories b/hertzbeat-otel/src/main/resources/META-INF/spring.factories index cb9d601c41b..246f2ad72ad 100644 --- a/hertzbeat-otel/src/main/resources/META-INF/spring.factories +++ b/hertzbeat-otel/src/main/resources/META-INF/spring.factories @@ -14,4 +14,4 @@ # limitations under the License. org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -org.apache.hertzbeat.otel.config.LogAutoConfiguration \ No newline at end of file +org.apache.hertzbeat.otel.config.OpenTelemetryAutoConfiguration diff --git a/hertzbeat-startup/src/main/resources/sureness.yml b/hertzbeat-startup/src/main/resources/sureness.yml index ee294024421..406dc1fd669 100644 --- a/hertzbeat-startup/src/main/resources/sureness.yml +++ b/hertzbeat-startup/src/main/resources/sureness.yml @@ -90,12 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] - /api/account/token===get===[admin] - /api/account/token/**===post===[admin] - /api/account/token/**===delete===[admin] @@ -110,7 +107,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java index 19401144d7b..c88cafe1940 100644 --- a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/ContextTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import jakarta.annotation.Resource; import org.apache.hertzbeat.alert.AlerterProperties; import org.apache.hertzbeat.alert.AlerterWorkerPool; @@ -43,8 +44,8 @@ import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue; import org.apache.hertzbeat.common.support.SpringContextHolder; import org.apache.hertzbeat.alert.service.impl.TencentSmsClientImpl; -import org.apache.hertzbeat.log.controller.OtlpSignalController; -import org.apache.hertzbeat.log.controller.ThreeSignalQueryController; +import org.apache.hertzbeat.observability.controller.OtlpSignalController; +import org.apache.hertzbeat.observability.controller.ThreeSignalQueryController; import org.apache.hertzbeat.warehouse.WarehouseWorkerPool; import org.apache.hertzbeat.warehouse.controller.MetricsDataController; import org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb.IotDbDataStorage; @@ -54,6 +55,8 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.context.ApplicationContext; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; /** * Manager Test @@ -113,4 +116,14 @@ void testAutoImport() { assertThrows(NoSuchBeanDefinitionException.class, () -> ctx.getBean(ThreeSignalQueryController.class)); } + @Test + void canonicalOtlpLogRouteShouldBeAvailableWithoutGreptime() { + RequestMappingHandlerMapping handlerMapping = ctx.getBean( + "requestMappingHandlerMapping", RequestMappingHandlerMapping.class); + + assertTrue(handlerMapping.getHandlerMethods().keySet().stream() + .anyMatch(mapping -> mapping.getPatternValues().contains("/api/otlp/v1/logs") + && mapping.getMethodsCondition().getMethods().contains(RequestMethod.POST))); + } + } diff --git a/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java new file mode 100644 index 00000000000..2af52a5161e --- /dev/null +++ b/hertzbeat-startup/src/test/java/org/apache/hertzbeat/startup/EntityFreeObservabilityTransitionContractTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.startup; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +class EntityFreeObservabilityTransitionContractTest { + + private static final Path REPOSITORY_ROOT = repositoryRoot(); + + @Test + void reactorAndRuntimeShouldOwnOneObservabilityModule() throws IOException { + String rootPom = Files.readString(REPOSITORY_ROOT.resolve("pom.xml")); + String managerPom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-manager/pom.xml")); + + assertTrue(rootPom.contains("hertzbeat-observability")); + assertFalse(rootPom.contains("hertzbeat-log")); + assertTrue(managerPom.contains("hertzbeat-observability")); + assertFalse(managerPom.contains("hertzbeat-log")); + assertFalse(Files.exists(REPOSITORY_ROOT.resolve("hertzbeat-log"))); + } + + @Test + void productionSourcesShouldExposeOnlyTheCanonicalEntityFreeContract() throws IOException { + Path sourceRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main/java"); + assertTrue(Files.isDirectory(sourceRoot)); + + String productionSources; + try (Stream sources = Files.walk(sourceRoot)) { + productionSources = sources + .filter(path -> path.toString().endsWith(".java")) + .map(EntityFreeObservabilityTransitionContractTest::readSource) + .reduce("", (left, right) -> left + '\n' + right); + } + + assertTrue(productionSources.contains("/api/otlp/v1")); + assertTrue(productionSources.contains("/api/observability")); + assertFalse(productionSources.contains("/api/logs/otlp")); + assertFalse(productionSources.contains("/api/logs/ingest")); + assertFalse(productionSources.contains("/api/logs")); + assertFalse(productionSources.contains("/api/ingestion/otlp")); + assertFalse(productionSources.contains("/api/traces")); + assertFalse(productionSources.contains("org.apache.hertzbeat.manager.service.entity")); + assertFalse(productionSources.contains("HERTZBEAT_ENTITY_ID")); + assertFalse(productionSources.contains("HERTZBEAT_ENTITY_TYPE")); + assertFalse(productionSources.contains("EntityObservability")); + assertFalse(productionSources.contains("EntityTrace")); + assertFalse(productionSources.contains("OtlpEntity")); + } + + @Test + void warehouseShouldOwnGreptimeSignalStorageSchemaAndQueries() throws IOException { + Path warehouseRoot = REPOSITORY_ROOT.resolve("hertzbeat-warehouse/src/main"); + Path observabilityRoot = REPOSITORY_ROOT.resolve("hertzbeat-observability/src/main"); + + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve("resources/greptime/tables/hertzbeat_traces.sql"))); + assertTrue(Files.isRegularFile(warehouseRoot.resolve( + "resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml"))); + + assertFalse(Files.exists(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/config/GreptimeSignalInitializer.java"))); + assertFalse(Files.exists(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/service/impl/GreptimeThreeSignalQueryService.java"))); + assertFalse(Files.exists(observabilityRoot.resolve("resources/greptime"))); + + String warehousePom = Files.readString(REPOSITORY_ROOT.resolve("hertzbeat-warehouse/pom.xml")); + assertFalse(warehousePom.contains("hertzbeat-observability")); + String observabilityForwarder = Files.readString(observabilityRoot.resolve( + "java/org/apache/hertzbeat/observability/service/impl/GreptimeOtlpSignalForwarder.java")); + assertTrue(observabilityForwarder.contains("OtlpSignalStorage")); + assertFalse(observabilityForwarder.contains("RestTemplate")); + } + + @Test + void currentSecurityConfigurationsShouldProtectOnlyCanonicalObservabilityRoutes() throws IOException { + for (String relativePath : new String[] { + "hertzbeat-startup/src/main/resources/sureness.yml", + "hertzbeat-manager/src/test/resources/sureness.yml", + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml", + "script/sureness.yml", + "script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml", + "script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml", + "script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml" + }) { + String securityConfig = Files.readString(REPOSITORY_ROOT.resolve(relativePath)); + assertTrue(securityConfig.contains("/api/otlp/v1/**===post===[admin,user]"), relativePath); + assertTrue(securityConfig.contains("/api/observability/**===get===[admin,user,guest]"), relativePath); + assertTrue(securityConfig.contains("/api/observability/logs===delete===[admin]"), relativePath); + assertFalse(securityConfig.contains("/api/logs/"), relativePath); + assertFalse(securityConfig.contains("/api/ingestion/otlp"), relativePath); + assertFalse(securityConfig.contains("/api/traces/"), relativePath); + assertFalse(securityConfig.contains("/api/otlp/**"), relativePath); + assertFalse(securityConfig.contains("/api/observability/capability===get"), relativePath); + } + } + + @Test + void productAndSelfTelemetryShouldUseSeparateSignalTables() throws IOException { + String traceSchema = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql")); + assertTrue(traceSchema.contains("CREATE TABLE IF NOT EXISTS hertzbeat_traces")); + assertFalse(traceSchema.contains("CREATE TABLE IF NOT EXISTS hzb_traces")); + + String selfTelemetry = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-otel/src/main/java/org/apache/hertzbeat/otel/config/OpenTelemetryConfig.java")); + assertTrue(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hzb_internal_logs\"")); + assertTrue(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hzb_internal_traces\"")); + assertFalse(selfTelemetry.contains("DEFAULT_LOGS_TABLE_NAME = \"hertzbeat_logs\"")); + assertFalse(selfTelemetry.contains("DEFAULT_TRACES_TABLE_NAME = \"hertzbeat_traces\"")); + } + + @Test + void currentClientsDocsAndProbesShouldUseCanonicalObservabilityRoutes() throws IOException { + for (String relativePath : new String[] { + "web-app/src/app/service/log.service.ts", + "web-app/src/app/service/observability.service.ts", + "web-app/src/app/routes/log/log-stream/log-stream.component.ts", + "web-app/src/assets/doc/log-integration/otlp.en-US.md", + "web-app/src/assets/doc/log-integration/otlp.zh-CN.md", + "home/docs/help/log_integration.md", + "home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md", + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml" + }) { + String content = Files.readString(REPOSITORY_ROOT.resolve(relativePath)); + assertFalse(content.contains("/api/logs"), relativePath); + assertFalse(content.contains("/logs/list"), relativePath); + assertFalse(content.contains("/logs/stats"), relativePath); + assertFalse(content.contains("/ingestion/otlp/metrics"), relativePath); + assertFalse(content.contains("/traces/list"), relativePath); + assertFalse(content.contains("/traces/stats"), relativePath); + } + + String logService = Files.readString(REPOSITORY_ROOT.resolve("web-app/src/app/service/log.service.ts")); + assertTrue(logService.contains("/observability/logs")); + String observabilityService = Files.readString( + REPOSITORY_ROOT.resolve("web-app/src/app/service/observability.service.ts")); + assertTrue(observabilityService.contains("/observability/metrics/query")); + assertTrue(observabilityService.contains("/observability/traces")); + String streamComponent = Files.readString(REPOSITORY_ROOT.resolve( + "web-app/src/app/routes/log/log-stream/log-stream.component.ts")); + assertTrue(streamComponent.contains("/api/observability/logs/stream")); + String vectorConfig = Files.readString(REPOSITORY_ROOT.resolve( + "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/vector.yml")); + assertTrue(vectorConfig.contains("/api/otlp/v1/logs")); + } + + private static String readSource(Path path) { + try { + return Files.readString(path); + } catch (IOException exception) { + throw new IllegalStateException("Failed to read " + path, exception); + } + } + + private static Path repositoryRoot() { + Path candidate = Path.of("").toAbsolutePath(); + while (candidate != null && !Files.isRegularFile(candidate.resolve("mvnw"))) { + candidate = candidate.getParent(); + } + if (candidate == null) { + throw new IllegalStateException("Unable to locate repository root"); + } + return candidate; + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java index ff8ef5ca20d..3ec129a4e9f 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/config/WarehouseAutoConfiguration.java @@ -19,7 +19,9 @@ import org.apache.hertzbeat.common.constants.ConfigConstants; import org.apache.hertzbeat.common.constants.SignConstants; +import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; /** @@ -31,5 +33,6 @@ @ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG + SignConstants.DOT + ConfigConstants.FunctionModuleConstants.WAREHOUSE) +@EnableConfigurationProperties(GreptimeProperties.class) public class WarehouseAutoConfiguration { } diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java similarity index 56% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java index a33e861c001..a9d27b88616 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/LogProtocolAdapter.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/OtlpSignalStorage.java @@ -15,27 +15,17 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.warehouse.service; -/** - * Adapter interface for ingesting logs pushed via different protocols - * (e.g. OTLP, Loki, Filebeat, Vector). - * Implementations should: - * 1. Parse raw HTTP payload of their protocol. - * 2. Convert data to LogEntry. - * 3. Forward / persist it to downstream pipeline. - */ -public interface LogProtocolAdapter { +/** Storage boundary for validated OTLP protobuf signals. */ +public interface OtlpSignalStorage { /** - * Ingest log payload pushed from external system. + * Persist one validated OTLP export request. * - * @param content raw request body string - */ - void ingest(String content); - - /** - * Identifier of the protocol this adapter supports ("otlp", "vector", etc.) + * @param signal metrics, logs, or traces + * @param content encoded OTLP protobuf request + * @return encoded OTLP protobuf response */ - String supportProtocol(); -} \ No newline at end of file + byte[] writeProtobuf(String signal, byte[] content); +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java similarity index 98% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java index 5b5409f252e..8c1f0fb4a01 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/ThreeSignalQueryService.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/ThreeSignalQueryService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service; +package org.apache.hertzbeat.warehouse.service; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsConsole; import org.apache.hertzbeat.common.entity.dto.observability.OtlpMetricsInventory; diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java similarity index 99% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java index dd85d7d2c57..fd9f04b5a47 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryService.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.warehouse.service.impl; import java.net.URI; import java.time.Instant; @@ -40,9 +40,9 @@ import org.apache.hertzbeat.common.entity.dto.observability.TraceSpanNode; import org.apache.hertzbeat.common.util.JsonUtil; import org.apache.hertzbeat.common.support.exception.StorageUnavailableException; -import org.apache.hertzbeat.log.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; +import org.apache.hertzbeat.warehouse.service.ThreeSignalQueryService; import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.beans.factory.annotation.Qualifier; @@ -63,7 +63,7 @@ @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeThreeSignalQueryService implements ThreeSignalQueryService { - private static final String TRACE_TABLE = "hzb_traces"; + private static final String TRACE_TABLE = "hertzbeat_traces"; private static final Pattern SAFE_IDENTIFIER = Pattern.compile("[A-Za-z_:][A-Za-z0-9_:.-]*"); private static final Pattern SAFE_LABEL = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); private static final Set AGGREGATIONS = Set.of("sum", "avg", "min", "max", "count"); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java index d0132de5b14..5ccbe4b84a5 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorage.java @@ -101,6 +101,10 @@ public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { private static final String LABEL_KEY_END_TIME = "end"; private static final String LABEL_KEY_TS = "ts"; private static final int LOG_BATCH_SIZE = 500; + private static final Map OTLP_RESOURCE_KEY_ALIASES = Map.of( + "service_name", "service.name", + "service_namespace", "service.namespace", + "deployment_environment_name", "deployment.environment.name"); private GreptimeDB greptimeDb; @@ -756,7 +760,7 @@ private void buildObservabilityWhereConditions(StringBuilder sql, LogQueryFilter addJsonCondition(resourceConditions, "deployment.environment.name", filter.environment()); if (StringUtils.hasText(filter.resourceFilter())) { for (ResourceFilterExpression.Clause clause : ResourceFilterExpression.parse(filter.resourceFilter())) { - String attribute = "json_get_string(resource, '$[\"" + clause.key() + "\"]')"; + String attribute = resourceAttributeExpression(clause.key()); switch (clause.operator()) { case EQUALS -> resourceConditions.add(attribute + " = '" + safeString(clause.value()) + "'"); case NOT_EQUALS -> resourceConditions.add(attribute + " <> '" + safeString(clause.value()) + "'"); @@ -774,10 +778,20 @@ private void buildObservabilityWhereConditions(StringBuilder sql, LogQueryFilter private void addJsonCondition(List conditions, String key, String value) { if (StringUtils.hasText(value)) { - conditions.add("json_get_string(resource, '$[\"" + key + "\"]') = '" + safeString(value) + "'"); + conditions.add(resourceAttributeExpression(key) + " = '" + safeString(value) + "'"); } } + private String resourceAttributeExpression(String key) { + String normalizedKey = key.replace('.', '_'); + String canonical = "json_get_string(resource, '$[\"" + key + "\"]')"; + if (normalizedKey.equals(key)) { + return canonical; + } + String normalized = "json_get_string(resource, '$[\"" + normalizedKey + "\"]')"; + return "COALESCE(" + canonical + ", " + normalized + ")"; + } + private static long msToNs(Long ms) { return ms * 1_000_000L; } @@ -857,7 +871,8 @@ private List mapRowsToLogEntries(List> rows) { Object bodyObj = parseJsonMaybe(row.get("body")); Map attributes = castToMap(parseJsonMaybe(row.get("attributes"))); - Map resource = castToMap(parseJsonMaybe(row.get("resource"))); + Map resource = canonicalizeOtlpResource( + castToMap(parseJsonMaybe(row.get("resource")))); LogEntry entry = LogEntry.builder() .timeUnixNano(castToLong(row.get("time_unix_nano"))) @@ -913,6 +928,19 @@ private static Map castToMap(Object obj) { return null; } + private static Map canonicalizeOtlpResource(Map resource) { + if (resource == null || resource.isEmpty()) { + return resource; + } + Map canonical = new LinkedHashMap<>(resource); + OTLP_RESOURCE_KEY_ALIASES.forEach((normalized, dotted) -> { + if (!canonical.containsKey(dotted) && resource.containsKey(normalized)) { + canonical.put(dotted, resource.get(normalized)); + } + }); + return canonical; + } + private static Long castToLong(Object obj) { if (obj == null) return null; if (obj instanceof Number n) return n.longValue(); diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java new file mode 100644 index 00000000000..96a4c920954 --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorage.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; +import org.apache.hertzbeat.warehouse.service.OtlpSignalStorage; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; + +/** GreptimeDB storage implementation for validated OTLP protobuf requests. */ +@Service +@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") +public class GreptimeOtlpSignalStorage implements OtlpSignalStorage { + + private static final MediaType PROTOBUF = MediaType.parseMediaType("application/x-protobuf"); + private static final String GREPTIME_DATABASE_HEADER = "X-Greptime-DB-Name"; + private static final String GREPTIME_TRACE_TABLE_HEADER = "X-Greptime-Trace-Table-Name"; + private static final String GREPTIME_PIPELINE_HEADER = "X-Greptime-Pipeline-Name"; + private static final String GREPTIME_LOG_TABLE_HEADER = "X-Greptime-Log-Table-Name"; + private static final String GREPTIME_LOG_PIPELINE_HEADER = "X-Greptime-Log-Pipeline-Name"; + private static final String GREPTIME_PROMOTE_RESOURCE_HEADER = + "X-Greptime-OTLP-Metric-Promote-Resource-Attrs"; + private static final String PROMOTED_RESOURCE_ATTRIBUTES = String.join(";", List.of( + "service.name", "service.namespace", "service.version", "deployment.environment.name", + "host.name", "k8s.namespace.name", "k8s.pod.name")); + private static final Set SIGNALS = Set.of("metrics", "logs", "traces"); + + private final GreptimeProperties greptimeProperties; + private final RestTemplate restTemplate; + + public GreptimeOtlpSignalStorage(GreptimeProperties greptimeProperties, + @Qualifier(WarehouseConstants.GREPTIME_WRITE_REST_TEMPLATE) + RestTemplate restTemplate) { + this.greptimeProperties = greptimeProperties; + this.restTemplate = restTemplate; + } + + @Override + public byte[] writeProtobuf(String signal, byte[] content) { + String normalizedSignal = normalizeSignal(signal); + HttpHeaders headers = greptimeHeaders(normalizedSignal); + ResponseEntity response = restTemplate.exchange( + endpoint(greptimeProperties.httpEndpoint(), "/v1/otlp/v1/" + normalizedSignal), + HttpMethod.POST, + new HttpEntity<>(content == null ? new byte[0] : content, headers), + byte[].class); + if (!response.getStatusCode().is2xxSuccessful()) { + throw new IllegalStateException("GreptimeDB rejected OTLP " + normalizedSignal); + } + return response.getBody() == null ? new byte[0] : response.getBody(); + } + + private HttpHeaders greptimeHeaders(String signal) { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(PROTOBUF); + headers.setAccept(List.of(PROTOBUF)); + headers.set(GREPTIME_DATABASE_HEADER, StringUtils.hasText(greptimeProperties.database()) + ? greptimeProperties.database() : "public"); + if ("metrics".equals(signal)) { + headers.set(GREPTIME_PROMOTE_RESOURCE_HEADER, PROMOTED_RESOURCE_ATTRIBUTES); + } else if ("traces".equals(signal)) { + headers.set(GREPTIME_TRACE_TABLE_HEADER, "hertzbeat_traces"); + headers.set(GREPTIME_PIPELINE_HEADER, "greptime_trace_v1"); + } else { + headers.set(GREPTIME_LOG_TABLE_HEADER, WarehouseConstants.LOG_TABLE_NAME); + headers.set(GREPTIME_LOG_PIPELINE_HEADER, "hertzbeat_otlp_log_v1"); + } + if (StringUtils.hasText(greptimeProperties.username()) + && StringUtils.hasText(greptimeProperties.password())) { + headers.setBasicAuth(greptimeProperties.username(), greptimeProperties.password(), + StandardCharsets.UTF_8); + } + return headers; + } + + private String normalizeSignal(String signal) { + String normalized = StringUtils.hasText(signal) ? signal.toLowerCase(Locale.ROOT) : ""; + if (!SIGNALS.contains(normalized)) { + throw new IllegalArgumentException("Unsupported OTLP signal"); + } + return normalized; + } + + private String endpoint(String base, String path) { + return StringUtils.trimTrailingCharacter(base, '/') + path; + } +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java similarity index 94% rename from hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java rename to hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java index 24c87254570..2cfb09c991c 100644 --- a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializer.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializer.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import java.nio.charset.StandardCharsets; import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -43,7 +42,7 @@ @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") public class GreptimeSignalInitializer { - private static final String TRACE_SCHEMA = "greptime/tables/hzb_traces.sql"; + private static final String TRACE_SCHEMA = "greptime/tables/hertzbeat_traces.sql"; private static final String LOG_SCHEMA = "greptime/tables/hertzbeat_logs.sql"; private static final String LOG_PIPELINE = "greptime/pipelines/hertzbeat_otlp_log_v1.yaml"; private final GreptimeProperties greptimeProperties; @@ -65,9 +64,9 @@ public void initialize() { String traceSchema = new ClassPathResource(TRACE_SCHEMA) .getContentAsString(StandardCharsets.UTF_8).strip(); sqlQueryExecutor.execute(StringUtils.trimTrailingCharacter(traceSchema, ';')); - sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " + sqlQueryExecutor.execute("ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS " + "\"resource_attributes.service.namespace\" STRING NULL"); - sqlQueryExecutor.execute("ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS " + sqlQueryExecutor.execute("ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS " + "\"resource_attributes.deployment.environment.name\" STRING NULL"); String logSchema = new ClassPathResource(LOG_SCHEMA) .getContentAsString(StandardCharsets.UTF_8).strip(); diff --git a/hertzbeat-log/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml b/hertzbeat-warehouse/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml similarity index 100% rename from hertzbeat-log/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml rename to hertzbeat-warehouse/src/main/resources/greptime/pipelines/hertzbeat_otlp_log_v1.yaml diff --git a/hertzbeat-log/src/main/resources/greptime/tables/hertzbeat_logs.sql b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_logs.sql similarity index 100% rename from hertzbeat-log/src/main/resources/greptime/tables/hertzbeat_logs.sql rename to hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_logs.sql diff --git a/hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql similarity index 97% rename from hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql rename to hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql index c0d62a0a445..0a1fdeae4b1 100644 --- a/hertzbeat-log/src/main/resources/greptime/tables/hzb_traces.sql +++ b/hertzbeat-warehouse/src/main/resources/greptime/tables/hertzbeat_traces.sql @@ -13,7 +13,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -CREATE TABLE IF NOT EXISTS hzb_traces ( +CREATE TABLE IF NOT EXISTS hertzbeat_traces ( "timestamp" TIMESTAMP(9) TIME INDEX, "timestamp_end" TIMESTAMP(9) NULL, "duration_nano" BIGINT UNSIGNED NULL, diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java similarity index 99% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java index 5a377ac1969..563091104b7 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/service/impl/GreptimeThreeSignalQueryServiceTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/service/impl/GreptimeThreeSignalQueryServiceTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.service.impl; +package org.apache.hertzbeat.warehouse.service.impl; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java index 3de76446b6c..91e4ccac623 100644 --- a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeDbDataStorageTest.java @@ -53,6 +53,7 @@ import org.apache.hertzbeat.common.entity.arrow.ArrowCell; import org.apache.hertzbeat.common.entity.arrow.RowWrapper; import org.apache.hertzbeat.common.entity.dto.Value; +import org.apache.hertzbeat.common.entity.dto.observability.LogQueryFilter; import org.apache.hertzbeat.common.entity.log.LogEntry; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; @@ -296,6 +297,38 @@ void testQueryLogsWithPagination() { } } + @Test + void shouldQueryNormalizedOtlpResourceKeysAndRestoreCanonicalNames() { + try (MockedStatic mockedStatic = mockStatic(GreptimeDB.class)) { + mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb); + greptimeDbDataStorage = new GreptimeDbDataStorage( + greptimeProperties, restTemplate, greptimeSqlQueryExecutor); + Map row = new HashMap<>(); + row.put("time_unix_nano", System.nanoTime()); + row.put("severity_text", "INFO"); + row.put("body", "OTLP resource proof"); + row.put("resource", Map.of( + "service_name", "checkout-api", + "service_namespace", "storefront", + "deployment_environment_name", "preview")); + when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(List.of(row)); + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + + List result = greptimeDbDataStorage.queryObservabilityLogs( + new LogQueryFilter(null, null, null, null, null, null, null, + "checkout-api", "storefront", "preview", null), + 0, 20); + + verify(greptimeSqlQueryExecutor).execute(sqlCaptor.capture()); + assertTrue(sqlCaptor.getValue().contains("service_name")); + assertTrue(sqlCaptor.getValue().contains("service_namespace")); + assertTrue(sqlCaptor.getValue().contains("deployment_environment_name")); + assertEquals("checkout-api", result.getFirst().getResource().get("service.name")); + assertEquals("storefront", result.getFirst().getResource().get("service.namespace")); + assertEquals("preview", result.getFirst().getResource().get("deployment.environment.name")); + } + } + @Test void testBatchDeleteLogsWithValidList() { diff --git a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java new file mode 100644 index 00000000000..58ef091ae56 --- /dev/null +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeOtlpSignalStorageTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +@ExtendWith(MockitoExtension.class) +class GreptimeOtlpSignalStorageTest { + + @Mock + private RestTemplate restTemplate; + + private GreptimeOtlpSignalStorage storage; + + @BeforeEach + void setUp() { + storage = new GreptimeOtlpSignalStorage( + new GreptimeProperties(true, "127.0.0.1:4001", "http://127.0.0.1:4000/", + "public", "greptime", "secret"), + restTemplate); + } + + @Test + void shouldWriteMetricsWithResourcePromotionAndAuthentication() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenReturn(ResponseEntity.ok("response".getBytes(StandardCharsets.UTF_8))); + + byte[] response = storage.writeProtobuf("metrics", new byte[] {1, 2}); + + ArgumentCaptor> request = ArgumentCaptor.forClass(HttpEntity.class); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/metrics"), eq(HttpMethod.POST), + request.capture(), eq(byte[].class)); + assertThat(request.getValue().getHeaders().getFirst( + "X-Greptime-OTLP-Metric-Promote-Resource-Attrs")) + .contains("service.name", "deployment.environment.name"); + assertThat(request.getValue().getHeaders().getFirst("Authorization")).startsWith("Basic "); + assertThat(request.getValue().getBody()).containsExactly(1, 2); + assertThat(response).isEqualTo("response".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void shouldSelectWarehouseOwnedTraceAndLogSchemas() { + when(restTemplate.exchange(any(String.class), eq(HttpMethod.POST), any(HttpEntity.class), eq(byte[].class))) + .thenReturn(ResponseEntity.ok(new byte[0])); + + storage.writeProtobuf("traces", new byte[0]); + storage.writeProtobuf("logs", new byte[0]); + + ArgumentCaptor> requests = ArgumentCaptor.forClass(HttpEntity.class); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/traces"), eq(HttpMethod.POST), + requests.capture(), eq(byte[].class)); + verify(restTemplate).exchange(eq("http://127.0.0.1:4000/v1/otlp/v1/logs"), eq(HttpMethod.POST), + requests.capture(), eq(byte[].class)); + assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Trace-Table-Name")) + .isEqualTo("hertzbeat_traces"); + assertThat(requests.getAllValues().get(0).getHeaders().getFirst("X-Greptime-Pipeline-Name")) + .isEqualTo("greptime_trace_v1"); + assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Table-Name")) + .isEqualTo("hertzbeat_logs"); + assertThat(requests.getAllValues().get(1).getHeaders().getFirst("X-Greptime-Log-Pipeline-Name")) + .isEqualTo("hertzbeat_otlp_log_v1"); + } + + @Test + void shouldRejectUnsupportedSignalBeforeStorageCall() { + assertThatThrownBy(() -> storage.writeProtobuf("profiles", new byte[0])) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported OTLP signal"); + } +} diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java similarity index 90% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java index 1d21fc7a630..7c9c1b65a85 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalInitializerTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalInitializerTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -28,7 +28,6 @@ import java.util.Map; import org.mockito.InOrder; import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor; -import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -71,11 +70,11 @@ void shouldPrepareTraceSchemaPipelineAndReadiness() { InOrder sqlOrder = org.mockito.Mockito.inOrder(sqlQueryExecutor); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( - "CREATE TABLE IF NOT EXISTS hzb_traces")); + "CREATE TABLE IF NOT EXISTS hertzbeat_traces")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( - "ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\"")); + "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.service.namespace\"")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.startsWith( - "ALTER TABLE hzb_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\"")); + "ALTER TABLE hertzbeat_traces ADD COLUMN IF NOT EXISTS \"resource_attributes.deployment.environment.name\"")); sqlOrder.verify(sqlQueryExecutor).execute(org.mockito.ArgumentMatchers.contains( "CREATE TABLE IF NOT EXISTS hertzbeat_logs")); sqlOrder.verify(sqlQueryExecutor).execute("SELECT 1 AS ready"); diff --git a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java similarity index 95% rename from hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java rename to hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java index 1e29119ec48..7859196365c 100644 --- a/hertzbeat-log/src/test/java/org/apache/hertzbeat/log/config/GreptimeSignalPipelineTest.java +++ b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/history/tsdb/greptime/GreptimeSignalPipelineTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hertzbeat.log.config; +package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime; import static org.assertj.core.api.Assertions.assertThat; diff --git a/home/docs/help/log_integration.md b/home/docs/help/log_integration.md index ffdd3aaaa91..85429f993b6 100644 --- a/home/docs/help/log_integration.md +++ b/home/docs/help/log_integration.md @@ -11,6 +11,10 @@ keywords: [open source monitoring, log integration, log management, multi-source The log integration feature is currently in Beta (experimental) stage. There may be potential defects and limitations. The feature is under active development and iteration. ::: +:::info HertzBeat 1.9.0 transition +Metrics, logs, and traces share `/api/otlp/v1/{signal}` for ingestion and `/api/observability/**` for queries. This release intentionally does not create or bind Entity records from telemetry. External OTLP signal tables are also separate from HertzBeat's internal self-telemetry tables. +::: + ## Core Capabilities - **Multi-source Log Integration**: Support receiving log data from mainstream platforms such as OpenTelemetry, Filebeat, Vector, Loki @@ -36,7 +40,7 @@ You can view specific integration methods and configuration examples through Her HertzBeat provides the following interface for receiving OTLP log data: ```text -POST /api/logs/otlp/v1/logs +POST /api/otlp/v1/logs ``` ### Request Configuration @@ -112,7 +116,7 @@ Add HertzBeat as a log export target in the OpenTelemetry Collector configuratio ```yaml exporters: otlphttp: - logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs + logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs compression: none encoding: json headers: diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md index 9e3a29bf6d3..a564d080e2a 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/log_integration.md @@ -11,6 +11,10 @@ keywords: [开源监控, 日志集成, 日志管理, 多源日志] 日志集成功能目前处于 Beta（实验性）阶段，可能存在潜在缺陷和局限性。该功能正在积极开发和迭代中。 ::: +:::info HertzBeat 1.9.0 过渡版本 +指标、日志和链路统一通过 `/api/otlp/v1/{signal}` 接收，并通过 `/api/observability/**` 查询。该版本不会根据遥测数据创建或绑定 Entity；外部 OTLP 三信号表也与 HertzBeat 自身遥测表分开存储。 +::: + ## 核心能力 - **多源日志接入**：支持从 OpenTelemetry、Filebeat、Vector、Loki 等主流平台接收日志数据 @@ -36,7 +40,7 @@ HertzBeat 当前已支持以下协议进行日志数据接入： HertzBeat 提供以下接口用于接收 OTLP 日志数据： ```text -POST /api/logs/otlp/v1/logs +POST /api/otlp/v1/logs ``` ### 请求配置 @@ -112,7 +116,7 @@ POST /api/logs/otlp/v1/logs ```yaml exporters: otlphttp: - logs_endpoint: http://{hertzbeat_host}:1157/api/logs/otlp/v1/logs + logs_endpoint: http://{hertzbeat_host}:1157/api/otlp/v1/logs compression: none encoding: json headers: diff --git a/pom.xml b/pom.xml index 087c7119a78..ad64ca64ea9 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ hertzbeat-otel hertzbeat-e2e hertzbeat-base - hertzbeat-log + hertzbeat-observability hertzbeat-ai @@ -268,7 +268,7 @@ org.apache.hertzbeat - hertzbeat-log + hertzbeat-observability ${hertzbeat.version} diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml index fd283a60c68..44c1c4079a6 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml @@ -90,8 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -103,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml index fd283a60c68..44c1c4079a6 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml @@ -90,8 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -103,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml index fd283a60c68..44c1c4079a6 100644 --- a/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml @@ -90,8 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -103,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml index 2c056a8448d..4a8ffbf04e1 100644 --- a/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml @@ -90,12 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -107,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml index fd283a60c68..44c1c4079a6 100644 --- a/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml +++ b/script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml @@ -90,8 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -103,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/script/sureness.yml b/script/sureness.yml index 2c056a8448d..4a8ffbf04e1 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -90,12 +90,9 @@ resourceRole: - /api/ai/**===post===[admin] - /api/ai/**===put===[admin] - /api/ai/**===delete===[admin] - - /api/logs/sse/**===get===[admin,user,guest] - - /api/logs/ingest/**===post===[admin,user] - - /api/otlp/**===post===[admin,user] - - /api/ingestion/otlp/**===get===[admin,user,guest] - - /api/logs/**===get===[admin,user,guest] - - /api/traces/**===get===[admin,user,guest] + - /api/otlp/v1/**===post===[admin,user] + - /api/observability/logs===delete===[admin] + - /api/observability/**===get===[admin,user,guest] # spring boot actuator exposes jvm, http and datasource internals for scraping - /actuator/**===get===[admin] @@ -107,7 +104,6 @@ excludedResource: - /api/account/auth/**===* - /api/i18n/**===get - /api/apps/hierarchy===get - - /api/observability/capability===get - /api/push/**===* - /api/status/page/public/**===* - /api/manager/sse/**===* diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.html b/web-app/src/app/routes/log/log-manage/log-manage.component.html index d70c4a7fd05..d8a48ee67a0 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.html +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.html @@ -15,111 +15,189 @@ limitations under the License. --> -
-

{{ 'observability.logs.title' | i18n }}

{{ 'observability.logs.subtitle' | i18n }}

- -
- - - - - - - - + +
+ +
+ + +
+
+ + + +
+
- -
{{ 'observability.logs.total' | i18n }}{{ overview.totalCount || 0 }}
ERROR{{ overview.errorCount || 0 }}
WARN{{ overview.warnCount || 0 }}
{{ 'observability.logs.with-trace' | i18n }}{{ overview.traceCoverage?.withTrace || 0 }}
+ + + +
+ {{ 'observability.filter.advanced' | i18n }} +
+ + + +
+
+
+ +
+ +
+
{{ 'observability.logs.total' | i18n }}{{ overview.totalCount || 0 }}
-

{{ 'observability.logs.trend' | i18n }}

{{ total }}
ERROR{{ overview.errorCount || 0 }}
-
-

{{ 'observability.logs.results' | i18n }}

{{ total }}
WARN{{ overview.warnCount || 0 }} - {{ 'observability.logs.with-trace' | i18n }}{{ overview.traceCoverage?.withTrace || 0 }} - {{ 'observability.column.time' | i18n }}{{ 'observability.logs.severity' | i18n }}{{ 'observability.column.service' | i18n }}{{ 'observability.logs.message' | i18n }}Trace ID - {{ (log.timeUnixNano || 0) / 1000000 | date : 'yyyy-MM-dd HH:mm:ss.SSS' }}{{ log.severityText || log.severityNumber || '-' }}{{ service(log) }}{{ bodyText(log) }}- + +
+
+
+
+

{{ 'observability.logs.trend' | i18n }}

+ {{ total }} +
+
+
+
+ +
+
+ +
+
+
+

{{ 'observability.logs.results' | i18n }}

+ {{ total }} +
+
+ - + {{ 'observability.column.time' | i18n }}{{ 'observability.logs.severity' | i18n }}{{ 'observability.column.service' | i18n }}{{ 'observability.logs.message' | i18n }}Trace ID + {{ (log.timeUnixNano || 0) / 1000000 | date : 'yyyy-MM-dd HH:mm:ss.SSS' }}{{ log.severityText || log.severityNumber || '-' }}{{ service(log) }}{{ bodyText(log) }}- + +
+ 
{{ detailJson() }}
 - - +
+ +
- -
+ + diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.less b/web-app/src/app/routes/log/log-manage/log-manage.component.less index e37a150eaff..91c729f4fd1 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.less +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.less @@ -1,116 +1,44 @@ @import '~src/styles/theme'; -.signal-page { - min-height: 100%; -} - -.page-heading { - display: flex; - gap: 24px; - align-items: flex-start; - justify-content: space-between; - padding-bottom: 18px; - border-bottom: 1px solid @border-color-split; -} - -.page-heading h1 { - margin: 0 0 5px; - font-weight: 600; - font-size: 24px; -} - -.page-heading p { - margin: 0; - color: @text-color-secondary; -} - .heading-actions { display: flex; - gap: 16px; + gap: 12px; align-items: center; } .mode-switch { display: flex; - gap: 0; + gap: 2px; + padding: 3px; + background: @background-color-light; + border: 1px solid @border-color-split; + border-radius: 6px; } -.mode-switch button + button { - margin-left: -1px; +.mode-switch button { + min-width: 62px; + border: 0; + box-shadow: none; } -.query-toolbar { - display: grid; - grid-template-columns: repeat(4, minmax(150px, 1fr)); - gap: 8px; - align-items: center; - padding: 16px 0; +.logs-query-fields { + grid-template-columns: repeat(4, minmax(0, 1fr)); } -.query-toolbar app-signal-time-range { - grid-column: 1 / -1; +.logs-advanced-filter-grid { + grid-template-columns: repeat(3, minmax(0, 1fr)); } .resource-expression { width: 100%; } -.overview-strip { - display: grid; - grid-template-columns: repeat(4, 1fr); - margin-bottom: 16px; - border: 1px solid @border-color-split; -} - -.overview-strip div { - display: flex; - flex-direction: column; - gap: 5px; - padding: 10px 16px; - border-right: 1px solid @border-color-split; -} - -.overview-strip div:last-child { - border-right: 0; -} - -.overview-strip span { - color: @text-color-secondary; - font-size: 12px; -} - -.overview-strip strong { - font-size: 20px; -} - -.chart-region { - min-height: 240px; - border: 1px solid @border-color-split; +.logs-overview { + grid-template-columns: repeat(4, minmax(0, 1fr)); } .trend-chart { - height: 240px; -} - -.chart-region nz-empty { - display: block; - padding-top: 48px; -} - -.section-title { - display: flex; - gap: 8px; - align-items: center; - margin: 20px 0 10px; -} - -.section-title h2 { - margin: 0; - font-size: 16px; -} - -.section-title span { - color: @text-color-secondary; + height: 196px; } .message-cell { @@ -121,12 +49,16 @@ } .mono { - max-width: 220px; + max-width: 260px; overflow: hidden; font-family: SFMono-Regular, Consolas, monospace; text-overflow: ellipsis; } +.stream-evidence { + margin-top: 16px; +} + .drawer-actions { display: flex; gap: 8px; @@ -141,12 +73,13 @@ pre { color: #d7e0ea; background: #0d1117; } -@media (max-width: 1400px) { - .query-toolbar { - grid-template-columns: repeat(4, minmax(150px, 1fr)); + +@media (max-width: 1100px) { + .logs-query-fields { + grid-template-columns: repeat(2, minmax(0, 1fr)); } - .query-toolbar > button:last-child { - grid-column: -2 / -1; + .logs-advanced-filter-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); } } diff --git a/web-app/src/app/routes/log/log-manage/log-manage.component.ts b/web-app/src/app/routes/log/log-manage/log-manage.component.ts index 97cda52845d..3c147088790 100644 --- a/web-app/src/app/routes/log/log-manage/log-manage.component.ts +++ b/web-app/src/app/routes/log/log-manage/log-manage.component.ts @@ -44,6 +44,7 @@ import { } from '../../observability/signal-query-context'; import { SignalStorageGuideComponent } from '../../observability/signal-storage-guide.component'; import { SignalTimeRangeComponent } from '../../observability/signal-time-range.component'; +import { SignalWorkspaceComponent } from '../../observability/signal-workspace.component'; import { LogStreamComponent } from '../log-stream/log-stream.component'; type LogViewMode = 'query' | 'stream'; @@ -72,6 +73,7 @@ export function trendBucketMillis(value: string): number { SignalNavigationComponent, SignalStorageGuideComponent, SignalTimeRangeComponent, + SignalWorkspaceComponent, LogStreamComponent ], templateUrl: './log-manage.component.html', diff --git a/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts b/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts index b3e6908e32e..08d1c7f1517 100644 --- a/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts +++ b/web-app/src/app/routes/log/log-stream/log-stream.component.spec.ts @@ -54,7 +54,7 @@ describe('LogStreamComponent', () => { it('should authenticate the log stream request', () => { expect(fetchSpy).toHaveBeenCalledWith( - '/api/logs/sse/subscribe', + '/api/observability/logs/stream', jasmine.objectContaining({ method: 'GET', headers: jasmine.objectContaining({ diff --git a/web-app/src/app/routes/log/log-stream/log-stream.component.ts b/web-app/src/app/routes/log/log-stream/log-stream.component.ts index c1c03937127..61330b1aa3d 100644 --- a/web-app/src/app/routes/log/log-stream/log-stream.component.ts +++ b/web-app/src/app/routes/log/log-stream/log-stream.component.ts @@ -163,7 +163,7 @@ export class LogStreamComponent implements OnInit, OnDestroy, AfterViewInit { // Build filter parameters const filterParams = this.buildFilterParams(); - const url = `/api/logs/sse/subscribe${filterParams ? `?${filterParams}` : ''}`; + const url = `/api/observability/logs/stream${filterParams ? `?${filterParams}` : ''}`; const token = this.localStorageService.getAuthorizationToken(); const headers: Record = { Accept: 'text/event-stream', diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.html b/web-app/src/app/routes/metrics/metrics-manage.component.html index 03dc7be18f1..f2f3cde54ff 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.html +++ b/web-app/src/app/routes/metrics/metrics-manage.component.html @@ -15,45 +15,60 @@ limitations under the License. --> -
-
-
-

{{ 'observability.metrics.title' | i18n }}

-

{{ 'observability.metrics.subtitle' | i18n }}

-
-
- - {{ series.length }} {{ 'observability.metrics.series' | i18n }} -
-
+ +
+ +
+ + -
- - - - {{ name }} - - -
- +
-
+
{{ 'observability.metrics.total-series' | i18n }}{{ series.length }}
{{ 'observability.metrics.title' | i18n }} >
-
-

{{ 'observability.metrics.trend' | i18n }}

{{ series.length }} -
- -
-
- -
+
+
+
+
+

{{ 'observability.metrics.trend' | i18n }}

+ {{ series.length }} {{ 'observability.metrics.series' | i18n }} +
+
+
+
+ +
+
+ +
+
-
- -
+
+
+
+

{{ 'observability.metrics.samples' | i18n }}

+ {{ rows.length }} +
+
+ + {{ 'observability.column.time' | i18n }}{{ 'observability.column.value' | i18n }}{{ 'observability.column.labels' | i18n }} + + + {{ row.timestamp | date : 'yyyy-MM-dd HH:mm:ss' }}{{ row.value }}{{ labelText(row.labels) }} + + + +
-
-

{{ 'observability.metrics.samples' | i18n }}

- {{ rows.length }} -
- - {{ 'observability.column.time' | i18n }}{{ 'observability.column.value' | i18n }}{{ 'observability.column.labels' | i18n }} - - - {{ row.timestamp | date : 'yyyy-MM-dd HH:mm:ss' }}{{ row.value }}{{ labelText(row.labels) }} - - - - - - -
-
{{ label.key }}
{{ label.value }}
 -
- - -
+ +
+
{{ label.key }}
{{ label.value }}
 +
+ + + + diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.less b/web-app/src/app/routes/metrics/metrics-manage.component.less index 277be38e3ac..7994969001d 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.less +++ b/web-app/src/app/routes/metrics/metrics-manage.component.less @@ -1,95 +1,31 @@ @import '~src/styles/theme'; -.signal-page { - min-height: 100%; +.metrics-query-fields { + grid-template-columns: minmax(320px, 1fr) 150px 112px; } -.page-heading { - display: flex; - align-items: flex-start; - justify-content: space-between; - padding-bottom: 18px; - border-bottom: 1px solid @border-color-split; -} - -.page-heading h1 { - margin: 0 0 5px; - font-weight: 600; - font-size: 24px; -} - -.page-heading p, -.result-count { - margin: 0; - color: @text-color-secondary; -} - -.heading-actions { - display: flex; - gap: 16px; - align-items: center; -} - -.query-toolbar { - display: grid; - grid-template-columns: minmax(400px, 1fr) 140px auto; - gap: 8px; - align-items: center; - padding: 16px 0; -} - -.query-toolbar app-signal-time-range { - grid-column: 1 / -1; +.metrics-overview { + grid-template-columns: repeat(4, minmax(0, 1fr)); } .promql-input { width: 100%; } -.chart-region { - min-height: 240px; - background: @component-background; - border: 1px solid @border-color-split; -} - .metric-chart { - height: 240px; -} - -.overview-strip { - display: grid; - grid-template-columns: repeat(4, 1fr); - margin-bottom: 0; - border: 1px solid @border-color-split; -} - -.overview-strip div { - display: flex; - flex-direction: column; - gap: 5px; - padding: 10px 16px; - border-right: 1px solid @border-color-split; + height: 196px; } -.overview-strip div:last-child { - border-right: 0; -} - -.overview-strip span { - color: @text-color-secondary; - font-size: 12px; -} - -.overview-strip strong { - font-size: 20px; +.loading-region { + opacity: 0.55; } .series-strip { display: flex; gap: 6px; - padding: 8px 0; + padding: 8px 12px; overflow-x: auto; - border-bottom: 1px solid @border-color-split; + border-top: 1px solid @border-color-split; } .series-strip button { @@ -100,31 +36,6 @@ text-overflow: ellipsis; } -.chart-region nz-empty { - display: block; - padding-top: 60px; -} - -.loading-region { - opacity: 0.55; -} - -.section-title { - display: flex; - gap: 8px; - align-items: center; - margin: 18px 0 10px; -} - -.section-title h2 { - margin: 0; - font-size: 16px; -} - -.section-title span { - color: @text-color-secondary; -} - .mono { max-width: 640px; overflow: hidden; @@ -149,12 +60,9 @@ .detail-list dt { color: @text-color-secondary; } -@media (max-width: 1400px) { - .query-toolbar { - grid-template-columns: minmax(320px, 1fr) 130px auto; - } - .query-toolbar > button:last-child { - grid-column: -2 / -1; +@media (max-width: 1000px) { + .metrics-query-fields { + grid-template-columns: minmax(260px, 1fr) 130px 104px; } } diff --git a/web-app/src/app/routes/metrics/metrics-manage.component.ts b/web-app/src/app/routes/metrics/metrics-manage.component.ts index a6d8dfe0695..61242623dd0 100644 --- a/web-app/src/app/routes/metrics/metrics-manage.component.ts +++ b/web-app/src/app/routes/metrics/metrics-manage.component.ts @@ -42,6 +42,7 @@ import { } from '../observability/signal-query-context'; import { SignalStorageGuideComponent } from '../observability/signal-storage-guide.component'; import { SignalTimeRangeComponent } from '../observability/signal-time-range.component'; +import { SignalWorkspaceComponent } from '../observability/signal-workspace.component'; interface MetricTableRow { timestamp: number; @@ -64,7 +65,8 @@ interface MetricTableRow { NzAutocompleteModule, SignalNavigationComponent, SignalStorageGuideComponent, - SignalTimeRangeComponent + SignalTimeRangeComponent, + SignalWorkspaceComponent ], templateUrl: './metrics-manage.component.html', styleUrl: './metrics-manage.component.less' diff --git a/web-app/src/app/routes/observability/signal-navigation.component.ts b/web-app/src/app/routes/observability/signal-navigation.component.ts index c1bd3389fef..d60b1615b29 100644 --- a/web-app/src/app/routes/observability/signal-navigation.component.ts +++ b/web-app/src/app/routes/observability/signal-navigation.component.ts @@ -31,10 +31,12 @@ type SignalKind = 'metrics' | 'logs' | 'traces'; standalone: true, imports: [I18nPipe, RouterModule, SharedModule], template: ` -