From 45242fec4606fb65cceb9c3397cc122c7f4f09b9 Mon Sep 17 00:00:00 2001 From: Ila Hadi-Assar Date: Fri, 17 Apr 2026 12:48:24 +0200 Subject: [PATCH 01/12] Add tracing support to eCAL. --- cmake/submodule_dependencies.cmake | 1 + conanfile.py | 7 +- ecal/core/CMakeLists.txt | 21 ++ ecal/core/include/ecal/config/configuration.h | 2 + ecal/core/include/ecal/config/tracing.h | 38 +++ ecal/core/include/ecal/util.h | 18 ++ .../core/src/config/configuration_to_yaml.cpp | 24 ++ ecal/core/src/config/configuration_to_yaml.h | 16 ++ .../core/src/config/default_configuration.cpp | 5 + ecal/core/src/config/ecal_path_processing.cpp | 51 ++-- ecal/core/src/config/ecal_path_processing.h | 18 ++ ecal/core/src/ecal.cpp | 2 + ecal/core/src/ecal_def.h | 2 + ecal/core/src/ecal_global_accessors.cpp | 21 ++ ecal/core/src/ecal_global_accessors.h | 10 + ecal/core/src/ecal_util.cpp | 5 + ecal/core/src/pubsub/ecal_publisher_impl.cpp | 43 +++ ecal/core/src/pubsub/ecal_publisher_impl.h | 1 + ecal/core/src/pubsub/ecal_subscriber_impl.cpp | 42 ++- ecal/core/src/tracing/span.cpp | 74 ++++++ ecal/core/src/tracing/span.h | 69 +++++ ecal/core/src/tracing/trace_provider.cpp | 38 +++ ecal/core/src/tracing/trace_provider.h | 50 ++++ .../src/tracing/trace_provider_default.cpp | 102 +++++++ .../core/src/tracing/trace_provider_default.h | 73 +++++ ecal/core/src/tracing/trace_provider_noop.h | 36 +++ ecal/core/src/tracing/tracing.h | 128 +++++++++ ecal/core/src/tracing/tracing_writer.h | 45 ++++ .../core/src/tracing/tracing_writer_jsonl.cpp | 154 +++++++++++ ecal/core/src/tracing/tracing_writer_jsonl.h | 64 +++++ ecal/tests/CMakeLists.txt | 2 +- .../config_test/src/path_processing_test.cpp | 65 +++++ .../config_test/src/yaml_processing_test.cpp | 4 + ecal/tests/cpp/tracing_test/CMakeLists.txt | 58 ++++ .../tracing_test/src/trace_provider_test.cpp | 127 +++++++++ .../tracing_test/src/tracing_test_helpers.h | 154 +++++++++++ .../tracing_test/src/tracing_writer_test.cpp | 251 ++++++++++++++++++ .../nlohmann_json/build-nlohmann_json.cmake | 13 + 38 files changed, 1810 insertions(+), 24 deletions(-) create mode 100644 ecal/core/include/ecal/config/tracing.h create mode 100644 ecal/core/src/tracing/span.cpp create mode 100644 ecal/core/src/tracing/span.h create mode 100644 ecal/core/src/tracing/trace_provider.cpp create mode 100644 ecal/core/src/tracing/trace_provider.h create mode 100644 ecal/core/src/tracing/trace_provider_default.cpp create mode 100644 ecal/core/src/tracing/trace_provider_default.h create mode 100644 ecal/core/src/tracing/trace_provider_noop.h create mode 100644 ecal/core/src/tracing/tracing.h create mode 100644 ecal/core/src/tracing/tracing_writer.h create mode 100644 ecal/core/src/tracing/tracing_writer_jsonl.cpp create mode 100644 ecal/core/src/tracing/tracing_writer_jsonl.h create mode 100644 ecal/tests/cpp/tracing_test/CMakeLists.txt create mode 100644 ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp create mode 100644 ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h create mode 100644 ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp create mode 100644 thirdparty/nlohmann_json/build-nlohmann_json.cmake diff --git a/cmake/submodule_dependencies.cmake b/cmake/submodule_dependencies.cmake index 622a81b21c..c411c57239 100644 --- a/cmake/submodule_dependencies.cmake +++ b/cmake/submodule_dependencies.cmake @@ -15,6 +15,7 @@ set(ecal_submodule_dependencies HDF5 #libssh2 nanobind + nlohmann_json Protobuf protozero qwt diff --git a/conanfile.py b/conanfile.py index e0584304f6..a04a9f19bd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -17,6 +17,7 @@ def build_requirements(self): def requirements(self): self.requires("hdf5/1.10.6") self.requires("protobuf/3.17.1") + self.requires("nlohmann_json/3.11.2") self.requires("libcurl/7.78.0") self.requires("qt/5.15.2") self.requires("spdlog/1.9.2") @@ -51,6 +52,6 @@ def generate(self): else: tc.variables["Protobuf_PROTOC_EXECUTABLE"] = os.path.join(self.deps_cpp_info["protobuf"].rootpath, "bin", "protoc") tc.generate() - - - + + + diff --git a/ecal/core/CMakeLists.txt b/ecal/core/CMakeLists.txt index 9f31e3e087..ad607ba6c1 100644 --- a/ecal/core/CMakeLists.txt +++ b/ecal/core/CMakeLists.txt @@ -43,6 +43,7 @@ find_package(asio REQUIRED) find_package(Threads REQUIRED) find_package(ecaludp REQUIRED) find_package(protozero REQUIRED) +find_package(nlohmann_json REQUIRED) if (ECAL_CORE_CONFIGURATION) find_package(yaml-cpp REQUIRED) @@ -457,6 +458,23 @@ if(ECAL_CORE_TIMEPLUGIN) ) endif() +###################################### +# tracing +###################################### +set(ecal_tracing_src + src/tracing/tracing.h + src/tracing/span.cpp + src/tracing/span.h + src/tracing/trace_provider.cpp + src/tracing/trace_provider.h + src/tracing/trace_provider_default.cpp + src/tracing/trace_provider_default.h + src/tracing/trace_provider_noop.h + src/tracing/tracing_writer.h + src/tracing/tracing_writer_jsonl.cpp + src/tracing/tracing_writer_jsonl.h +) + ###################################### # util ###################################### @@ -629,6 +647,7 @@ set(ecal_sources ${ecal_serialization_src} ${ecal_service_src} ${ecal_time_src} + ${ecal_tracing_src} ${ecal_util_src} ${ecal_cmn_src} ${ecal_builder_src} @@ -707,6 +726,8 @@ target_link_libraries(ecal_core_private $<$:tcp_pubsub::tcp_pubsub> $<$:ecaltime> ecaludp::ecaludp + nlohmann_json::nlohmann_json + $<$,$>>:dl> ${CMAKE_DL_LIBS} $<$,$>,$>>:rt> $<$,$>,$>>:atomic> diff --git a/ecal/core/include/ecal/config/configuration.h b/ecal/core/include/ecal/config/configuration.h index 77728641a5..437cf535db 100644 --- a/ecal/core/include/ecal/config/configuration.h +++ b/ecal/core/include/ecal/config/configuration.h @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -58,6 +59,7 @@ namespace eCAL Time::Configuration timesync; Application::Configuration application; Logging::Configuration logging; + Tracing::Configuration tracing; eCommunicationMode communication_mode { eCommunicationMode::local }; /*!< eCAL components communication mode: local: local host only communication (default) diff --git a/ecal/core/include/ecal/config/tracing.h b/ecal/core/include/ecal/config/tracing.h new file mode 100644 index 0000000000..7f2ec86aa1 --- /dev/null +++ b/ecal/core/include/ecal/config/tracing.h @@ -0,0 +1,38 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +/** + * @file config/tracing.h + * @brief eCAL tracing configuration +**/ + +#pragma once + +#include + +namespace eCAL +{ + namespace Tracing + { + struct Configuration + { + bool enabled { false }; //!< Enable tracing (Default: false) + }; + } +} diff --git a/ecal/core/include/ecal/util.h b/ecal/core/include/ecal/util.h index d957336326..1b5793ee6a 100644 --- a/ecal/core/include/ecal/util.h +++ b/ecal/core/include/ecal/util.h @@ -64,6 +64,24 @@ namespace eCAL **/ ECAL_API std::string GeteCALLogDir(); + /** + * @brief Returns the path to the eCAL trace directory. + * + * Searches in following order: + * 1. Environment variable ECAL_TRACE_DIR + * 2. Environment variable ECAL_DATA (also checking for traces subdirectory) + * 3. The path where ecal.yaml was loaded from (also checking for traces subdirectory) + * 4. The temporary directory (e.g. /tmp [unix], Appdata/local/Temp [win]) + * 5. Fallback path /ecal_tmp + * + * In case of 4/5, a unique temporary folder will be created. + * + * @returns The path to the eCAL trace directory. + * The subdirectory traces might not exist yet. + * Returns empty string if no root path could be found. + **/ + ECAL_API std::string GeteCALTraceDir(); + /** * @brief Send shutdown event to specified local user process using it's unit name. * diff --git a/ecal/core/src/config/configuration_to_yaml.cpp b/ecal/core/src/config/configuration_to_yaml.cpp index 58867f8e0b..de43de28f4 100644 --- a/ecal/core/src/config/configuration_to_yaml.cpp +++ b/ecal/core/src/config/configuration_to_yaml.cpp @@ -673,6 +673,28 @@ namespace YAML } + /* + ______ _ + /_ __/______ _____(_)__ ___ _ + / / / __/ _ `/ __/ / _ \/ _ `/ + /_/ /_/ \_,_/\__/_/_//_/\_, / + /___/ + */ + + Node convert::encode(const eCAL::Tracing::Configuration& config_) + { + Node node; + node["enabled"] = config_.enabled; + return node; + } + + bool convert::decode(const Node& node_, eCAL::Tracing::Configuration& config_) + { + AssignValue(config_.enabled, node_, "enabled"); + return true; + } + + /* __ ___ _ ____ __ _ / |/ /__ _(_)__ _______ ___ / _(_)__ ___ _________ _/ /_(_)__ ___ @@ -691,6 +713,7 @@ namespace YAML node["time"] = config_.timesync; node["application"] = config_.application; node["logging"] = config_.logging; + node["tracing"] = config_.tracing; node["communication_mode"] = config_.communication_mode == eCAL::eCommunicationMode::network ? "network" : "local"; return node; @@ -705,6 +728,7 @@ namespace YAML AssignValue(config_.timesync, node_, "time"); AssignValue(config_.application, node_, "application"); AssignValue(config_.logging, node_, "logging"); + AssignValue(config_.tracing, node_, "tracing"); std::string communication_mode; AssignValue(communication_mode, node_, "communication_mode"); diff --git a/ecal/core/src/config/configuration_to_yaml.h b/ecal/core/src/config/configuration_to_yaml.h index 012c3e2a0d..8ad27f25f2 100644 --- a/ecal/core/src/config/configuration_to_yaml.h +++ b/ecal/core/src/config/configuration_to_yaml.h @@ -343,6 +343,22 @@ namespace YAML }; + /* + ______ _ + /_ __/______ _____(_)__ ___ _ + / / / __/ _ `/ __/ / _ \/ _ `/ + /_/ /_/ \_,_/\__/_/_//_/\_, / + /___/ + */ + template<> + struct convert + { + static Node encode(const eCAL::Tracing::Configuration& config_); + + static bool decode(const Node& node_, eCAL::Tracing::Configuration& config_); + }; + + /* __ ___ _ ____ __ _ / |/ /__ _(_)__ _______ ___ / _(_)__ ___ _________ _/ /_(_)__ ___ diff --git a/ecal/core/src/config/default_configuration.cpp b/ecal/core/src/config/default_configuration.cpp index 02facd3b24..1e538dd624 100644 --- a/ecal/core/src/config/default_configuration.cpp +++ b/ecal/core/src/config/default_configuration.cpp @@ -381,6 +381,11 @@ namespace eCAL ss << R"( # UDP Port for sending logging data)" << "\n"; ss << R"( port: )" << config_.logging.receiver.udp_config.port << "\n"; ss << R"()" << "\n"; + ss << R"(# Tracing configuration)" << "\n"; + ss << R"(tracing:)" << "\n"; + ss << R"( # Enable tracing (Default: false))" << "\n"; + ss << R"( enabled: )" << config_.tracing.enabled << "\n"; + ss << R"()" << "\n"; return ss; } diff --git a/ecal/core/src/config/ecal_path_processing.cpp b/ecal/core/src/config/ecal_path_processing.cpp index 01ad66357d..9b8c59be01 100644 --- a/ecal/core/src/config/ecal_path_processing.cpp +++ b/ecal/core/src/config/ecal_path_processing.cpp @@ -150,6 +150,31 @@ namespace #endif } + + std::string GeteCALOutputDirImpl(const eCAL::Util::IDirProvider& dir_provider_, const eCAL::Util::IDirManager& dir_manager_, const std::string& env_var_name_, const std::string& output_subdirectory_, const std::string& config_file_path_, const std::string& config_output_dir_) + { + const std::string config_file_dir = dir_manager_.getDirectoryPath(config_file_path_); + const std::string ecal_data_env_dir = dir_provider_.eCALEnvVar(ECAL_DATA_VAR); + + const std::vector output_paths = { + dir_provider_.eCALEnvVar(env_var_name_), + buildPath(ecal_data_env_dir, output_subdirectory_), + ecal_data_env_dir, + config_output_dir_, + buildPath(config_file_dir, output_subdirectory_), + config_file_dir + }; + + for (const auto& path : output_paths) + { + if (!path.empty() && dir_manager_.dirExists(path) && dir_manager_.canWriteToDirectory(path)) + { + return path; + } + } + + return dir_provider_.uniqueTmpDir(dir_manager_); + } } namespace eCAL @@ -315,27 +340,15 @@ namespace eCAL { std::string GeteCALLogDirImpl(const Util::IDirProvider& dir_provider_ /* = Util::DirProvider() */, const Util::IDirManager& dir_manager_ /* = Util::DirManager() */, const eCAL::Configuration& config_ /* = eCAL::GetConfiguration() */) { - const std::string config_file_dir = dir_manager_.getDirectoryPath(config_.GetConfigurationFilePath()); - const std::string ecal_data_env_dir = dir_provider_.eCALEnvVar(ECAL_DATA_VAR); - - const std::vector log_paths = { - dir_provider_.eCALEnvVar(ECAL_LOG_VAR), - buildPath(ecal_data_env_dir, ECAL_FOLDER_NAME_LOG), - ecal_data_env_dir, - config_.logging.provider.file_config.path, - buildPath(config_file_dir, ECAL_FOLDER_NAME_LOG), - config_file_dir - }; - - for (const auto& path : log_paths) - { - if (!path.empty() && dir_manager_.dirExists(path) && dir_manager_.canWriteToDirectory(path)) - return path; - } - // if no path is available, we create temp directories for logging // check now for a tmp directory and return - return dir_provider_.uniqueTmpDir(dir_manager_); + return GeteCALOutputDirImpl(dir_provider_, dir_manager_, ECAL_LOG_VAR, ECAL_FOLDER_NAME_LOG, config_.GetConfigurationFilePath(), config_.logging.provider.file_config.path); + } + + std::string GeteCALTraceDirImpl(const Util::IDirProvider& dir_provider_ /* = Util::DirProvider() */, const Util::IDirManager& dir_manager_ /* = Util::DirManager() */, const eCAL::Configuration& config_ /* = eCAL::GetConfiguration() */) + { + // if no path is available, we create temp directories for tracing + return GeteCALOutputDirImpl(dir_provider_, dir_manager_, ECAL_TRACE_VAR, ECAL_FOLDER_NAME_TRACE, config_.GetConfigurationFilePath(), {}); } std::string checkForValidConfigFilePath(const std::string& config_file_, const Util::DirProvider& dir_provider_ /* = Util::DirProvider() */, const Util::DirManager& dir_manager_ /* = Util::DirManager() */) diff --git a/ecal/core/src/config/ecal_path_processing.h b/ecal/core/src/config/ecal_path_processing.h index 75147618a0..a0143e4bbc 100644 --- a/ecal/core/src/config/ecal_path_processing.h +++ b/ecal/core/src/config/ecal_path_processing.h @@ -221,6 +221,24 @@ namespace eCAL */ std::string GeteCALLogDirImpl(const Util::IDirProvider& dir_provider_ = Util::DirProvider(), const Util::IDirManager& dir_manager_ = Util::DirManager(), const eCAL::Configuration& config_ = eCAL::GetConfiguration()); + /** + * @brief Returns the path to the eCAL trace directory. + * + * Searches in following order: + * 1. Environment variable ECAL_TRACE_DIR + * 2. Environment variable ECAL_DATA (also checking for traces subdirectory) + * 3. The path where ecal.yaml was loaded from (also checking for traces subdirectory) + * 4. The temporary directory (e.g. /tmp [unix], Appdata/local/Temp [win]) + * 5. Fallback path /ecal_tmp + * + * In case of 4/5, a unique temporary folder will be created. + * + * @returns The path to the eCAL trace directory. + * The subdirectory traces might not exist yet. + * Returns empty string if no root path could be found. + */ + std::string GeteCALTraceDirImpl(const Util::IDirProvider& dir_provider_ = Util::DirProvider(), const Util::IDirManager& dir_manager_ = Util::DirManager(), const eCAL::Configuration& config_ = eCAL::GetConfiguration()); + /** * @brief Returns the path to the eCAL data directory. Searches in following order: * diff --git a/ecal/core/src/ecal.cpp b/ecal/core/src/ecal.cpp index 6f661a9f48..032f383885 100644 --- a/ecal/core/src/ecal.cpp +++ b/ecal/core/src/ecal.cpp @@ -114,6 +114,7 @@ namespace eCAL SetGlobalUnitName(unit_name_.c_str()); if ((components_ & Init::Logging) != 0u) InitializeLogging(config_); + InitializeTracing(config_); auto globals_instance = CreateGlobalsInstance(); if (!globals_instance) return false; @@ -165,6 +166,7 @@ namespace eCAL ResetGlobalEcalConfiguration(); ResetLogging(); + ResetTracing(); return finalized; } diff --git a/ecal/core/src/ecal_def.h b/ecal/core/src/ecal_def.h index d77747bf7d..18c82aa6bb 100644 --- a/ecal/core/src/ecal_def.h +++ b/ecal/core/src/ecal_def.h @@ -35,6 +35,7 @@ constexpr const char* ECAL_FOLDER_NAME_WINDOWS = "eCAL"; constexpr const char* ECAL_FOLDER_NAME_LINUX = "ecal"; constexpr const char* ECAL_FOLDER_NAME_HOME_LINUX = ".ecal"; constexpr const char* ECAL_FOLDER_NAME_LOG = "logs"; +constexpr const char* ECAL_FOLDER_NAME_TRACE = "traces"; constexpr const char* ECAL_FOLDER_NAME_TMP_WINDOWS = "Temp"; #ifdef ECAL_OS_WINDOWS @@ -51,6 +52,7 @@ constexpr const char* ECAL_DEFAULT_CFG = "ecal.yaml"; /* environment variables */ constexpr const char* ECAL_DATA_VAR = "ECAL_DATA"; constexpr const char* ECAL_LOG_VAR = "ECAL_LOG_DIR"; +constexpr const char* ECAL_TRACE_VAR = "ECAL_TRACE_DIR"; constexpr const char* ECAL_LINUX_HOME_VAR = "HOME"; constexpr const char* ECAL_LINUX_TMP_VAR = "TMPDIR"; diff --git a/ecal/core/src/ecal_global_accessors.cpp b/ecal/core/src/ecal_global_accessors.cpp index 2970bf58b5..33e20fd128 100644 --- a/ecal/core/src/ecal_global_accessors.cpp +++ b/ecal/core/src/ecal_global_accessors.cpp @@ -32,6 +32,9 @@ #include "config/builder/logging_attribute_builder.h" #include "logging/ecal_log_provider.h" #include "logging/ecal_log_receiver.h" +#include "tracing/trace_provider.h" +#include "tracing/trace_provider_default.h" +#include "tracing/trace_provider_noop.h" #include #include @@ -57,6 +60,8 @@ namespace eCAL std::shared_ptr g_log_provider_instance; std::shared_ptr g_log_receiver_instance; + std::shared_ptr g_trace_provider_instance; + void SetGlobalUnitName(const char *unit_name_) { if(unit_name_ != nullptr) g_unit_name = unit_name_; @@ -118,6 +123,22 @@ namespace eCAL return nullptr; } + void InitializeTracing(const eCAL::Configuration& config_) + { + g_trace_provider_instance = tracing::TraceProvider::Create(config_.tracing); + } + + void ResetTracing() + { + g_trace_provider_instance.reset(); + } + + std::shared_ptr g_trace_provider() + { + if (auto provider = g_trace_provider_instance; provider) return provider; + return nullptr; + } + std::shared_ptr g_globals() { if (auto globals_instance = g_globals_instance; globals_instance) diff --git a/ecal/core/src/ecal_global_accessors.h b/ecal/core/src/ecal_global_accessors.h index a4562f6383..3fd8ba8ee4 100644 --- a/ecal/core/src/ecal_global_accessors.h +++ b/ecal/core/src/ecal_global_accessors.h @@ -43,6 +43,11 @@ namespace eCAL class CLogReceiver; } + namespace tracing + { + class TraceProvider; + } + #if ECAL_CORE_MONITORING class CMonitoring; #endif @@ -80,6 +85,9 @@ namespace eCAL void InitializeLogging(const eCAL::Configuration& config_); void ResetLogging(); + void InitializeTracing(const eCAL::Configuration& config_); + void ResetTracing(); + // Declaration of getter functions for globally accessible variable instances std::shared_ptr g_globals(); #if ECAL_CORE_MONITORING @@ -110,6 +118,8 @@ namespace eCAL std::shared_ptr g_logging_provider(); std::shared_ptr g_logging_receiver(); + std::shared_ptr g_trace_provider(); + // declaration of globally accessible variables extern std::string g_default_ini_file; extern Configuration g_ecal_configuration; diff --git a/ecal/core/src/ecal_util.cpp b/ecal/core/src/ecal_util.cpp index ae9c4815ba..5cb42ffb18 100644 --- a/ecal/core/src/ecal_util.cpp +++ b/ecal/core/src/ecal_util.cpp @@ -46,6 +46,11 @@ namespace eCAL return eCAL::Config::GeteCALLogDirImpl(); } + std::string GeteCALTraceDir() + { + return eCAL::Config::GeteCALTraceDirImpl(); + } + #if ECAL_CORE_MONITORING // take monitoring snapshot static Monitoring::SMonitoring GetMonitoring() diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.cpp b/ecal/core/src/pubsub/ecal_publisher_impl.cpp index 437217e9d8..28f9152da4 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.cpp +++ b/ecal/core/src/pubsub/ecal_publisher_impl.cpp @@ -46,6 +46,10 @@ #include "registration/ecal_registration_provider.h" +#include "tracing/tracing.h" +#include "tracing/span.h" +#include "tracing/trace_provider.h" + #include #include #include @@ -122,6 +126,22 @@ namespace eCAL m_topic_id.topic_id.host_name = m_attributes.host_name; m_topic_id.topic_id.process_id = m_attributes.process_id; + // record topic metadata for tracing + { + if (auto provider = g_trace_provider(); provider) { + eCAL::tracing::STopicMetadata meta; + meta.entity_id = m_publisher_id; + meta.process_id = m_attributes.process_id; + meta.host_name = m_attributes.host_name; + meta.topic_name = m_attributes.topic_name; + meta.encoding = m_topic_info.encoding; + meta.type_name = m_topic_info.name; + meta.direction = eCAL::tracing::topic_direction::publisher; + + provider->WriteMetadata(meta); + } + } + // mark as created m_created = true; } @@ -179,6 +199,29 @@ namespace eCAL // prepare counter and internal states const size_t snd_hash = PrepareWrite(filter_id_, payload_buf_size); + // determine active transport layer for tracing + eCAL::tracing::eTracingLayerType active_layer = eCAL::tracing::tl_trace_none; + { +#if ECAL_CORE_TRANSPORT_SHM + if (m_writer_shm) { active_layer = static_cast(active_layer | eCAL::tracing::tl_trace_shm); } +#endif +#if ECAL_CORE_TRANSPORT_UDP + if (m_writer_udp) { active_layer = static_cast(active_layer | eCAL::tracing::tl_trace_udp); } +#endif +#if ECAL_CORE_TRANSPORT_TCP + if (m_writer_tcp) { active_layer = static_cast(active_layer | eCAL::tracing::tl_trace_tcp); } +#endif + } + + // create tracing span for the send operation + eCAL::tracing::CPublisherSpan send_span( + m_topic_id, + m_clock, + active_layer, + payload_buf_size, + eCAL::tracing::operation_type::send + ); + // did we write anything bool written(false); diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.h b/ecal/core/src/pubsub/ecal_publisher_impl.h index 1f0a37216b..43820a72ec 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.h +++ b/ecal/core/src/pubsub/ecal_publisher_impl.h @@ -118,6 +118,7 @@ namespace eCAL void GetRegistrationSample(Registration::Sample& sample); void GetUnregistrationSample(Registration::Sample& sample); + long long GetClock() const { return m_clock; } bool StartUdpLayer(); bool StartShmLayer(); diff --git a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp index dea6e0b20a..108e3405fd 100644 --- a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp +++ b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp @@ -23,6 +23,9 @@ **/ #include "ecal_subscriber_impl.h" +#include "tracing/tracing.h" +#include "tracing/span.h" +#include "tracing/trace_provider.h" #include #include #include @@ -93,6 +96,22 @@ namespace eCAL m_topic_id.topic_id.host_name = m_attributes.host_name; m_topic_id.topic_id.process_id = m_attributes.process_id; + // record topic metadata for tracing + { + if (auto provider = g_trace_provider(); provider) { + eCAL::tracing::STopicMetadata meta; + meta.entity_id = m_subscriber_id; + meta.process_id = m_attributes.process_id; + meta.host_name = m_attributes.host_name; + meta.topic_name = m_attributes.topic_name; + meta.encoding = m_topic_info.encoding; + meta.type_name = m_topic_info.name; + meta.direction = eCAL::tracing::topic_direction::subscriber; + + provider->WriteMetadata(meta); + } + } + // start transport layers InitializeLayers(); StartTransportLayer(); @@ -373,6 +392,16 @@ namespace eCAL size_t CSubscriberImpl::ApplySample(const Payload::TopicInfo& topic_info_, const char* payload_, size_t size_, long long id_, long long clock_, long long time_, size_t /*hash_*/, eTLayerType layer_) { + + eCAL::tracing::CSubscriberSpan receive_span( + m_subscriber_id, + topic_info_, + clock_, + eCAL::tracing::toTracingLayerType(layer_), + size_, + eCAL::tracing::operation_type::receive + ); + // ensure thread safety const std::lock_guard lock(m_receive_callback_mutex); if (!m_created) return(0); @@ -450,7 +479,18 @@ namespace eCAL // execute it const std::lock_guard exec_lock(m_connection_map_mtx); - (m_receive_callback)(topic_id, m_connection_map[pub_info].data_type_info, cb_data); + { + eCAL::tracing::CSubscriberSpan callback_span( + m_subscriber_id, + topic_info_, + clock_, + eCAL::tracing::toTracingLayerType(layer_), + size_, + eCAL::tracing::operation_type::callback_execution + ); + + + (m_receive_callback)(topic_id, m_connection_map[pub_info].data_type_info, cb_data); } processed = true; } } diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp new file mode 100644 index 0000000000..506e9f3478 --- /dev/null +++ b/ecal/core/src/tracing/span.cpp @@ -0,0 +1,74 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "span.h" +#include "trace_provider.h" +#include "../ecal_global_accessors.h" + +#include + +using namespace std::chrono; + +namespace eCAL +{ + namespace tracing + { + // Send span constructor + CPublisherSpan::CPublisherSpan(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + { + auto now = system_clock::now(); + data.start_ns = duration_cast(now.time_since_epoch()).count(); + data.entity_id = topic_id.topic_id.entity_id; + data.process_id = topic_id.topic_id.process_id; + data.payload_size = payload_size; + data.clock = clock; + data.layer = layer; + data.op_type = op_type; + } + + CPublisherSpan::~CPublisherSpan() + { + auto now = system_clock::now(); + data.end_ns = duration_cast(now.time_since_epoch()).count(); + if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); + } + + // Receive span constructor + CSubscriberSpan::CSubscriberSpan(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + { + auto now = system_clock::now(); + data.start_ns = duration_cast(now.time_since_epoch()).count(); + data.entity_id = entity_id; + data.topic_id = topic_info.topic_id; + data.process_id = topic_info.process_id; + data.payload_size = payload_size; + data.clock = clock; + data.layer = layer; + data.op_type = op_type; + } + + CSubscriberSpan::~CSubscriberSpan() + { + auto now = system_clock::now(); + data.end_ns = duration_cast(now.time_since_epoch()).count(); + if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); + } + } +} diff --git a/ecal/core/src/tracing/span.h b/ecal/core/src/tracing/span.h new file mode 100644 index 0000000000..06f6187055 --- /dev/null +++ b/ecal/core/src/tracing/span.h @@ -0,0 +1,69 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include "tracing.h" + +#include +#include +#include + +namespace eCAL +{ + namespace tracing + { + // RAII span for send (publisher) operations. + // Records start_ns on construction, end_ns + buffer on destruction. + class CPublisherSpan + { + public: + CPublisherSpan(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + + ~CPublisherSpan(); + + CPublisherSpan(const CPublisherSpan&) = delete; + CPublisherSpan& operator=(const CPublisherSpan&) = delete; + CPublisherSpan(CPublisherSpan&&) = delete; + CPublisherSpan& operator=(CPublisherSpan&&) = delete; + + private: + SPublisherSpanData data{}; + }; + + // RAII span for receive (subscriber) operations. + // Records start_ns on construction, end_ns + buffer on destruction. + class CSubscriberSpan + { + public: + CSubscriberSpan(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + + ~CSubscriberSpan(); + + CSubscriberSpan(const CSubscriberSpan&) = delete; + CSubscriberSpan& operator=(const CSubscriberSpan&) = delete; + CSubscriberSpan(CSubscriberSpan&&) = delete; + CSubscriberSpan& operator=(CSubscriberSpan&&) = delete; + + private: + SSubscriberSpanData data{}; + }; + } +} diff --git a/ecal/core/src/tracing/trace_provider.cpp b/ecal/core/src/tracing/trace_provider.cpp new file mode 100644 index 0000000000..71b5cd3a90 --- /dev/null +++ b/ecal/core/src/tracing/trace_provider.cpp @@ -0,0 +1,38 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "trace_provider.h" +#include "trace_provider_default.h" +#include "trace_provider_noop.h" + +namespace eCAL +{ + namespace tracing + { + std::shared_ptr TraceProvider::Create(const eCAL::Tracing::Configuration& config_) + { + if (config_.enabled) + { + return CTraceProviderDefault::Create(); + } + return std::make_shared(); + } + } +} diff --git a/ecal/core/src/tracing/trace_provider.h b/ecal/core/src/tracing/trace_provider.h new file mode 100644 index 0000000000..98df533c62 --- /dev/null +++ b/ecal/core/src/tracing/trace_provider.h @@ -0,0 +1,50 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include "tracing.h" + +#include + +#include + +namespace eCAL +{ + namespace tracing + { + class TraceProvider + { + public: + TraceProvider() = default; + virtual ~TraceProvider() = default; + + TraceProvider(const TraceProvider&) = delete; + TraceProvider& operator=(const TraceProvider&) = delete; + TraceProvider(TraceProvider&&) = delete; + TraceProvider& operator=(TraceProvider&&) = delete; + + static std::shared_ptr Create(const eCAL::Tracing::Configuration& config_); + + virtual void WriteSpan(const SpanDataVariant& span_data) = 0; + virtual void WriteMetadata(const STopicMetadata& metadata) = 0; + }; + } +} diff --git a/ecal/core/src/tracing/trace_provider_default.cpp b/ecal/core/src/tracing/trace_provider_default.cpp new file mode 100644 index 0000000000..1627d80c5e --- /dev/null +++ b/ecal/core/src/tracing/trace_provider_default.cpp @@ -0,0 +1,102 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "trace_provider_default.h" +#include "tracing_writer.h" +#include "tracing_writer_jsonl.h" +#include "util/single_instance_helper.h" + +#include +#include +#include + +namespace eCAL +{ + namespace tracing + { + std::shared_ptr CTraceProviderDefault::Create(std::unique_ptr writer, size_t batch_size) + { + try + { + return Util::CSingleInstanceHelper::Create(std::move(writer), batch_size); + } + catch (const std::exception& e) + { + return nullptr; + } + } + + CTraceProviderDefault::CTraceProviderDefault(std::unique_ptr writer, size_t batch_size) + : batch_size_(batch_size), writer_(std::move(writer)) + { + writer_thread_ = std::thread(&CTraceProviderDefault::WriterThreadLoop, this); + } + + CTraceProviderDefault::~CTraceProviderDefault() + { + { + std::lock_guard lock(thread_mutex); + stop_thread_ = true; + write_cv_.notify_all(); + } + writer_thread_.join(); + } + + void CTraceProviderDefault::WriteSpan(const SpanDataVariant& span_data) + { + std::lock_guard lock(thread_mutex); + span_buffer_.push_back(span_data); + if (span_buffer_.size() >= batch_size_) + { + write_cv_.notify_one(); + } + } + + void CTraceProviderDefault::WriterThreadLoop() + { + std::vector span_flusher; + while (true) + { + { + std::unique_lock lock(thread_mutex); + write_cv_.wait(lock, [this]() + { + return stop_thread_ || (span_buffer_.size() >= batch_size_); + }); + if (stop_thread_ && span_buffer_.empty()) + { + break; + } + span_flusher.swap(span_buffer_); + } + if (!span_flusher.empty()) + { + writer_->WriteSpansToFile(span_flusher); + span_flusher.clear(); + } + } + } + + void CTraceProviderDefault::WriteMetadata(const STopicMetadata& metadata) + { + writer_->WriteMetadataToFile(metadata); + } + } +} diff --git a/ecal/core/src/tracing/trace_provider_default.h b/ecal/core/src/tracing/trace_provider_default.h new file mode 100644 index 0000000000..4b14529c07 --- /dev/null +++ b/ecal/core/src/tracing/trace_provider_default.h @@ -0,0 +1,73 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include "trace_provider.h" +#include "tracing.h" +#include "tracing_writer.h" +#include "tracing_writer_jsonl.h" +#include "util/single_instance_helper.h" + +#include +#include +#include +#include +#include +#include + +namespace eCAL +{ + namespace tracing + { + class CTraceProviderDefault : public TraceProvider + { + friend class Util::CSingleInstanceHelper; + + public: + static std::shared_ptr Create(std::unique_ptr writer = std::make_unique(), size_t batch_size = kDefaultTracingBatchSize); + + CTraceProviderDefault(const CTraceProviderDefault&) = delete; + CTraceProviderDefault& operator=(const CTraceProviderDefault&) = delete; + CTraceProviderDefault(CTraceProviderDefault&&) = delete; + CTraceProviderDefault& operator=(CTraceProviderDefault&&) = delete; + + ~CTraceProviderDefault() override; + + // Write span data to buffer (accepts any span type via variant) + void WriteSpan(const SpanDataVariant& span_data) override; + + // metadata — written directly to file (no buffering) + void WriteMetadata(const STopicMetadata& metadata) override; + + private: + CTraceProviderDefault(std::unique_ptr writer, size_t batch_size); + void WriterThreadLoop(); + + std::atomic batch_size_{kDefaultTracingBatchSize}; + std::vector span_buffer_; + mutable std::mutex thread_mutex; + std::condition_variable write_cv_; + bool stop_thread_{false}; + std::thread writer_thread_; + std::unique_ptr writer_; + }; + } +} diff --git a/ecal/core/src/tracing/trace_provider_noop.h b/ecal/core/src/tracing/trace_provider_noop.h new file mode 100644 index 0000000000..3e696e98ea --- /dev/null +++ b/ecal/core/src/tracing/trace_provider_noop.h @@ -0,0 +1,36 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include "trace_provider.h" + +namespace eCAL +{ + namespace tracing + { + class CNoOpTraceProvider : public TraceProvider + { + public: + void WriteSpan(const SpanDataVariant& /*span_data*/) override {} + void WriteMetadata(const STopicMetadata& /*metadata*/) override {} + }; + } +} diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h new file mode 100644 index 0000000000..829c09a31a --- /dev/null +++ b/ecal/core/src/tracing/tracing.h @@ -0,0 +1,128 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +/** + * @file tracing/tracing.h + * @brief Shared tracing types, enums, and constants used across the tracing subsystem. +**/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace eCAL +{ + namespace tracing + { + // Version of the tracing implementation. + constexpr const char* kTracingVersion = "1.0.0"; + + // Default batch size for span buffering before flushing to backend (jsonl file) + constexpr size_t kDefaultTracingBatchSize = 10; + + // Specifies the type of operation being traced + enum operation_type + { + send = 0, + receive = 1, + callback_execution = 2 + }; + + // Specifies the direction of the topic (publisher or subscriber) + enum topic_direction + { + publisher = 0, + subscriber = 1 + }; + + // Bitmask enum for active transport layers used in tracing spans. + // These use power-of-two values so combinations can be expressed with bitwise OR. + enum eTracingLayerType : uint64_t + { + tl_trace_none = 0, + tl_trace_shm = 1 << 0, // 1 + tl_trace_udp = 1 << 1, // 2 + tl_trace_tcp = 1 << 2, // 4 + tl_trace_shm_udp = tl_trace_shm | tl_trace_udp, // 3 + tl_trace_shm_tcp = tl_trace_shm | tl_trace_tcp, // 5 + tl_trace_udp_tcp = tl_trace_udp | tl_trace_tcp, // 6 + tl_trace_all = tl_trace_shm | tl_trace_udp | tl_trace_tcp, // 7 + }; + + // Convert from eTLayerType (used in core APIs) to eTracingLayerType. + inline eTracingLayerType toTracingLayerType(eTLayerType layer) + { + switch (layer) + { + case tl_ecal_shm: return tl_trace_shm; + case tl_ecal_udp: return tl_trace_udp; + case tl_ecal_tcp: return tl_trace_tcp; + case tl_all: return tl_trace_all; + default: return tl_trace_none; + } + } + + // Metadata captured when a topic is created + struct STopicMetadata + { + std::string tracing_version{kTracingVersion}; // tracing format version + uint64_t entity_id; // unique entity id + int32_t process_id; // PID of the owning process + std::string host_name; // host that created the topic + std::string topic_name; // topic name used for pub/sub matching + std::string encoding; // datatype encoding (e.g. protobuf) + std::string type_name; // datatype name + topic_direction direction; // publisher or subscriber + }; + + struct SPublisherSpanData + { + operation_type op_type; + uint64_t entity_id; + uint64_t process_id; + size_t payload_size; + long long clock; + uint64_t layer; + long long start_ns; // start timestamp in nanoseconds + long long end_ns; // end timestamp in nanoseconds + }; + + struct SSubscriberSpanData + { + operation_type op_type; + uint64_t entity_id; + uint64_t topic_id; + uint64_t process_id; + size_t payload_size; + long long clock; + uint64_t layer; + long long start_ns; // start timestamp in nanoseconds + long long end_ns; // end timestamp in nanoseconds + }; + + // Variant type for buffering heterogeneous span data. + // Extend this variant when new span types are added (e.g. client/server). + using SpanDataVariant = std::variant; + } +} diff --git a/ecal/core/src/tracing/tracing_writer.h b/ecal/core/src/tracing/tracing_writer.h new file mode 100644 index 0000000000..7ff26e0332 --- /dev/null +++ b/ecal/core/src/tracing/tracing_writer.h @@ -0,0 +1,45 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include +#include + +#include "tracing.h" + +namespace eCAL +{ + namespace tracing + { + // Interface for tracing writers. + class TracingWriter + { + public: + virtual ~TracingWriter() = default; + + // Write a batch of spans (heterogeneous via variant) + virtual void WriteSpansToFile(const std::vector& batch) = 0; + + // Write a single topic metadata entry + virtual void WriteMetadataToFile(const STopicMetadata& metadata) = 0; + }; + } +} diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp new file mode 100644 index 0000000000..8a35ea235d --- /dev/null +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -0,0 +1,154 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "tracing_writer_jsonl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +using json = nlohmann::json; + +namespace eCAL +{ + namespace tracing + { + static std::string GetCurrentTimestamp() + { + std::time_t now = std::time(nullptr); + std::tm* tm_info = std::localtime(&now); + std::ostringstream oss; + oss << std::put_time(tm_info, "%Y%m%d_%H%M%S"); + return oss.str(); + } + + CTracingWriterJSONL::CTracingWriterJSONL() + : timestamp_(GetCurrentTimestamp()) + , trace_dir_(eCAL::Util::GeteCALTraceDir()) + {} + + std::string CTracingWriterJSONL::GetSpansFilePath() const + { + return trace_dir_ + "/ecal_spans_" + std::to_string(eCAL::Process::GetProcessID()) + "_" + timestamp_ + ".jsonl"; + } + + std::string CTracingWriterJSONL::GetTopicMetadataFilePath() const + { + return trace_dir_ + "/ecal_topic_metadata_" + std::to_string(eCAL::Process::GetProcessID()) + "_" + timestamp_ + ".jsonl"; + } + + void CTracingWriterJSONL::WriteSpansToFile(const std::vector& batch) + { + try + { + std::lock_guard lock(spans_mutex_); + std::string filepath = GetSpansFilePath(); + + std::ofstream output_file(filepath, std::ios::app); + if (output_file.is_open()) + { + for (const auto& span_variant : batch) + { + json span_obj; + std::visit([&span_obj](const auto& span) + { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + span_obj["entity_id"] = span.entity_id; + span_obj["process_id"] = span.process_id; + span_obj["payload_size"] = span.payload_size; + span_obj["clock"] = span.clock; + span_obj["layer"] = span.layer; + span_obj["start_ns"] = span.start_ns; + span_obj["end_ns"] = span.end_ns; + span_obj["op_type"] = span.op_type; + } + else if constexpr (std::is_same_v) + { + span_obj["entity_id"] = span.entity_id; + span_obj["topic_id"] = span.topic_id; + span_obj["process_id"] = span.process_id; + span_obj["payload_size"] = span.payload_size; + span_obj["clock"] = span.clock; + span_obj["layer"] = span.layer; + span_obj["start_ns"] = span.start_ns; + span_obj["end_ns"] = span.end_ns; + span_obj["op_type"] = span.op_type; + } + }, span_variant); + output_file << span_obj.dump() << "\n"; + } + output_file.close(); + } + else + { + std::cerr << "Warning: Could not open spans file: " << filepath << std::endl; + } + } + catch (const std::exception& e) + { + std::cerr << "Error writing spans to JSONL: " << e.what() << std::endl; + } + } + + void CTracingWriterJSONL::WriteMetadataToFile(const STopicMetadata& metadata) + { + try + { + std::lock_guard lock(metadata_mutex_); + std::string filepath = GetTopicMetadataFilePath(); + + json obj; + obj["tracing_version"] = metadata.tracing_version; + obj["entity_id"] = metadata.entity_id; + obj["process_id"] = metadata.process_id; + obj["host_name"] = metadata.host_name; + obj["topic_name"] = metadata.topic_name; + obj["encoding"] = metadata.encoding; + obj["type_name"] = metadata.type_name; + obj["direction"] = (metadata.direction == topic_direction::publisher) ? "publisher" : "subscriber"; + + std::ofstream output_file(filepath, std::ios::app); + if (output_file.is_open()) + { + output_file << obj.dump() << "\n"; + output_file.close(); + } + else + { + std::cerr << "Warning: Could not open topic metadata file: " << filepath << std::endl; + } + } + catch (const std::exception& e) + { + std::cerr << "Error writing topic metadata to JSONL: " << e.what() << std::endl; + } + } + } +} diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.h b/ecal/core/src/tracing/tracing_writer_jsonl.h new file mode 100644 index 0000000000..a09c77ca9a --- /dev/null +++ b/ecal/core/src/tracing/tracing_writer_jsonl.h @@ -0,0 +1,64 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include +#include +#include + +#include "tracing_writer.h" +#include "tracing.h" + +namespace eCAL +{ + namespace tracing + { + // Responsible for serializing span and metadata to JSONL files. + // Separated from CTraceProvider to isolate the I/O concern. + class CTracingWriterJSONL : public TracingWriter + { + public: + CTracingWriterJSONL(); + ~CTracingWriterJSONL() override = default; + + CTracingWriterJSONL(const CTracingWriterJSONL&) = delete; + CTracingWriterJSONL& operator=(const CTracingWriterJSONL&) = delete; + CTracingWriterJSONL(CTracingWriterJSONL&&) = delete; + CTracingWriterJSONL& operator=(CTracingWriterJSONL&&) = delete; + + // Write a batch of spans to the JSONL spans file + void WriteSpansToFile(const std::vector& batch) override; + + // Write a single topic metadata entry to the JSONL metadata file + void WriteMetadataToFile(const STopicMetadata& metadata) override; + + // File path accessors (path is fixed at construction time) + std::string GetSpansFilePath() const; + std::string GetTopicMetadataFilePath() const; + + private: + mutable std::mutex spans_mutex_; + mutable std::mutex metadata_mutex_; + std::string timestamp_; + std::string trace_dir_; + }; + } +} diff --git a/ecal/tests/CMakeLists.txt b/ecal/tests/CMakeLists.txt index 298e02e01e..438d79fbdb 100644 --- a/ecal/tests/CMakeLists.txt +++ b/ecal/tests/CMakeLists.txt @@ -27,7 +27,7 @@ add_subdirectory(cpp/process_test) add_subdirectory(cpp/descgate_test) add_subdirectory(cpp/config_test) - +add_subdirectory(cpp/tracing_test) if(ECAL_CORE_REGISTRATION) add_subdirectory(cpp/registration_test) diff --git a/ecal/tests/cpp/config_test/src/path_processing_test.cpp b/ecal/tests/cpp/config_test/src/path_processing_test.cpp index 531cf18135..f91068c0df 100644 --- a/ecal/tests/cpp/config_test/src/path_processing_test.cpp +++ b/ecal/tests/cpp/config_test/src/path_processing_test.cpp @@ -444,3 +444,68 @@ TEST(core_cpp_dir_manager /*unused*/, can_write_to_directory_unicode_path /*unus std::filesystem::remove_all(unicode_dir_path, ec); } #endif /* ECAL_OS_WINDOWS */ + + +TEST(core_cpp_path_processing /*unused*/, ecal_trace_order_test /*unused*/) +{ + const std::string ecal_trace_env_var = "/ecal/trace/env"; + const std::string ecal_data_env_var = "/ecal/data/env"; + const std::string ecal_data_env_trace_var = ecal_data_env_var + path_separator + ECAL_FOLDER_NAME_TRACE; + const std::string ecal_yaml_dir = "/dir/to/current/yaml"; + const std::string ecal_yaml_trace_dir = ecal_yaml_dir + path_separator + ECAL_FOLDER_NAME_TRACE; + const std::string unique_tmp_dir = "/tmp/unique"; + + const MockDirProvider mock_dir_provider; + const NiceMock mock_dir_manager; + + EXPECT_CALL(mock_dir_provider, eCALEnvVar(ECAL_TRACE_VAR)) + .Times(6) + .WillOnce(testing::Return(ecal_trace_env_var)) + .WillRepeatedly(testing::Return("")); + EXPECT_CALL(mock_dir_provider, eCALEnvVar(ECAL_DATA_VAR)) + .Times(6) + .WillRepeatedly(testing::Return(ecal_data_env_var)); + + EXPECT_CALL(mock_dir_manager, getDirectoryPath(testing::_)) + .Times(6) + .WillRepeatedly(testing::Return(ecal_yaml_dir)); + + EXPECT_CALL(mock_dir_manager, dirExists(ecal_trace_env_var)) + .Times(1) + .WillOnce(testing::Return(true)); + EXPECT_CALL(mock_dir_manager, dirExists(ecal_data_env_trace_var)) + .Times(5) + .WillOnce(testing::Return(true)) + .WillRepeatedly(testing::Return(false)); + EXPECT_CALL(mock_dir_manager, dirExists(ecal_data_env_var)) + .Times(4) + .WillOnce(testing::Return(true)) + .WillRepeatedly(testing::Return(false)); + + EXPECT_CALL(mock_dir_manager, dirExists(ecal_yaml_trace_dir)) + .Times(3) + .WillOnce(testing::Return(true)) + .WillRepeatedly(testing::Return(false)); + EXPECT_CALL(mock_dir_manager, dirExists(ecal_yaml_dir)) + .Times(2) + .WillOnce(testing::Return(true)) + .WillRepeatedly(testing::Return(false)); + EXPECT_CALL(mock_dir_manager, dirExistsOrCreate(testing::_)) + .Times(0); + + EXPECT_CALL(mock_dir_provider, uniqueTmpDir(::testing::Ref(mock_dir_manager))) + .Times(1) + .WillRepeatedly(testing::Return(unique_tmp_dir)); + + ON_CALL(mock_dir_manager, dirExists(testing::_)).WillByDefault(testing::Return(false)); + ON_CALL(mock_dir_manager, canWriteToDirectory(testing::_)).WillByDefault(testing::Return(true)); + + auto config = eCAL::GetConfiguration(); + + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), ecal_trace_env_var); + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), ecal_data_env_trace_var); + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), ecal_data_env_var); + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), ecal_yaml_trace_dir); + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), ecal_yaml_dir); + EXPECT_EQ(eCAL::Config::GeteCALTraceDirImpl(mock_dir_provider, mock_dir_manager, config), unique_tmp_dir); +} diff --git a/ecal/tests/cpp/config_test/src/yaml_processing_test.cpp b/ecal/tests/cpp/config_test/src/yaml_processing_test.cpp index ef05d22690..cad63b057a 100644 --- a/ecal/tests/cpp/config_test/src/yaml_processing_test.cpp +++ b/ecal/tests/cpp/config_test/src/yaml_processing_test.cpp @@ -91,6 +91,8 @@ TEST(core_cpp_config_yaml /*unused*/, yaml_processing_comparison /*unused*/) config.application.startup.terminal_emulator = "term_emulator"; config.application.sys.filter_excl = "filter_excl"; + config.tracing.enabled = true; + config.logging.provider.console.enable = false; config.logging.provider.console.log_level = eCAL::Logging::eLogLevel::log_level_debug1; config.logging.provider.file.enable = true; @@ -161,6 +163,7 @@ TEST(core_cpp_config_yaml /*unused*/, yaml_processing_comparison /*unused*/) EXPECT_EQ(config.logging.provider.udp_config.port, config_from_yaml.logging.provider.udp_config.port); EXPECT_EQ(config.logging.receiver.enable, config_from_yaml.logging.receiver.enable); EXPECT_EQ(config.logging.receiver.udp_config.port, config_from_yaml.logging.receiver.udp_config.port); + EXPECT_EQ(config.tracing.enabled, config_from_yaml.tracing.enabled); auto yaml_from_config = YAML::Node(config); eCAL::Configuration config_from_yaml_config = yaml_from_config.as(); @@ -220,6 +223,7 @@ TEST(core_cpp_config_yaml /*unused*/, yaml_processing_comparison /*unused*/) EXPECT_EQ(config.logging.provider.udp_config.port, config_from_yaml_config.logging.provider.udp_config.port); EXPECT_EQ(config.logging.receiver.enable, config_from_yaml_config.logging.receiver.enable); EXPECT_EQ(config.logging.receiver.udp_config.port, config_from_yaml_config.logging.receiver.udp_config.port); + EXPECT_EQ(config.tracing.enabled, config_from_yaml_config.tracing.enabled); } TEST(core_cpp_config /*unused*/, read_write_file_test /*unused*/) diff --git a/ecal/tests/cpp/tracing_test/CMakeLists.txt b/ecal/tests/cpp/tracing_test/CMakeLists.txt new file mode 100644 index 0000000000..4953253852 --- /dev/null +++ b/ecal/tests/cpp/tracing_test/CMakeLists.txt @@ -0,0 +1,58 @@ +# ========================= eCAL LICENSE ================================= +# +# Copyright (C) 2016 - 2025 Continental Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================= eCAL LICENSE ================================= + +project(test_tracing) + +find_package(Threads REQUIRED) +find_package(GTest REQUIRED) +find_package(nlohmann_json REQUIRED) + +set(tracing_test_src + src/tracing_test_helpers.h + src/trace_provider_test.cpp + src/tracing_writer_test.cpp +) + +ecal_add_gtest(${PROJECT_NAME} ${tracing_test_src}) + +target_include_directories(${PROJECT_NAME} PRIVATE + $ + ${CMAKE_CURRENT_LIST_DIR}/src + ${ECAL_CORE_PROJECT_ROOT}/core/src + ${ECAL_CORE_PROJECT_ROOT}/core/src/serialization +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + ecal_core_private + eCAL::ecal-utils + Threads::Threads + nlohmann_json::nlohmann_json +) + +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) + +target_compile_definitions(${PROJECT_NAME} PRIVATE ECAL_CORE_COMMAND_LINE) + +ecal_install_gtest(${PROJECT_NAME}) + +set_property(TARGET ${PROJECT_NAME} PROPERTY FOLDER tests/cpp/tracing) + +source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" FILES + ${tracing_test_src} +) diff --git a/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp new file mode 100644 index 0000000000..eed25aad91 --- /dev/null +++ b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp @@ -0,0 +1,127 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "tracing_test_helpers.h" + +#include +#include + +#include + +#include + +#include +#include +#include + +TEST(TestTraceProvider, ConcurrentSpanWrites) +{ + constexpr size_t num_threads = 100; + constexpr size_t spans_per_thread = 100; + constexpr size_t total_spans = num_threads * spans_per_thread; // 10000 + constexpr size_t batch_size = 500; // Flush every 500 spans + + MockTracingWriter mock_writer; + + auto provider = eCAL::tracing::CTraceProviderDefault::Create( + std::make_unique(mock_writer), batch_size); + ASSERT_NE(provider, nullptr); + + Barrier barrier(num_threads); + + std::vector threads; + threads.reserve(num_threads); + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t]() + { + barrier.wait(); + for (size_t i = 0; i < spans_per_thread; ++i) + { + eCAL::tracing::SPublisherSpanData span{}; + span.op_type = eCAL::tracing::operation_type::send; + span.entity_id = static_cast(t * spans_per_thread + i); + span.process_id = 1; + span.payload_size = 42; + span.clock = static_cast(i); + span.layer = eCAL::tracing::tl_trace_shm; + span.start_ns = 1000 + static_cast(i); + span.end_ns = 2000 + static_cast(i); + provider->WriteSpan(span); + } + }); + } + + // Join all writer threads. + for (auto& th : threads) + th.join(); + + // Destroy the provider to flush any remaining buffered spans. + provider.reset(); + + // Every span must be written exactly once — no duplicates, no drops. + EXPECT_EQ(mock_writer.SpanCount(), total_spans); +} + +TEST(TestTraceProvider, ConcurrentMetadataWrites) +{ + constexpr size_t num_threads = 100; + constexpr size_t metadata_per_thread = 100; + constexpr size_t total_metadata = num_threads * metadata_per_thread; // 10000 + constexpr size_t batch_size = 500; + + MockTracingWriter mock_writer; + + auto provider = eCAL::tracing::CTraceProviderDefault::Create( + std::make_unique(mock_writer), batch_size); + ASSERT_NE(provider, nullptr); + + Barrier barrier(num_threads); + + std::vector threads; + threads.reserve(num_threads); + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t]() + { + barrier.wait(); + for (size_t i = 0; i < metadata_per_thread; ++i) + { + eCAL::tracing::STopicMetadata metadata{}; + metadata.entity_id = static_cast(t * metadata_per_thread + i); + metadata.process_id = 1; + metadata.host_name = "test_host"; + metadata.topic_name = "topic_" + std::to_string(t) + "_" + std::to_string(i); + metadata.encoding = "protobuf"; + metadata.type_name = "TestType"; + metadata.direction = eCAL::tracing::topic_direction::publisher; + provider->WriteMetadata(metadata); + } + }); + } + + for (auto& th : threads) + th.join(); + + provider.reset(); + + EXPECT_EQ(mock_writer.MetadataCount(), total_metadata); +} \ No newline at end of file diff --git a/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h new file mode 100644 index 0000000000..cc86564517 --- /dev/null +++ b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h @@ -0,0 +1,154 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +class ScopedTraceDirOverride +{ +public: + explicit ScopedTraceDirOverride(const std::string& path) + { + if (const char* previous_value = std::getenv(kTraceDirEnvVar)) + { + had_previous_value_ = true; + previous_value_ = previous_value; + } + +#ifdef _WIN32 + _putenv_s(kTraceDirEnvVar, path.c_str()); +#else + setenv(kTraceDirEnvVar, path.c_str(), 1); +#endif + } + + ~ScopedTraceDirOverride() + { +#ifdef _WIN32 + _putenv_s(kTraceDirEnvVar, had_previous_value_ ? previous_value_.c_str() : ""); +#else + if (had_previous_value_) + setenv(kTraceDirEnvVar, previous_value_.c_str(), 1); + else + unsetenv(kTraceDirEnvVar); +#endif + } + +private: + static constexpr const char* kTraceDirEnvVar = "ECAL_TRACE_DIR"; + + bool had_previous_value_{false}; + std::string previous_value_; +}; + +inline eCAL::Configuration GetTracingConfiguration() +{ + return eCAL::Configuration{}; +} + +// Mock writer that counts spans and metadata for test assertions. +class MockTracingWriter : public eCAL::tracing::TracingWriter +{ +public: + void WriteSpansToFile(const std::vector& batch) override + { + span_count_ += batch.size(); + } + + void WriteMetadataToFile(const eCAL::tracing::STopicMetadata& /*metadata*/) override + { + metadata_count_++; + } + + size_t SpanCount() const + { + return span_count_.load(); + } + + size_t MetadataCount() const + { + return metadata_count_.load(); + } + + void Clear() + { + span_count_ = 0; + metadata_count_ = 0; + + } + +private: + mutable std::mutex mutex_; + std::atomic span_count_{0}; + std::atomic metadata_count_{0}; +}; + +// This allows the mock to outlive the CTraceProvider that only owns this proxy. +class ProxyTracingWriter : public eCAL::tracing::TracingWriter +{ +public: + explicit ProxyTracingWriter(MockTracingWriter& target) : target_(target) {} + + void WriteSpansToFile(const std::vector& batch) override + { + target_.WriteSpansToFile(batch); + } + + void WriteMetadataToFile(const eCAL::tracing::STopicMetadata& metadata) override + { + target_.WriteMetadataToFile(metadata); + } + +private: + MockTracingWriter& target_; +}; + +// Count the number of lines in a file and validate each line is valid JSON. +inline size_t CountAndValidateJsonlLines(const std::string& filepath) +{ + std::ifstream file(filepath); + EXPECT_TRUE(file.is_open()) << "Failed to open: " << filepath; + + size_t count = 0; + std::string line; + while (std::getline(file, line)) + { + if (line.empty()) continue; + // Every line must be valid JSON — a corrupted/interleaved write would break parsing. + EXPECT_NO_THROW(nlohmann::json::parse(line)) + << "Invalid JSON on line " << (count + 1) << ": " << line; + ++count; + } + return count; +} + diff --git a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp new file mode 100644 index 0000000000..1a8c2203df --- /dev/null +++ b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp @@ -0,0 +1,251 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include "tracing_test_helpers.h" + +#include + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +TEST(TestTracingWriterJSONL, ConcurrentSpanWrites) +{ + constexpr size_t num_threads = 100; + constexpr size_t batches_per_thread = 50; + constexpr size_t spans_per_batch = 500; + constexpr size_t total_spans = num_threads * batches_per_thread * spans_per_batch; + ScopedTraceDirOverride trace_dir_override("./"); + auto ecal_config = GetTracingConfiguration(); + ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + + std::string spans_path; + + { + eCAL::tracing::CTracingWriterJSONL writer; + spans_path = writer.GetSpansFilePath(); + + Barrier barrier(num_threads); + + std::vector threads; + threads.reserve(num_threads); + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t]() + { + barrier.wait(); + + for (size_t b = 0; b < batches_per_thread; ++b) + { + std::vector batch; + batch.reserve(spans_per_batch); + + for (size_t i = 0; i < spans_per_batch; ++i) + { + eCAL::tracing::SPublisherSpanData span{}; + span.op_type = eCAL::tracing::operation_type::send; + span.entity_id = static_cast(t * batches_per_thread * spans_per_batch + b * spans_per_batch + i); + span.process_id = 1; + span.payload_size = 64; + span.clock = static_cast(i); + span.layer = eCAL::tracing::tl_trace_shm; + span.start_ns = 1000; + span.end_ns = 2000; + batch.push_back(span); + } + writer.WriteSpansToFile(batch); + } + }); + } + + for (auto& th : threads) th.join(); + } + + eCAL::Finalize(); + + // Verify: every span line must be valid JSON and the total count must match. + size_t line_count = CountAndValidateJsonlLines(spans_path); + EXPECT_EQ(line_count, total_spans); + + if (!spans_path.empty()) std::remove(spans_path.c_str()); +} + +TEST(TestTracingWriterJSONL, ConcurrentMetadataWrites) +{ + constexpr size_t num_threads = 100; + constexpr size_t metadata_per_thread = 200; + constexpr size_t total_metadata = num_threads * metadata_per_thread; + ScopedTraceDirOverride trace_dir_override("./"); + auto ecal_config = GetTracingConfiguration(); + ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + + std::string metadata_path; + + { + eCAL::tracing::CTracingWriterJSONL writer; + metadata_path = writer.GetTopicMetadataFilePath(); + + Barrier barrier(num_threads); + + std::vector threads; + threads.reserve(num_threads); + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t]() + { + barrier.wait(); + + for (size_t i = 0; i < metadata_per_thread; ++i) + { + eCAL::tracing::STopicMetadata metadata{}; + metadata.entity_id = static_cast(t * metadata_per_thread + i); + metadata.process_id = 1; + metadata.host_name = "test_host"; + metadata.topic_name = "topic_" + std::to_string(t) + "_" + std::to_string(i); + metadata.encoding = "protobuf"; + metadata.type_name = "TestType"; + metadata.direction = eCAL::tracing::topic_direction::publisher; + writer.WriteMetadataToFile(metadata); + } + }); + } + + for (auto& th : threads) th.join(); + } + + eCAL::Finalize(); + + size_t line_count = CountAndValidateJsonlLines(metadata_path); + EXPECT_EQ(line_count, total_metadata); + + if (!metadata_path.empty()) std::remove(metadata_path.c_str()); +} + +TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) +{ + ScopedTraceDirOverride trace_dir_override("./"); + auto ecal_config = GetTracingConfiguration(); + ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + + std::string spans_path; + + { + eCAL::tracing::CTracingWriterJSONL writer; + spans_path = writer.GetSpansFilePath(); + + eCAL::tracing::SPublisherSpanData span{}; + span.op_type = eCAL::tracing::operation_type::send; + span.entity_id = 42; + span.process_id = 123; + span.payload_size = 256; + span.clock = 7; + span.layer = eCAL::tracing::tl_trace_udp; + span.start_ns = 1000; + span.end_ns = 2000; + + std::vector batch; + batch.push_back(span); + writer.WriteSpansToFile(batch); + } + + eCAL::Finalize(); + + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); + + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); + + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); + EXPECT_EQ(j.at("entity_id"), 42u); + EXPECT_EQ(j.at("process_id"), 123u); + EXPECT_EQ(j.at("payload_size"), 256u); + EXPECT_EQ(j.at("clock"), 7); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); + EXPECT_EQ(j.at("start_ns"), 1000); + EXPECT_EQ(j.at("end_ns"), 2000); + + if (!spans_path.empty()) std::remove(spans_path.c_str()); +} + +TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) +{ + ScopedTraceDirOverride trace_dir_override("./"); + auto ecal_config = GetTracingConfiguration(); + ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + + std::string spans_path; + + { + eCAL::tracing::CTracingWriterJSONL writer; + spans_path = writer.GetSpansFilePath(); + + eCAL::tracing::SSubscriberSpanData span{}; + span.op_type = eCAL::tracing::operation_type::receive; + span.entity_id = 99; + span.topic_id = 55; + span.process_id = 456; + span.payload_size = 512; + span.clock = 3; + span.layer = eCAL::tracing::tl_trace_tcp; + span.start_ns = 3000; + span.end_ns = 4000; + + std::vector batch; + batch.push_back(span); + writer.WriteSpansToFile(batch); + } + + eCAL::Finalize(); + + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); + + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); + + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); + EXPECT_EQ(j.at("entity_id"), 99u); + EXPECT_EQ(j.at("topic_id"), 55u); + EXPECT_EQ(j.at("process_id"), 456u); + EXPECT_EQ(j.at("payload_size"), 512u); + EXPECT_EQ(j.at("clock"), 3); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); + EXPECT_EQ(j.at("start_ns"), 3000); + EXPECT_EQ(j.at("end_ns"), 4000); + + if (!spans_path.empty()) std::remove(spans_path.c_str()); +} + diff --git a/thirdparty/nlohmann_json/build-nlohmann_json.cmake b/thirdparty/nlohmann_json/build-nlohmann_json.cmake new file mode 100644 index 0000000000..ae2984d190 --- /dev/null +++ b/thirdparty/nlohmann_json/build-nlohmann_json.cmake @@ -0,0 +1,13 @@ +include_guard(GLOBAL) +include(FetchContent) + +FetchContent_Declare( + nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d +) + +set(JSON_BuildTests OFF CACHE BOOL "" FORCE) +set(JSON_Install OFF CACHE BOOL "" FORCE) + +FetchContent_MakeAvailable(nlohmann_json) From cc260377400901ea5771bd9f1172ee6bf76fa83c Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Tue, 19 May 2026 17:28:15 +0200 Subject: [PATCH 02/12] Some code review remarks. --- ecal/core/src/ecal.cpp | 2 +- ecal/core/src/tracing/span.cpp | 11 ++++++----- ecal/core/src/tracing/tracing.h | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ecal/core/src/ecal.cpp b/ecal/core/src/ecal.cpp index 032f383885..b6c5611d68 100644 --- a/ecal/core/src/ecal.cpp +++ b/ecal/core/src/ecal.cpp @@ -165,8 +165,8 @@ namespace eCAL ResetGlobalEcalConfiguration(); - ResetLogging(); ResetTracing(); + ResetLogging(); return finalized; } diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp index 506e9f3478..34ed220e11 100644 --- a/ecal/core/src/tracing/span.cpp +++ b/ecal/core/src/tracing/span.cpp @@ -20,10 +20,11 @@ #include "span.h" #include "trace_provider.h" -#include "../ecal_global_accessors.h" +#include "ecal_global_accessors.h" #include +using tracing_clock = std::chrono::steady_clock; using namespace std::chrono; namespace eCAL @@ -33,7 +34,7 @@ namespace eCAL // Send span constructor CPublisherSpan::CPublisherSpan(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) { - auto now = system_clock::now(); + auto now = tracing_clock::now(); data.start_ns = duration_cast(now.time_since_epoch()).count(); data.entity_id = topic_id.topic_id.entity_id; data.process_id = topic_id.topic_id.process_id; @@ -45,7 +46,7 @@ namespace eCAL CPublisherSpan::~CPublisherSpan() { - auto now = system_clock::now(); + auto now = tracing_clock::now(); data.end_ns = duration_cast(now.time_since_epoch()).count(); if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); } @@ -53,7 +54,7 @@ namespace eCAL // Receive span constructor CSubscriberSpan::CSubscriberSpan(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) { - auto now = system_clock::now(); + auto now = tracing_clock::now(); data.start_ns = duration_cast(now.time_since_epoch()).count(); data.entity_id = entity_id; data.topic_id = topic_info.topic_id; @@ -66,7 +67,7 @@ namespace eCAL CSubscriberSpan::~CSubscriberSpan() { - auto now = system_clock::now(); + auto now = tracing_clock::now(); data.end_ns = duration_cast(now.time_since_epoch()).count(); if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); } diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h index 829c09a31a..964203afe1 100644 --- a/ecal/core/src/tracing/tracing.h +++ b/ecal/core/src/tracing/tracing.h @@ -27,6 +27,7 @@ #include #include +#include #include #include #include From cd128b8ee7fbcc9205740ecd00cd9f182ca57552 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Tue, 19 May 2026 18:19:56 +0200 Subject: [PATCH 03/12] More efficient disabled tracing. --- ecal/core/CMakeLists.txt | 1 - ecal/core/src/ecal_global_accessors.cpp | 6 +-- ecal/core/src/pubsub/ecal_publisher_impl.cpp | 8 +--- ecal/core/src/pubsub/ecal_subscriber_impl.cpp | 5 +-- ecal/core/src/tracing/span.cpp | 45 +++++++++++++++++-- ecal/core/src/tracing/span.h | 38 +++++++++++++--- ecal/core/src/tracing/trace_provider.cpp | 3 +- ecal/core/src/tracing/trace_provider_noop.h | 36 --------------- 8 files changed, 80 insertions(+), 62 deletions(-) delete mode 100644 ecal/core/src/tracing/trace_provider_noop.h diff --git a/ecal/core/CMakeLists.txt b/ecal/core/CMakeLists.txt index ad607ba6c1..f97acb5f4d 100644 --- a/ecal/core/CMakeLists.txt +++ b/ecal/core/CMakeLists.txt @@ -469,7 +469,6 @@ set(ecal_tracing_src src/tracing/trace_provider.h src/tracing/trace_provider_default.cpp src/tracing/trace_provider_default.h - src/tracing/trace_provider_noop.h src/tracing/tracing_writer.h src/tracing/tracing_writer_jsonl.cpp src/tracing/tracing_writer_jsonl.h diff --git a/ecal/core/src/ecal_global_accessors.cpp b/ecal/core/src/ecal_global_accessors.cpp index 33e20fd128..2473d8136b 100644 --- a/ecal/core/src/ecal_global_accessors.cpp +++ b/ecal/core/src/ecal_global_accessors.cpp @@ -34,7 +34,6 @@ #include "logging/ecal_log_receiver.h" #include "tracing/trace_provider.h" #include "tracing/trace_provider_default.h" -#include "tracing/trace_provider_noop.h" #include #include @@ -133,10 +132,9 @@ namespace eCAL g_trace_provider_instance.reset(); } - std::shared_ptr g_trace_provider() + std::shared_ptr g_trace_provider() { - if (auto provider = g_trace_provider_instance; provider) return provider; - return nullptr; + return g_trace_provider_instance; } std::shared_ptr g_globals() diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.cpp b/ecal/core/src/pubsub/ecal_publisher_impl.cpp index 28f9152da4..3c9601cc67 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.cpp +++ b/ecal/core/src/pubsub/ecal_publisher_impl.cpp @@ -214,13 +214,7 @@ namespace eCAL } // create tracing span for the send operation - eCAL::tracing::CPublisherSpan send_span( - m_topic_id, - m_clock, - active_layer, - payload_buf_size, - eCAL::tracing::operation_type::send - ); + auto send_span = eCAL::tracing::CPublisherSpan::Create(m_topic_id, m_clock, active_layer, payload_buf_size, eCAL::tracing::operation_type::send); // did we write anything bool written(false); diff --git a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp index 108e3405fd..a204a6808c 100644 --- a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp +++ b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp @@ -392,8 +392,7 @@ namespace eCAL size_t CSubscriberImpl::ApplySample(const Payload::TopicInfo& topic_info_, const char* payload_, size_t size_, long long id_, long long clock_, long long time_, size_t /*hash_*/, eTLayerType layer_) { - - eCAL::tracing::CSubscriberSpan receive_span( + auto receive_span = eCAL::tracing::CSubscriberSpan::Create( m_subscriber_id, topic_info_, clock_, @@ -480,7 +479,7 @@ namespace eCAL // execute it const std::lock_guard exec_lock(m_connection_map_mtx); { - eCAL::tracing::CSubscriberSpan callback_span( + auto callback_span = eCAL::tracing::CSubscriberSpan::Create( m_subscriber_id, topic_info_, clock_, diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp index 34ed220e11..877d731c04 100644 --- a/ecal/core/src/tracing/span.cpp +++ b/ecal/core/src/tracing/span.cpp @@ -31,8 +31,26 @@ namespace eCAL { namespace tracing { + OptionalPublisherSpan CPublisherSpan::Create(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + { + auto provider = g_trace_provider(); + if (!provider) return std::nullopt; + + return OptionalPublisherSpan{ + std::in_place, + CPublisherSpan::ConstructionToken{}, + std::move(provider), + topic_id, + clock, + layer, + payload_size, + op_type + }; + } + // Send span constructor - CPublisherSpan::CPublisherSpan(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + CPublisherSpan::CPublisherSpan(ConstructionToken /*token*/, std::shared_ptr provider_, const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + : provider(std::move(provider_)) { auto now = tracing_clock::now(); data.start_ns = duration_cast(now.time_since_epoch()).count(); @@ -48,11 +66,30 @@ namespace eCAL { auto now = tracing_clock::now(); data.end_ns = duration_cast(now.time_since_epoch()).count(); - if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); + provider->WriteSpan(data); + } + + OptionalSubscriberSpan CSubscriberSpan::Create(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + { + auto provider = g_trace_provider(); + if (!provider) return std::nullopt; + + return OptionalSubscriberSpan{ + std::in_place, + CSubscriberSpan::ConstructionToken{}, + std::move(provider), + entity_id, + topic_info, + clock, + layer, + payload_size, + op_type + }; } // Receive span constructor - CSubscriberSpan::CSubscriberSpan(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + CSubscriberSpan::CSubscriberSpan(ConstructionToken /*token*/, std::shared_ptr provider_, EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type) + : provider(std::move(provider_)) { auto now = tracing_clock::now(); data.start_ns = duration_cast(now.time_since_epoch()).count(); @@ -69,7 +106,7 @@ namespace eCAL { auto now = tracing_clock::now(); data.end_ns = duration_cast(now.time_since_epoch()).count(); - if (auto provider = g_trace_provider(); provider) provider->WriteSpan(data); + provider->WriteSpan(data); } } } diff --git a/ecal/core/src/tracing/span.h b/ecal/core/src/tracing/span.h index 06f6187055..aec75d5827 100644 --- a/ecal/core/src/tracing/span.h +++ b/ecal/core/src/tracing/span.h @@ -20,6 +20,7 @@ #pragma once +#include #include "tracing.h" #include @@ -30,40 +31,67 @@ namespace eCAL { namespace tracing { + class TraceProvider; + // RAII span for send (publisher) operations. // Records start_ns on construction, end_ns + buffer on destruction. class CPublisherSpan { + private: + struct ConstructionToken + { + private: + ConstructionToken() = default; + friend class CPublisherSpan; + }; + public: - CPublisherSpan(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + static std::optional Create(const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + CPublisherSpan(ConstructionToken token, std::shared_ptr provider_, const STopicId& topic_id, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); ~CPublisherSpan(); CPublisherSpan(const CPublisherSpan&) = delete; CPublisherSpan& operator=(const CPublisherSpan&) = delete; - CPublisherSpan(CPublisherSpan&&) = delete; + CPublisherSpan(CPublisherSpan&&) = default; CPublisherSpan& operator=(CPublisherSpan&&) = delete; private: SPublisherSpanData data{}; + std::shared_ptr provider; }; + using OptionalPublisherSpan = std::optional; + // RAII span for receive (subscriber) operations. // Records start_ns on construction, end_ns + buffer on destruction. class CSubscriberSpan { - public: - CSubscriberSpan(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + private: + struct ConstructionToken + { + private: + ConstructionToken() = default; + friend class CSubscriberSpan; + }; + public: + static std::optional Create(EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); + CSubscriberSpan(ConstructionToken token, std::shared_ptr provider_, EntityIdT entity_id, const eCAL::Payload::TopicInfo& topic_info, long long clock, eTracingLayerType layer, size_t payload_size, operation_type op_type); ~CSubscriberSpan(); + private: CSubscriberSpan(const CSubscriberSpan&) = delete; CSubscriberSpan& operator=(const CSubscriberSpan&) = delete; - CSubscriberSpan(CSubscriberSpan&&) = delete; + CSubscriberSpan(CSubscriberSpan&&) = default; CSubscriberSpan& operator=(CSubscriberSpan&&) = delete; private: SSubscriberSpanData data{}; + std::shared_ptr provider; }; + + using OptionalSubscriberSpan = std::optional; + } } diff --git a/ecal/core/src/tracing/trace_provider.cpp b/ecal/core/src/tracing/trace_provider.cpp index 71b5cd3a90..771171fff3 100644 --- a/ecal/core/src/tracing/trace_provider.cpp +++ b/ecal/core/src/tracing/trace_provider.cpp @@ -20,7 +20,6 @@ #include "trace_provider.h" #include "trace_provider_default.h" -#include "trace_provider_noop.h" namespace eCAL { @@ -32,7 +31,7 @@ namespace eCAL { return CTraceProviderDefault::Create(); } - return std::make_shared(); + return nullptr; } } } diff --git a/ecal/core/src/tracing/trace_provider_noop.h b/ecal/core/src/tracing/trace_provider_noop.h deleted file mode 100644 index 3e696e98ea..0000000000 --- a/ecal/core/src/tracing/trace_provider_noop.h +++ /dev/null @@ -1,36 +0,0 @@ -/* ========================= eCAL LICENSE ================================= - * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * ========================= eCAL LICENSE ================================= -*/ - -#pragma once - -#include "trace_provider.h" - -namespace eCAL -{ - namespace tracing - { - class CNoOpTraceProvider : public TraceProvider - { - public: - void WriteSpan(const SpanDataVariant& /*span_data*/) override {} - void WriteMetadata(const STopicMetadata& /*metadata*/) override {} - }; - } -} From b6a3395bc65d0851c8d3b3bfe1def1e02ba939cc Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Wed, 20 May 2026 13:08:00 +0200 Subject: [PATCH 04/12] Remove unnecessary function. --- ecal/core/src/pubsub/ecal_publisher_impl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.h b/ecal/core/src/pubsub/ecal_publisher_impl.h index 43820a72ec..1f0a37216b 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.h +++ b/ecal/core/src/pubsub/ecal_publisher_impl.h @@ -118,7 +118,6 @@ namespace eCAL void GetRegistrationSample(Registration::Sample& sample); void GetUnregistrationSample(Registration::Sample& sample); - long long GetClock() const { return m_clock; } bool StartUdpLayer(); bool StartShmLayer(); From 7f9c64d43e596c0e17f925b610ee080340591120 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Wed, 20 May 2026 13:16:13 +0200 Subject: [PATCH 05/12] Fix CMakeLists.txt --- ecal/core/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/ecal/core/CMakeLists.txt b/ecal/core/CMakeLists.txt index f97acb5f4d..23fca42f61 100644 --- a/ecal/core/CMakeLists.txt +++ b/ecal/core/CMakeLists.txt @@ -726,7 +726,6 @@ target_link_libraries(ecal_core_private $<$:ecaltime> ecaludp::ecaludp nlohmann_json::nlohmann_json - $<$,$>>:dl> ${CMAKE_DL_LIBS} $<$,$>,$>>:rt> $<$,$>,$>>:atomic> From 34b04f59a9e72dd4e6004687f02a3736e7d839e5 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Thu, 21 May 2026 14:58:47 +0200 Subject: [PATCH 06/12] Build errors. --- ecal/core/CMakeLists.txt | 3 ++- ecal/core/src/tracing/span.cpp | 3 +-- ecal/core/src/tracing/span.h | 5 +++-- ecal/core/src/tracing/trace_provider.cpp | 3 +-- ecal/core/src/tracing/trace_provider.h | 3 +-- ecal/core/src/tracing/trace_provider_default.cpp | 3 +-- ecal/core/src/tracing/trace_provider_default.h | 3 +-- ecal/core/src/tracing/tracing.h | 3 +-- ecal/core/src/tracing/tracing_writer.h | 3 +-- ecal/core/src/tracing/tracing_writer_jsonl.cpp | 3 +-- ecal/core/src/tracing/tracing_writer_jsonl.h | 3 +-- 11 files changed, 14 insertions(+), 21 deletions(-) diff --git a/ecal/core/CMakeLists.txt b/ecal/core/CMakeLists.txt index 23fca42f61..59d0dea3a3 100644 --- a/ecal/core/CMakeLists.txt +++ b/ecal/core/CMakeLists.txt @@ -1,7 +1,7 @@ # ========================= eCAL LICENSE ================================= # # Copyright (C) 2016 - 2025 Continental Corporation -# Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. +# Copyright 2025 - 2026 AUMOVIO and subsidiaries. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -584,6 +584,7 @@ set(ecal_public_headers include/ecal/config/registration.h include/ecal/config/subscriber.h include/ecal/config/time.h + include/ecal/config/tracing.h include/ecal/config/transport_layer.h include/ecal/pubsub/subscriber.h include/ecal/pubsub/types.h diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp index 877d731c04..5c9ffd5eff 100644 --- a/ecal/core/src/tracing/span.cpp +++ b/ecal/core/src/tracing/span.cpp @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/span.h b/ecal/core/src/tracing/span.h index aec75d5827..fcc02053f6 100644 --- a/ecal/core/src/tracing/span.h +++ b/ecal/core/src/tracing/span.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +19,9 @@ #pragma once +#include #include + #include "tracing.h" #include diff --git a/ecal/core/src/tracing/trace_provider.cpp b/ecal/core/src/tracing/trace_provider.cpp index 771171fff3..6f9ec62dd2 100644 --- a/ecal/core/src/tracing/trace_provider.cpp +++ b/ecal/core/src/tracing/trace_provider.cpp @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/trace_provider.h b/ecal/core/src/tracing/trace_provider.h index 98df533c62..142a729e97 100644 --- a/ecal/core/src/tracing/trace_provider.h +++ b/ecal/core/src/tracing/trace_provider.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/trace_provider_default.cpp b/ecal/core/src/tracing/trace_provider_default.cpp index 1627d80c5e..65d59de8ab 100644 --- a/ecal/core/src/tracing/trace_provider_default.cpp +++ b/ecal/core/src/tracing/trace_provider_default.cpp @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/trace_provider_default.h b/ecal/core/src/tracing/trace_provider_default.h index 4b14529c07..3b2436243e 100644 --- a/ecal/core/src/tracing/trace_provider_default.h +++ b/ecal/core/src/tracing/trace_provider_default.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h index 964203afe1..6182b1f087 100644 --- a/ecal/core/src/tracing/tracing.h +++ b/ecal/core/src/tracing/tracing.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/tracing_writer.h b/ecal/core/src/tracing/tracing_writer.h index 7ff26e0332..72114cc2ef 100644 --- a/ecal/core/src/tracing/tracing_writer.h +++ b/ecal/core/src/tracing/tracing_writer.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp index 8a35ea235d..47fc8500dd 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.h b/ecal/core/src/tracing/tracing_writer_jsonl.h index a09c77ca9a..2647739005 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.h +++ b/ecal/core/src/tracing/tracing_writer_jsonl.h @@ -1,7 +1,6 @@ /* ========================= eCAL LICENSE ================================= * - * Copyright (C) 2016 - 2025 Continental Corporation - * Copyright 2025 AUMOVIO and subsidiaries. All rights reserved. + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From e224224e11108fbaebb22087fdc70f019fbf25bb Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:41:01 +0200 Subject: [PATCH 07/12] Batch process metadata as well as regular data. --- ecal/core/src/tracing/trace_provider.h | 2 +- .../src/tracing/trace_provider_default.cpp | 24 ++- .../core/src/tracing/trace_provider_default.h | 6 +- ecal/core/src/tracing/tracing.h | 7 +- ecal/core/src/tracing/tracing_writer.h | 6 +- .../core/src/tracing/tracing_writer_jsonl.cpp | 186 ++++++++++-------- ecal/core/src/tracing/tracing_writer_jsonl.h | 25 ++- .../tracing_test/src/tracing_test_helpers.h | 33 ++-- .../tracing_test/src/tracing_writer_test.cpp | 186 +++++++----------- 9 files changed, 240 insertions(+), 235 deletions(-) diff --git a/ecal/core/src/tracing/trace_provider.h b/ecal/core/src/tracing/trace_provider.h index 142a729e97..f48937bf13 100644 --- a/ecal/core/src/tracing/trace_provider.h +++ b/ecal/core/src/tracing/trace_provider.h @@ -42,7 +42,7 @@ namespace eCAL static std::shared_ptr Create(const eCAL::Tracing::Configuration& config_); - virtual void WriteSpan(const SpanDataVariant& span_data) = 0; + virtual void WriteSpan(const SpanData& span_data) = 0; virtual void WriteMetadata(const STopicMetadata& metadata) = 0; }; } diff --git a/ecal/core/src/tracing/trace_provider_default.cpp b/ecal/core/src/tracing/trace_provider_default.cpp index 65d59de8ab..042979d7cc 100644 --- a/ecal/core/src/tracing/trace_provider_default.cpp +++ b/ecal/core/src/tracing/trace_provider_default.cpp @@ -42,6 +42,7 @@ namespace eCAL } } + CTraceProviderDefault::CTraceProviderDefault(std::unique_ptr writer, size_t batch_size) : batch_size_(batch_size), writer_(std::move(writer)) { @@ -58,10 +59,20 @@ namespace eCAL writer_thread_.join(); } - void CTraceProviderDefault::WriteSpan(const SpanDataVariant& span_data) + void CTraceProviderDefault::WriteSpan(const SpanData& span_data) + { + WriteTraceInfo(span_data); + } + + void CTraceProviderDefault::WriteMetadata(const STopicMetadata& metadata) + { + WriteTraceInfo(metadata); + } + + void CTraceProviderDefault::WriteTraceInfo(const TraceInfo& info) { std::lock_guard lock(thread_mutex); - span_buffer_.push_back(span_data); + span_buffer_.push_back(info); if (span_buffer_.size() >= batch_size_) { write_cv_.notify_one(); @@ -70,7 +81,7 @@ namespace eCAL void CTraceProviderDefault::WriterThreadLoop() { - std::vector span_flusher; + std::vector span_flusher; while (true) { { @@ -87,15 +98,10 @@ namespace eCAL } if (!span_flusher.empty()) { - writer_->WriteSpansToFile(span_flusher); + writer_->WriteTraceInfo(span_flusher); span_flusher.clear(); } } } - - void CTraceProviderDefault::WriteMetadata(const STopicMetadata& metadata) - { - writer_->WriteMetadataToFile(metadata); - } } } diff --git a/ecal/core/src/tracing/trace_provider_default.h b/ecal/core/src/tracing/trace_provider_default.h index 3b2436243e..978afe904e 100644 --- a/ecal/core/src/tracing/trace_provider_default.h +++ b/ecal/core/src/tracing/trace_provider_default.h @@ -51,17 +51,19 @@ namespace eCAL ~CTraceProviderDefault() override; // Write span data to buffer (accepts any span type via variant) - void WriteSpan(const SpanDataVariant& span_data) override; + void WriteSpan(const SpanData& span_data) override; // metadata — written directly to file (no buffering) void WriteMetadata(const STopicMetadata& metadata) override; private: + void WriteTraceInfo(const TraceInfo& info); + CTraceProviderDefault(std::unique_ptr writer, size_t batch_size); void WriterThreadLoop(); std::atomic batch_size_{kDefaultTracingBatchSize}; - std::vector span_buffer_; + std::vector span_buffer_; mutable std::mutex thread_mutex; std::condition_variable write_cv_; bool stop_thread_{false}; diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h index 6182b1f087..3d0a697729 100644 --- a/ecal/core/src/tracing/tracing.h +++ b/ecal/core/src/tracing/tracing.h @@ -38,8 +38,8 @@ namespace eCAL // Version of the tracing implementation. constexpr const char* kTracingVersion = "1.0.0"; - // Default batch size for span buffering before flushing to backend (jsonl file) - constexpr size_t kDefaultTracingBatchSize = 10; + // Default batch size for span buffering before flushing to backend + constexpr size_t kDefaultTracingBatchSize = 4096; // Specifies the type of operation being traced enum operation_type @@ -123,6 +123,7 @@ namespace eCAL // Variant type for buffering heterogeneous span data. // Extend this variant when new span types are added (e.g. client/server). - using SpanDataVariant = std::variant; + using SpanData = std::variant; + using TraceInfo = std::variant< STopicMetadata, SpanData>; } } diff --git a/ecal/core/src/tracing/tracing_writer.h b/ecal/core/src/tracing/tracing_writer.h index 72114cc2ef..fc604703fe 100644 --- a/ecal/core/src/tracing/tracing_writer.h +++ b/ecal/core/src/tracing/tracing_writer.h @@ -34,11 +34,7 @@ namespace eCAL public: virtual ~TracingWriter() = default; - // Write a batch of spans (heterogeneous via variant) - virtual void WriteSpansToFile(const std::vector& batch) = 0; - - // Write a single topic metadata entry - virtual void WriteMetadataToFile(const STopicMetadata& metadata) = 0; + virtual void WriteTraceInfo(const std::vector& batch) = 0; }; } } diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp index 47fc8500dd..9d6d8f760c 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -33,81 +33,125 @@ #include using json = nlohmann::json; +namespace { + + template + struct Overloaded : Ts... { + using Ts::operator()...; + }; + + template + Overloaded(Ts...) -> Overloaded; + + + using namespace eCAL::tracing; + + json toJson(const STopicMetadata& metadata) + { + json metadata_as_json; + metadata_as_json["tracing_version"] = metadata.tracing_version; + metadata_as_json["entity_id"] = metadata.entity_id; + metadata_as_json["process_id"] = metadata.process_id; + metadata_as_json["host_name"] = metadata.host_name; + metadata_as_json["topic_name"] = metadata.topic_name; + metadata_as_json["encoding"] = metadata.encoding; + metadata_as_json["type_name"] = metadata.type_name; + metadata_as_json["direction"] = (metadata.direction == topic_direction::publisher) ? "publisher" : "subscriber"; + return metadata_as_json; + } + + json toJson(const SPublisherSpanData& span) + { + json span_as_json; + span_as_json["op_type"] = static_cast(span.op_type); + span_as_json["entity_id"] = span.entity_id; + span_as_json["process_id"] = span.process_id; + span_as_json["payload_size"] = span.payload_size; + span_as_json["clock"] = span.clock; + span_as_json["layer"] = span.layer; + span_as_json["start_ns"] = span.start_ns; + span_as_json["end_ns"] = span.end_ns; + return span_as_json; + } + + json toJson(const SSubscriberSpanData& span) + { + json span_as_json; + span_as_json["op_type"] = static_cast(span.op_type); + span_as_json["entity_id"] = span.entity_id; + span_as_json["topic_id"] = span.topic_id; + span_as_json["process_id"] = span.process_id; + span_as_json["payload_size"] = span.payload_size; + span_as_json["clock"] = span.clock; + span_as_json["layer"] = span.layer; + span_as_json["start_ns"] = span.start_ns; + span_as_json["end_ns"] = span.end_ns; + return span_as_json; + } + + std::string GetCurrentTimestamp() + { + std::time_t now = std::time(nullptr); + std::tm* tm_info = std::localtime(&now); + std::ostringstream oss; + oss << std::put_time(tm_info, "%Y%m%d_%H%M%S"); + return oss.str(); + } + + static int unique_cnt = 0; +} + namespace eCAL { namespace tracing { - static std::string GetCurrentTimestamp() + CTracingWriterJSONL::CTracingWriterJSONL() + : CTracingWriterJSONL(std::to_string(eCAL::Process::GetProcessID()) + "_" + std::to_string(++unique_cnt) + "_" + GetCurrentTimestamp(), std::filesystem::path(Util::GeteCALTraceDir())) { - std::time_t now = std::time(nullptr); - std::tm* tm_info = std::localtime(&now); - std::ostringstream oss; - oss << std::put_time(tm_info, "%Y%m%d_%H%M%S"); - return oss.str(); } - CTracingWriterJSONL::CTracingWriterJSONL() - : timestamp_(GetCurrentTimestamp()) - , trace_dir_(eCAL::Util::GeteCALTraceDir()) - {} - std::string CTracingWriterJSONL::GetSpansFilePath() const + CTracingWriterJSONL::CTracingWriterJSONL(const std::string& file_id, std::filesystem::path trace_directory) + : spans_file_path_(trace_directory / (std::string("ecal_spans_") + file_id + ".jsonl")) + , metadata_file_path_(trace_directory / (std::string("ecal_metadata_") + file_id + ".jsonl")) + , spans_file_(spans_file_path_, std::ios::out | std::ios::trunc) + , metadata_file_(metadata_file_path_, std::ios::out | std::ios::trunc) { - return trace_dir_ + "/ecal_spans_" + std::to_string(eCAL::Process::GetProcessID()) + "_" + timestamp_ + ".jsonl"; } - std::string CTracingWriterJSONL::GetTopicMetadataFilePath() const + std::filesystem::path CTracingWriterJSONL::GetSpansFilePath() const { - return trace_dir_ + "/ecal_topic_metadata_" + std::to_string(eCAL::Process::GetProcessID()) + "_" + timestamp_ + ".jsonl"; + return spans_file_path_; } - void CTracingWriterJSONL::WriteSpansToFile(const std::vector& batch) + std::filesystem::path CTracingWriterJSONL::GetTopicMetadataFilePath() const { - try + return metadata_file_path_; + } + + + void CTracingWriterJSONL::WriteTraceInfo(const std::vector& batch) + { + for (const auto& span_variant : batch) { - std::lock_guard lock(spans_mutex_); - std::string filepath = GetSpansFilePath(); + std::visit( + Overloaded{ + [this](const SpanData& span) { WriteSpanData(span); }, + [this](const STopicMetadata& metadata) { WriteTopicMetadata(metadata); } + }, + span_variant + ); + } + } - std::ofstream output_file(filepath, std::ios::app); - if (output_file.is_open()) - { - for (const auto& span_variant : batch) - { - json span_obj; - std::visit([&span_obj](const auto& span) - { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - span_obj["entity_id"] = span.entity_id; - span_obj["process_id"] = span.process_id; - span_obj["payload_size"] = span.payload_size; - span_obj["clock"] = span.clock; - span_obj["layer"] = span.layer; - span_obj["start_ns"] = span.start_ns; - span_obj["end_ns"] = span.end_ns; - span_obj["op_type"] = span.op_type; - } - else if constexpr (std::is_same_v) - { - span_obj["entity_id"] = span.entity_id; - span_obj["topic_id"] = span.topic_id; - span_obj["process_id"] = span.process_id; - span_obj["payload_size"] = span.payload_size; - span_obj["clock"] = span.clock; - span_obj["layer"] = span.layer; - span_obj["start_ns"] = span.start_ns; - span_obj["end_ns"] = span.end_ns; - span_obj["op_type"] = span.op_type; - } - }, span_variant); - output_file << span_obj.dump() << "\n"; - } - output_file.close(); - } - else + void CTracingWriterJSONL::WriteSpanData(const SpanData& span_data) + { + try + { + if (spans_file_.is_open()) { - std::cerr << "Warning: Could not open spans file: " << filepath << std::endl; + auto span_obj = std::visit([](const auto& span) { return toJson(span); }, span_data); + spans_file_ << span_obj.dump() << "\n"; } } catch (const std::exception& e) @@ -116,37 +160,19 @@ namespace eCAL } } - void CTracingWriterJSONL::WriteMetadataToFile(const STopicMetadata& metadata) + void CTracingWriterJSONL::WriteTopicMetadata(const STopicMetadata& metadata) { try { - std::lock_guard lock(metadata_mutex_); - std::string filepath = GetTopicMetadataFilePath(); - - json obj; - obj["tracing_version"] = metadata.tracing_version; - obj["entity_id"] = metadata.entity_id; - obj["process_id"] = metadata.process_id; - obj["host_name"] = metadata.host_name; - obj["topic_name"] = metadata.topic_name; - obj["encoding"] = metadata.encoding; - obj["type_name"] = metadata.type_name; - obj["direction"] = (metadata.direction == topic_direction::publisher) ? "publisher" : "subscriber"; - - std::ofstream output_file(filepath, std::ios::app); - if (output_file.is_open()) - { - output_file << obj.dump() << "\n"; - output_file.close(); - } - else + if (metadata_file_.is_open()) { - std::cerr << "Warning: Could not open topic metadata file: " << filepath << std::endl; + auto metadata_json = toJson(metadata); + metadata_file_ << metadata_json.dump() << "\n"; } } catch (const std::exception& e) { - std::cerr << "Error writing topic metadata to JSONL: " << e.what() << std::endl; + std::cerr << "Error writing metadata to JSONL: " << e.what() << std::endl; } } } diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.h b/ecal/core/src/tracing/tracing_writer_jsonl.h index 2647739005..16e102b389 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.h +++ b/ecal/core/src/tracing/tracing_writer_jsonl.h @@ -19,6 +19,8 @@ #pragma once +#include +#include #include #include #include @@ -32,10 +34,12 @@ namespace eCAL { // Responsible for serializing span and metadata to JSONL files. // Separated from CTraceProvider to isolate the I/O concern. + // This class is not thread-safe; CTraceProviderDefault handles synchronization and buffering before calling this writer. class CTracingWriterJSONL : public TracingWriter { public: CTracingWriterJSONL(); + CTracingWriterJSONL(const std::string& file_id, std::filesystem::path trace_directory); ~CTracingWriterJSONL() override = default; CTracingWriterJSONL(const CTracingWriterJSONL&) = delete; @@ -44,20 +48,21 @@ namespace eCAL CTracingWriterJSONL& operator=(CTracingWriterJSONL&&) = delete; // Write a batch of spans to the JSONL spans file - void WriteSpansToFile(const std::vector& batch) override; - - // Write a single topic metadata entry to the JSONL metadata file - void WriteMetadataToFile(const STopicMetadata& metadata) override; + void WriteTraceInfo(const std::vector& batch) override; // File path accessors (path is fixed at construction time) - std::string GetSpansFilePath() const; - std::string GetTopicMetadataFilePath() const; + std::filesystem::path GetSpansFilePath() const; + std::filesystem::path GetTopicMetadataFilePath() const; private: - mutable std::mutex spans_mutex_; - mutable std::mutex metadata_mutex_; - std::string timestamp_; - std::string trace_dir_; + void WriteSpanData(const SpanData& span_data); + void WriteTopicMetadata(const STopicMetadata& metadata); + + std::filesystem::path spans_file_path_; + std::filesystem::path metadata_file_path_; + + std::ofstream spans_file_; + std::ofstream metadata_file_; }; } } diff --git a/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h index cc86564517..5ae705a480 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h +++ b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h @@ -80,14 +80,12 @@ inline eCAL::Configuration GetTracingConfiguration() class MockTracingWriter : public eCAL::tracing::TracingWriter { public: - void WriteSpansToFile(const std::vector& batch) override + void WriteTraceInfo(const std::vector& batch) override { - span_count_ += batch.size(); - } - - void WriteMetadataToFile(const eCAL::tracing::STopicMetadata& /*metadata*/) override - { - metadata_count_++; + for (const auto& info : batch) + { + std::visit([this](const auto& span) { this->Write(span); }, info); + } } size_t SpanCount() const @@ -108,6 +106,16 @@ class MockTracingWriter : public eCAL::tracing::TracingWriter } private: + void Write(const eCAL::tracing::SpanData& /*spandata*/) + { + ++span_count_; + } + + void Write(const eCAL::tracing::STopicMetadata& /*metadata*/) + { + ++metadata_count_; + } + mutable std::mutex mutex_; std::atomic span_count_{0}; std::atomic metadata_count_{0}; @@ -119,14 +127,9 @@ class ProxyTracingWriter : public eCAL::tracing::TracingWriter public: explicit ProxyTracingWriter(MockTracingWriter& target) : target_(target) {} - void WriteSpansToFile(const std::vector& batch) override - { - target_.WriteSpansToFile(batch); - } - - void WriteMetadataToFile(const eCAL::tracing::STopicMetadata& metadata) override + void WriteTraceInfo(const std::vector& batch) override { - target_.WriteMetadataToFile(metadata); + target_.WriteTraceInfo(batch); } private: @@ -134,7 +137,7 @@ class ProxyTracingWriter : public eCAL::tracing::TracingWriter }; // Count the number of lines in a file and validate each line is valid JSON. -inline size_t CountAndValidateJsonlLines(const std::string& filepath) +inline size_t CountAndValidateJsonlLines(const std::filesystem::path& filepath) { std::ifstream file(filepath); EXPECT_TRUE(file.is_open()) << "Failed to open: " << filepath; diff --git a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp index 1a8c2203df..ddb8a2de6a 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp +++ b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp @@ -39,55 +39,35 @@ TEST(TestTracingWriterJSONL, ConcurrentSpanWrites) { - constexpr size_t num_threads = 100; - constexpr size_t batches_per_thread = 50; - constexpr size_t spans_per_batch = 500; - constexpr size_t total_spans = num_threads * batches_per_thread * spans_per_batch; + constexpr size_t total_spans = 5000; ScopedTraceDirOverride trace_dir_override("./"); - auto ecal_config = GetTracingConfiguration(); + auto ecal_config = GetTracingConfiguration(); ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); - std::string spans_path; + std::filesystem::path spans_path; { eCAL::tracing::CTracingWriterJSONL writer; spans_path = writer.GetSpansFilePath(); - Barrier barrier(num_threads); - - std::vector threads; - threads.reserve(num_threads); + std::vector batch; + batch.reserve(total_spans); - for (size_t t = 0; t < num_threads; ++t) + for (size_t i = 0; i < total_spans; ++i) { - threads.emplace_back([&, t]() - { - barrier.wait(); - - for (size_t b = 0; b < batches_per_thread; ++b) - { - std::vector batch; - batch.reserve(spans_per_batch); - - for (size_t i = 0; i < spans_per_batch; ++i) - { - eCAL::tracing::SPublisherSpanData span{}; - span.op_type = eCAL::tracing::operation_type::send; - span.entity_id = static_cast(t * batches_per_thread * spans_per_batch + b * spans_per_batch + i); - span.process_id = 1; - span.payload_size = 64; - span.clock = static_cast(i); - span.layer = eCAL::tracing::tl_trace_shm; - span.start_ns = 1000; - span.end_ns = 2000; - batch.push_back(span); - } - writer.WriteSpansToFile(batch); - } - }); + eCAL::tracing::SPublisherSpanData span{}; + span.op_type = eCAL::tracing::operation_type::send; + span.entity_id = static_cast(i); + span.process_id = 1; + span.payload_size = 64; + span.clock = static_cast(i); + span.layer = eCAL::tracing::tl_trace_shm; + span.start_ns = 1000; + span.end_ns = 2000; + batch.push_back(span); } - for (auto& th : threads) th.join(); + writer.WriteTraceInfo(batch); } eCAL::Finalize(); @@ -95,60 +75,42 @@ TEST(TestTracingWriterJSONL, ConcurrentSpanWrites) // Verify: every span line must be valid JSON and the total count must match. size_t line_count = CountAndValidateJsonlLines(spans_path); EXPECT_EQ(line_count, total_spans); - - if (!spans_path.empty()) std::remove(spans_path.c_str()); + std::filesystem::remove(spans_path); } -TEST(TestTracingWriterJSONL, ConcurrentMetadataWrites) +TEST(TestTracingWriterJSONL, MetadataWrites) { - constexpr size_t num_threads = 100; - constexpr size_t metadata_per_thread = 200; - constexpr size_t total_metadata = num_threads * metadata_per_thread; + constexpr size_t metadata_instances = 200; ScopedTraceDirOverride trace_dir_override("./"); auto ecal_config = GetTracingConfiguration(); ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); - std::string metadata_path; + std::filesystem::path metadata_path; { eCAL::tracing::CTracingWriterJSONL writer; metadata_path = writer.GetTopicMetadataFilePath(); - Barrier barrier(num_threads); - - std::vector threads; - threads.reserve(num_threads); - - for (size_t t = 0; t < num_threads; ++t) + for (size_t i = 0; i < metadata_instances; ++i) { - threads.emplace_back([&, t]() - { - barrier.wait(); - - for (size_t i = 0; i < metadata_per_thread; ++i) - { - eCAL::tracing::STopicMetadata metadata{}; - metadata.entity_id = static_cast(t * metadata_per_thread + i); - metadata.process_id = 1; - metadata.host_name = "test_host"; - metadata.topic_name = "topic_" + std::to_string(t) + "_" + std::to_string(i); - metadata.encoding = "protobuf"; - metadata.type_name = "TestType"; - metadata.direction = eCAL::tracing::topic_direction::publisher; - writer.WriteMetadataToFile(metadata); - } - }); + eCAL::tracing::STopicMetadata metadata{}; + metadata.entity_id = static_cast(i); + metadata.process_id = 1; + metadata.host_name = "test_host"; + metadata.topic_name = "topic_" + std::to_string(i); + metadata.encoding = "protobuf"; + metadata.type_name = "TestType"; + metadata.direction = eCAL::tracing::topic_direction::publisher; + writer.WriteTraceInfo({metadata}); } - - for (auto& th : threads) th.join(); } eCAL::Finalize(); size_t line_count = CountAndValidateJsonlLines(metadata_path); - EXPECT_EQ(line_count, total_metadata); + EXPECT_EQ(line_count, metadata_instances); - if (!metadata_path.empty()) std::remove(metadata_path.c_str()); + std::filesystem::remove(metadata_path); } TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) @@ -157,7 +119,7 @@ TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) auto ecal_config = GetTracingConfiguration(); ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); - std::string spans_path; + std::filesystem::path spans_path; { eCAL::tracing::CTracingWriterJSONL writer; @@ -173,30 +135,32 @@ TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) span.start_ns = 1000; span.end_ns = 2000; - std::vector batch; + std::vector batch; batch.push_back(span); - writer.WriteSpansToFile(batch); + writer.WriteTraceInfo(batch); } eCAL::Finalize(); - std::ifstream file(spans_path); - ASSERT_TRUE(file.is_open()); - - std::string line; - ASSERT_TRUE(std::getline(file, line)); - auto j = nlohmann::json::parse(line); - - EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); - EXPECT_EQ(j.at("entity_id"), 42u); - EXPECT_EQ(j.at("process_id"), 123u); - EXPECT_EQ(j.at("payload_size"), 256u); - EXPECT_EQ(j.at("clock"), 7); - EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); - EXPECT_EQ(j.at("start_ns"), 1000); - EXPECT_EQ(j.at("end_ns"), 2000); + { + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); + + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); + + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); + EXPECT_EQ(j.at("entity_id"), 42u); + EXPECT_EQ(j.at("process_id"), 123u); + EXPECT_EQ(j.at("payload_size"), 256u); + EXPECT_EQ(j.at("clock"), 7); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); + EXPECT_EQ(j.at("start_ns"), 1000); + EXPECT_EQ(j.at("end_ns"), 2000); + } - if (!spans_path.empty()) std::remove(spans_path.c_str()); + std::filesystem::remove(spans_path); } TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) @@ -205,7 +169,7 @@ TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) auto ecal_config = GetTracingConfiguration(); ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); - std::string spans_path; + std::filesystem::path spans_path; { eCAL::tracing::CTracingWriterJSONL writer; @@ -222,30 +186,32 @@ TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) span.start_ns = 3000; span.end_ns = 4000; - std::vector batch; + std::vector batch; batch.push_back(span); - writer.WriteSpansToFile(batch); + writer.WriteTraceInfo(batch); } eCAL::Finalize(); - std::ifstream file(spans_path); - ASSERT_TRUE(file.is_open()); - - std::string line; - ASSERT_TRUE(std::getline(file, line)); - auto j = nlohmann::json::parse(line); - - EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); - EXPECT_EQ(j.at("entity_id"), 99u); - EXPECT_EQ(j.at("topic_id"), 55u); - EXPECT_EQ(j.at("process_id"), 456u); - EXPECT_EQ(j.at("payload_size"), 512u); - EXPECT_EQ(j.at("clock"), 3); - EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); - EXPECT_EQ(j.at("start_ns"), 3000); - EXPECT_EQ(j.at("end_ns"), 4000); + { + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); + + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); + + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); + EXPECT_EQ(j.at("entity_id"), 99u); + EXPECT_EQ(j.at("topic_id"), 55u); + EXPECT_EQ(j.at("process_id"), 456u); + EXPECT_EQ(j.at("payload_size"), 512u); + EXPECT_EQ(j.at("clock"), 3); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); + EXPECT_EQ(j.at("start_ns"), 3000); + EXPECT_EQ(j.at("end_ns"), 4000); + } - if (!spans_path.empty()) std::remove(spans_path.c_str()); + std::filesystem::remove(spans_path); } From b662935989457c563e84779043c3411ec5981de9 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:45:15 +0200 Subject: [PATCH 08/12] More fixes. --- ecal/core/src/config/ecal_path_processing.cpp | 3 ++- ecal/core/src/tracing/tracing.h | 2 +- ecal/core/src/tracing/tracing_writer_jsonl.cpp | 12 ++++++++---- .../cpp/tracing_test/src/tracing_writer_test.cpp | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ecal/core/src/config/ecal_path_processing.cpp b/ecal/core/src/config/ecal_path_processing.cpp index 9b8c59be01..ab1b4cd546 100644 --- a/ecal/core/src/config/ecal_path_processing.cpp +++ b/ecal/core/src/config/ecal_path_processing.cpp @@ -206,7 +206,8 @@ namespace eCAL // create also logs directory const std::string log_path = buildPath(path_, ECAL_FOLDER_NAME_LOG); - return dirExistsOrCreate(log_path); + const std::string traces_path = buildPath(path_, ECAL_FOLDER_NAME_TRACE); + return dirExistsOrCreate(log_path) && dirExistsOrCreate(traces_path); } std::string DirManager::findFileInPaths(const std::vector& paths_, const std::string& file_name_) const diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h index 3d0a697729..607ca06476 100644 --- a/ecal/core/src/tracing/tracing.h +++ b/ecal/core/src/tracing/tracing.h @@ -39,7 +39,7 @@ namespace eCAL constexpr const char* kTracingVersion = "1.0.0"; // Default batch size for span buffering before flushing to backend - constexpr size_t kDefaultTracingBatchSize = 4096; + constexpr size_t kDefaultTracingBatchSize = 128; // Specifies the type of operation being traced enum operation_type diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp index 9d6d8f760c..60ffd3d82e 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -98,7 +98,11 @@ namespace { return oss.str(); } - static int unique_cnt = 0; + std::ofstream CreateWithDirectory(const std::filesystem::path& file_path, std::ios_base::openmode flags) + { + std::filesystem::create_directories(file_path.parent_path()); + return std::ofstream(file_path, flags); + } } namespace eCAL @@ -106,7 +110,7 @@ namespace eCAL namespace tracing { CTracingWriterJSONL::CTracingWriterJSONL() - : CTracingWriterJSONL(std::to_string(eCAL::Process::GetProcessID()) + "_" + std::to_string(++unique_cnt) + "_" + GetCurrentTimestamp(), std::filesystem::path(Util::GeteCALTraceDir())) + : CTracingWriterJSONL(std::to_string(eCAL::Process::GetProcessID()) + "_" + GetCurrentTimestamp(), std::filesystem::path(Util::GeteCALTraceDir())) { } @@ -114,8 +118,8 @@ namespace eCAL CTracingWriterJSONL::CTracingWriterJSONL(const std::string& file_id, std::filesystem::path trace_directory) : spans_file_path_(trace_directory / (std::string("ecal_spans_") + file_id + ".jsonl")) , metadata_file_path_(trace_directory / (std::string("ecal_metadata_") + file_id + ".jsonl")) - , spans_file_(spans_file_path_, std::ios::out | std::ios::trunc) - , metadata_file_(metadata_file_path_, std::ios::out | std::ios::trunc) + , spans_file_(CreateWithDirectory(spans_file_path_, std::ios::out | std::ios::trunc)) + , metadata_file_(CreateWithDirectory(metadata_file_path_, std::ios::out | std::ios::trunc)) { } diff --git a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp index ddb8a2de6a..c7d9cce069 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp +++ b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp @@ -37,9 +37,9 @@ #include -TEST(TestTracingWriterJSONL, ConcurrentSpanWrites) +TEST(TestTracingWriterJSONL, SpanWrites) { - constexpr size_t total_spans = 5000; + constexpr size_t total_spans = 4000; ScopedTraceDirOverride trace_dir_override("./"); auto ecal_config = GetTracingConfiguration(); ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); From 0eb692a29a63f8dc9e017702a4d7dd44e6b29574 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:05:41 +0200 Subject: [PATCH 09/12] Revert back to system clock. --- ecal/core/src/tracing/span.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp index 5c9ffd5eff..276fa00640 100644 --- a/ecal/core/src/tracing/span.cpp +++ b/ecal/core/src/tracing/span.cpp @@ -23,7 +23,7 @@ #include -using tracing_clock = std::chrono::steady_clock; +using tracing_clock = std::chrono::system_clock; using namespace std::chrono; namespace eCAL From a225379f112d0ce5d1e9e991d980eb6bc9ba08ec Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:40:19 +0200 Subject: [PATCH 10/12] * Also export process name * remove process ids from span and keep only in metadata --- ecal/core/src/pubsub/ecal_publisher_impl.cpp | 15 ++++++++------- ecal/core/src/pubsub/ecal_subscriber_impl.cpp | 1 + ecal/core/src/tracing/span.cpp | 2 -- ecal/core/src/tracing/tracing.h | 9 ++++----- ecal/core/src/tracing/tracing_writer_jsonl.cpp | 3 +-- .../cpp/tracing_test/src/trace_provider_test.cpp | 1 - .../cpp/tracing_test/src/tracing_writer_test.cpp | 6 +----- 7 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.cpp b/ecal/core/src/pubsub/ecal_publisher_impl.cpp index 3c9601cc67..47ae47af7c 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.cpp +++ b/ecal/core/src/pubsub/ecal_publisher_impl.cpp @@ -130,13 +130,14 @@ namespace eCAL { if (auto provider = g_trace_provider(); provider) { eCAL::tracing::STopicMetadata meta; - meta.entity_id = m_publisher_id; - meta.process_id = m_attributes.process_id; - meta.host_name = m_attributes.host_name; - meta.topic_name = m_attributes.topic_name; - meta.encoding = m_topic_info.encoding; - meta.type_name = m_topic_info.name; - meta.direction = eCAL::tracing::topic_direction::publisher; + meta.entity_id = m_publisher_id; + meta.process_id = m_attributes.process_id; + meta.process_name = m_attributes.process_name; + meta.host_name = m_attributes.host_name; + meta.topic_name = m_attributes.topic_name; + meta.encoding = m_topic_info.encoding; + meta.type_name = m_topic_info.name; + meta.direction = eCAL::tracing::topic_direction::publisher; provider->WriteMetadata(meta); } diff --git a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp index a204a6808c..2edca61a9f 100644 --- a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp +++ b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp @@ -102,6 +102,7 @@ namespace eCAL eCAL::tracing::STopicMetadata meta; meta.entity_id = m_subscriber_id; meta.process_id = m_attributes.process_id; + meta.process_name = m_attributes.process_name; meta.host_name = m_attributes.host_name; meta.topic_name = m_attributes.topic_name; meta.encoding = m_topic_info.encoding; diff --git a/ecal/core/src/tracing/span.cpp b/ecal/core/src/tracing/span.cpp index 276fa00640..d26e10d177 100644 --- a/ecal/core/src/tracing/span.cpp +++ b/ecal/core/src/tracing/span.cpp @@ -54,7 +54,6 @@ namespace eCAL auto now = tracing_clock::now(); data.start_ns = duration_cast(now.time_since_epoch()).count(); data.entity_id = topic_id.topic_id.entity_id; - data.process_id = topic_id.topic_id.process_id; data.payload_size = payload_size; data.clock = clock; data.layer = layer; @@ -94,7 +93,6 @@ namespace eCAL data.start_ns = duration_cast(now.time_since_epoch()).count(); data.entity_id = entity_id; data.topic_id = topic_info.topic_id; - data.process_id = topic_info.process_id; data.payload_size = payload_size; data.clock = clock; data.layer = layer; diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h index 607ca06476..301a4009fd 100644 --- a/ecal/core/src/tracing/tracing.h +++ b/ecal/core/src/tracing/tracing.h @@ -87,20 +87,20 @@ namespace eCAL struct STopicMetadata { std::string tracing_version{kTracingVersion}; // tracing format version - uint64_t entity_id; // unique entity id - int32_t process_id; // PID of the owning process + uint64_t entity_id{ 0 }; // unique entity id + int32_t process_id{ 0 }; // PID of the owning process + std::string process_name; // process name of the owning process std::string host_name; // host that created the topic std::string topic_name; // topic name used for pub/sub matching std::string encoding; // datatype encoding (e.g. protobuf) std::string type_name; // datatype name - topic_direction direction; // publisher or subscriber + topic_direction direction{ topic_direction::publisher }; // publisher or subscriber }; struct SPublisherSpanData { operation_type op_type; uint64_t entity_id; - uint64_t process_id; size_t payload_size; long long clock; uint64_t layer; @@ -113,7 +113,6 @@ namespace eCAL operation_type op_type; uint64_t entity_id; uint64_t topic_id; - uint64_t process_id; size_t payload_size; long long clock; uint64_t layer; diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp index 60ffd3d82e..7f15dee0cb 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -52,6 +52,7 @@ namespace { metadata_as_json["tracing_version"] = metadata.tracing_version; metadata_as_json["entity_id"] = metadata.entity_id; metadata_as_json["process_id"] = metadata.process_id; + metadata_as_json["process_name"] = metadata.process_name; metadata_as_json["host_name"] = metadata.host_name; metadata_as_json["topic_name"] = metadata.topic_name; metadata_as_json["encoding"] = metadata.encoding; @@ -65,7 +66,6 @@ namespace { json span_as_json; span_as_json["op_type"] = static_cast(span.op_type); span_as_json["entity_id"] = span.entity_id; - span_as_json["process_id"] = span.process_id; span_as_json["payload_size"] = span.payload_size; span_as_json["clock"] = span.clock; span_as_json["layer"] = span.layer; @@ -80,7 +80,6 @@ namespace { span_as_json["op_type"] = static_cast(span.op_type); span_as_json["entity_id"] = span.entity_id; span_as_json["topic_id"] = span.topic_id; - span_as_json["process_id"] = span.process_id; span_as_json["payload_size"] = span.payload_size; span_as_json["clock"] = span.clock; span_as_json["layer"] = span.layer; diff --git a/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp index eed25aad91..5006ce3e9e 100644 --- a/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp +++ b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp @@ -58,7 +58,6 @@ TEST(TestTraceProvider, ConcurrentSpanWrites) eCAL::tracing::SPublisherSpanData span{}; span.op_type = eCAL::tracing::operation_type::send; span.entity_id = static_cast(t * spans_per_thread + i); - span.process_id = 1; span.payload_size = 42; span.clock = static_cast(i); span.layer = eCAL::tracing::tl_trace_shm; diff --git a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp index c7d9cce069..8fea66d5a6 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp +++ b/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp @@ -58,7 +58,6 @@ TEST(TestTracingWriterJSONL, SpanWrites) eCAL::tracing::SPublisherSpanData span{}; span.op_type = eCAL::tracing::operation_type::send; span.entity_id = static_cast(i); - span.process_id = 1; span.payload_size = 64; span.clock = static_cast(i); span.layer = eCAL::tracing::tl_trace_shm; @@ -96,6 +95,7 @@ TEST(TestTracingWriterJSONL, MetadataWrites) eCAL::tracing::STopicMetadata metadata{}; metadata.entity_id = static_cast(i); metadata.process_id = 1; + metadata.process_name = "my_process"; metadata.host_name = "test_host"; metadata.topic_name = "topic_" + std::to_string(i); metadata.encoding = "protobuf"; @@ -128,7 +128,6 @@ TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) eCAL::tracing::SPublisherSpanData span{}; span.op_type = eCAL::tracing::operation_type::send; span.entity_id = 42; - span.process_id = 123; span.payload_size = 256; span.clock = 7; span.layer = eCAL::tracing::tl_trace_udp; @@ -152,7 +151,6 @@ TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); EXPECT_EQ(j.at("entity_id"), 42u); - EXPECT_EQ(j.at("process_id"), 123u); EXPECT_EQ(j.at("payload_size"), 256u); EXPECT_EQ(j.at("clock"), 7); EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); @@ -179,7 +177,6 @@ TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) span.op_type = eCAL::tracing::operation_type::receive; span.entity_id = 99; span.topic_id = 55; - span.process_id = 456; span.payload_size = 512; span.clock = 3; span.layer = eCAL::tracing::tl_trace_tcp; @@ -204,7 +201,6 @@ TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); EXPECT_EQ(j.at("entity_id"), 99u); EXPECT_EQ(j.at("topic_id"), 55u); - EXPECT_EQ(j.at("process_id"), 456u); EXPECT_EQ(j.at("payload_size"), 512u); EXPECT_EQ(j.at("clock"), 3); EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); From 2b322afba5add78f3b604dd33701efd6018bec20 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Mon, 29 Jun 2026 11:01:33 +0200 Subject: [PATCH 11/12] Flush after every batch. --- ecal/core/src/tracing/trace_provider_default.cpp | 2 +- ecal/core/src/tracing/trace_provider_default.h | 2 +- ecal/core/src/tracing/tracing_writer_jsonl.cpp | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ecal/core/src/tracing/trace_provider_default.cpp b/ecal/core/src/tracing/trace_provider_default.cpp index 042979d7cc..d046f43c66 100644 --- a/ecal/core/src/tracing/trace_provider_default.cpp +++ b/ecal/core/src/tracing/trace_provider_default.cpp @@ -36,7 +36,7 @@ namespace eCAL { return Util::CSingleInstanceHelper::Create(std::move(writer), batch_size); } - catch (const std::exception& e) + catch (const std::exception&) { return nullptr; } diff --git a/ecal/core/src/tracing/trace_provider_default.h b/ecal/core/src/tracing/trace_provider_default.h index 978afe904e..62e01689ce 100644 --- a/ecal/core/src/tracing/trace_provider_default.h +++ b/ecal/core/src/tracing/trace_provider_default.h @@ -53,7 +53,7 @@ namespace eCAL // Write span data to buffer (accepts any span type via variant) void WriteSpan(const SpanData& span_data) override; - // metadata — written directly to file (no buffering) + void WriteMetadata(const STopicMetadata& metadata) override; private: diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/ecal/core/src/tracing/tracing_writer_jsonl.cpp index 7f15dee0cb..c51ff239e0 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/ecal/core/src/tracing/tracing_writer_jsonl.cpp @@ -145,6 +145,16 @@ namespace eCAL span_variant ); } + + if (spans_file_.is_open()) + { + spans_file_.flush(); + } + + if (metadata_file_.is_open()) + { + metadata_file_.flush(); + } } void CTracingWriterJSONL::WriteSpanData(const SpanData& span_data) From f5708f1f13a803f9bdbc913f955b8576ba64f5b2 Mon Sep 17 00:00:00 2001 From: Kerstin Keller <13848742+KerstinKeller@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:06:15 +0200 Subject: [PATCH 12/12] Move tracing basics to contrib folder --- CMakeLists.txt | 9 ++ contrib/tracing/CMakeLists.txt | 51 +++++++ contrib/tracing/include/ecal/tracing/types.h | 97 ++++++++++++ .../tracing/include/ecal/tracing/writer.h | 8 +- .../include/ecal/tracing/writer_jsonl.h | 15 +- .../tracing/src/writer_jsonl.cpp | 32 +--- contrib/tracing/tests/CMakeLists.txt | 49 ++++++ .../tracing/tests/src/tracing_test_helpers.h | 69 +++++++++ .../tests}/src/tracing_writer_test.cpp | 139 +++++++----------- ecal/core/CMakeLists.txt | 9 +- ecal/core/src/pubsub/ecal_publisher_impl.cpp | 3 +- ecal/core/src/pubsub/ecal_subscriber_impl.cpp | 7 +- ecal/core/src/tracing/span.h | 2 +- ecal/core/src/tracing/trace_provider.cpp | 2 +- ecal/core/src/tracing/trace_provider.h | 2 +- .../src/tracing/trace_provider_default.cpp | 30 +++- .../core/src/tracing/trace_provider_default.h | 13 +- ecal/core/src/tracing/tracing.h | 128 ---------------- ecal/tests/cpp/tracing_test/CMakeLists.txt | 3 - .../tracing_test/src/trace_provider_test.cpp | 4 +- .../tracing_test/src/tracing_test_helpers.h | 73 +-------- 21 files changed, 392 insertions(+), 353 deletions(-) create mode 100644 contrib/tracing/CMakeLists.txt create mode 100644 contrib/tracing/include/ecal/tracing/types.h rename ecal/core/src/tracing/tracing_writer.h => contrib/tracing/include/ecal/tracing/writer.h (92%) rename ecal/core/src/tracing/tracing_writer_jsonl.h => contrib/tracing/include/ecal/tracing/writer_jsonl.h (80%) rename ecal/core/src/tracing/tracing_writer_jsonl.cpp => contrib/tracing/src/writer_jsonl.cpp (89%) create mode 100644 contrib/tracing/tests/CMakeLists.txt create mode 100644 contrib/tracing/tests/src/tracing_test_helpers.h rename {ecal/tests/cpp/tracing_test => contrib/tracing/tests}/src/tracing_writer_test.cpp (50%) delete mode 100644 ecal/core/src/tracing/tracing.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7499aba143..cdd2b514ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -364,6 +364,15 @@ if(ECAL_USE_PROTOBUF) add_subdirectory(app/app_pb) endif() +# -------------------------------------------------------- +# ecal tracing +# -------------------------------------------------------- +if(ECAL_BUILD_TESTS) + enable_testing() +endif() + +add_subdirectory(contrib/tracing) + # -------------------------------------------------------- # ecal core # -------------------------------------------------------- diff --git a/contrib/tracing/CMakeLists.txt b/contrib/tracing/CMakeLists.txt new file mode 100644 index 0000000000..985001cc8a --- /dev/null +++ b/contrib/tracing/CMakeLists.txt @@ -0,0 +1,51 @@ +# ========================= eCAL LICENSE ================================= +# +# Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================= eCAL LICENSE ================================= + +project(tracing) + +find_package(nlohmann_json REQUIRED) + +add_library(${PROJECT_NAME}) +add_library(eCAL::tracing ALIAS ${PROJECT_NAME}) + +target_sources(${PROJECT_NAME} + PUBLIC + include/ecal/tracing/types.h + include/ecal/tracing/writer.h + include/ecal/tracing/writer_jsonl.h + PRIVATE + src/writer_jsonl.cpp +) + +target_include_directories(${PROJECT_NAME} + PUBLIC + $ +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + nlohmann_json::nlohmann_json +) + +target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17) + +set_property(TARGET ${PROJECT_NAME} PROPERTY FOLDER contrib/tracing) + +if(ECAL_BUILD_TESTS) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/contrib/tracing/include/ecal/tracing/types.h b/contrib/tracing/include/ecal/tracing/types.h new file mode 100644 index 0000000000..5d0a60513c --- /dev/null +++ b/contrib/tracing/include/ecal/tracing/types.h @@ -0,0 +1,97 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include +#include +#include +#include + +namespace eCAL +{ + namespace tracing + { + constexpr const char* kTracingVersion = "1.0.0"; + + enum operation_type + { + send = 0, + receive = 1, + callback_execution = 2 + }; + + enum topic_direction + { + publisher = 0, + subscriber = 1 + }; + + enum eTracingLayerType : uint64_t + { + tl_trace_none = 0, + tl_trace_shm = 1 << 0, + tl_trace_udp = 1 << 1, + tl_trace_tcp = 1 << 2, + tl_trace_shm_udp = tl_trace_shm | tl_trace_udp, + tl_trace_shm_tcp = tl_trace_shm | tl_trace_tcp, + tl_trace_udp_tcp = tl_trace_udp | tl_trace_tcp, + tl_trace_all = tl_trace_shm | tl_trace_udp | tl_trace_tcp, + }; + + struct STopicMetadata + { + std::string tracing_version{kTracingVersion}; + uint64_t entity_id{ 0 }; + int32_t process_id{ 0 }; + std::string process_name; + std::string host_name; + std::string topic_name; + std::string encoding; + std::string type_name; + topic_direction direction{ topic_direction::publisher }; + }; + + struct SPublisherSpanData + { + operation_type op_type; + uint64_t entity_id; + size_t payload_size; + long long clock; + uint64_t layer; + long long start_ns; + long long end_ns; + }; + + struct SSubscriberSpanData + { + operation_type op_type; + uint64_t entity_id; + uint64_t topic_id; + size_t payload_size; + long long clock; + uint64_t layer; + long long start_ns; + long long end_ns; + }; + + using SpanData = std::variant; + using TraceInfo = std::variant; + } +} \ No newline at end of file diff --git a/ecal/core/src/tracing/tracing_writer.h b/contrib/tracing/include/ecal/tracing/writer.h similarity index 92% rename from ecal/core/src/tracing/tracing_writer.h rename to contrib/tracing/include/ecal/tracing/writer.h index fc604703fe..6f8ae6e964 100644 --- a/ecal/core/src/tracing/tracing_writer.h +++ b/contrib/tracing/include/ecal/tracing/writer.h @@ -19,16 +19,14 @@ #pragma once -#include -#include +#include -#include "tracing.h" +#include namespace eCAL { namespace tracing { - // Interface for tracing writers. class TracingWriter { public: @@ -37,4 +35,4 @@ namespace eCAL virtual void WriteTraceInfo(const std::vector& batch) = 0; }; } -} +} \ No newline at end of file diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.h b/contrib/tracing/include/ecal/tracing/writer_jsonl.h similarity index 80% rename from ecal/core/src/tracing/tracing_writer_jsonl.h rename to contrib/tracing/include/ecal/tracing/writer_jsonl.h index 16e102b389..9d749bc4ed 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.h +++ b/contrib/tracing/include/ecal/tracing/writer_jsonl.h @@ -19,26 +19,21 @@ #pragma once +#include +#include + #include #include #include #include -#include - -#include "tracing_writer.h" -#include "tracing.h" namespace eCAL { namespace tracing { - // Responsible for serializing span and metadata to JSONL files. - // Separated from CTraceProvider to isolate the I/O concern. - // This class is not thread-safe; CTraceProviderDefault handles synchronization and buffering before calling this writer. class CTracingWriterJSONL : public TracingWriter { public: - CTracingWriterJSONL(); CTracingWriterJSONL(const std::string& file_id, std::filesystem::path trace_directory); ~CTracingWriterJSONL() override = default; @@ -47,10 +42,8 @@ namespace eCAL CTracingWriterJSONL(CTracingWriterJSONL&&) = delete; CTracingWriterJSONL& operator=(CTracingWriterJSONL&&) = delete; - // Write a batch of spans to the JSONL spans file void WriteTraceInfo(const std::vector& batch) override; - // File path accessors (path is fixed at construction time) std::filesystem::path GetSpansFilePath() const; std::filesystem::path GetTopicMetadataFilePath() const; @@ -65,4 +58,4 @@ namespace eCAL std::ofstream metadata_file_; }; } -} +} \ No newline at end of file diff --git a/ecal/core/src/tracing/tracing_writer_jsonl.cpp b/contrib/tracing/src/writer_jsonl.cpp similarity index 89% rename from ecal/core/src/tracing/tracing_writer_jsonl.cpp rename to contrib/tracing/src/writer_jsonl.cpp index c51ff239e0..708ef99c34 100644 --- a/ecal/core/src/tracing/tracing_writer_jsonl.cpp +++ b/contrib/tracing/src/writer_jsonl.cpp @@ -17,20 +17,15 @@ * ========================= eCAL LICENSE ================================= */ -#include "tracing_writer_jsonl.h" +#include +#include + +#include #include #include -#include -#include -#include -#include #include -#include -#include - -#include using json = nlohmann::json; namespace { @@ -43,7 +38,6 @@ namespace { template Overloaded(Ts...) -> Overloaded; - using namespace eCAL::tracing; json toJson(const STopicMetadata& metadata) @@ -88,15 +82,6 @@ namespace { return span_as_json; } - std::string GetCurrentTimestamp() - { - std::time_t now = std::time(nullptr); - std::tm* tm_info = std::localtime(&now); - std::ostringstream oss; - oss << std::put_time(tm_info, "%Y%m%d_%H%M%S"); - return oss.str(); - } - std::ofstream CreateWithDirectory(const std::filesystem::path& file_path, std::ios_base::openmode flags) { std::filesystem::create_directories(file_path.parent_path()); @@ -108,12 +93,6 @@ namespace eCAL { namespace tracing { - CTracingWriterJSONL::CTracingWriterJSONL() - : CTracingWriterJSONL(std::to_string(eCAL::Process::GetProcessID()) + "_" + GetCurrentTimestamp(), std::filesystem::path(Util::GeteCALTraceDir())) - { - } - - CTracingWriterJSONL::CTracingWriterJSONL(const std::string& file_id, std::filesystem::path trace_directory) : spans_file_path_(trace_directory / (std::string("ecal_spans_") + file_id + ".jsonl")) , metadata_file_path_(trace_directory / (std::string("ecal_metadata_") + file_id + ".jsonl")) @@ -132,7 +111,6 @@ namespace eCAL return metadata_file_path_; } - void CTracingWriterJSONL::WriteTraceInfo(const std::vector& batch) { for (const auto& span_variant : batch) @@ -189,4 +167,4 @@ namespace eCAL } } } -} +} \ No newline at end of file diff --git a/contrib/tracing/tests/CMakeLists.txt b/contrib/tracing/tests/CMakeLists.txt new file mode 100644 index 0000000000..0baf23d141 --- /dev/null +++ b/contrib/tracing/tests/CMakeLists.txt @@ -0,0 +1,49 @@ +# ========================= eCAL LICENSE ================================= +# +# Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================= eCAL LICENSE ================================= + +project(test_contrib_tracing) + +find_package(GTest REQUIRED) +find_package(nlohmann_json REQUIRED) + +set(tracing_test_src + src/tracing_test_helpers.h + src/tracing_writer_test.cpp +) + +ecal_add_gtest(${PROJECT_NAME} ${tracing_test_src}) + +target_include_directories(${PROJECT_NAME} PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/src +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + eCAL::tracing + nlohmann_json::nlohmann_json +) + +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) + +ecal_install_gtest(${PROJECT_NAME}) + +set_property(TARGET ${PROJECT_NAME} PROPERTY FOLDER tests/contrib/tracing) + +source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" FILES + ${tracing_test_src} +) \ No newline at end of file diff --git a/contrib/tracing/tests/src/tracing_test_helpers.h b/contrib/tracing/tests/src/tracing_test_helpers.h new file mode 100644 index 0000000000..0f9f4ac2bf --- /dev/null +++ b/contrib/tracing/tests/src/tracing_test_helpers.h @@ -0,0 +1,69 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#pragma once + +#include +#include + +#include +#include +#include + +class ScopedTestDirectory +{ +public: + explicit ScopedTestDirectory(const std::string& name) + : path_(std::filesystem::temp_directory_path() / name) + { + std::filesystem::remove_all(path_); + std::filesystem::create_directories(path_); + } + + ~ScopedTestDirectory() + { + std::filesystem::remove_all(path_); + } + + const std::filesystem::path& Path() const + { + return path_; + } + +private: + std::filesystem::path path_; +}; + +inline size_t CountAndValidateJsonlLines(const std::filesystem::path& filepath) +{ + std::ifstream file(filepath); + EXPECT_TRUE(file.is_open()) << "Failed to open: " << filepath; + + size_t count = 0; + std::string line; + while (std::getline(file, line)) + { + if (line.empty()) continue; + EXPECT_NO_THROW(nlohmann::json::parse(line)) + << "Invalid JSON on line " << (count + 1) << ": " << line; + ++count; + } + return count; +} \ No newline at end of file diff --git a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp b/contrib/tracing/tests/src/tracing_writer_test.cpp similarity index 50% rename from ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp rename to contrib/tracing/tests/src/tracing_writer_test.cpp index 8fea66d5a6..d5d6441e8d 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_writer_test.cpp +++ b/contrib/tracing/tests/src/tracing_writer_test.cpp @@ -1,6 +1,7 @@ /* ========================= eCAL LICENSE ================================= * * Copyright (C) 2016 - 2025 Continental Corporation + * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,35 +20,26 @@ #include "tracing_test_helpers.h" -#include - -#include -#include +#include +#include #include -#include - -#include -#include #include #include -#include #include #include TEST(TestTracingWriterJSONL, SpanWrites) { - constexpr size_t total_spans = 4000; - ScopedTraceDirOverride trace_dir_override("./"); - auto ecal_config = GetTracingConfiguration(); - ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + constexpr size_t total_spans = 4000; + ScopedTestDirectory trace_directory("ecal_tracing_writer_span_writes"); std::filesystem::path spans_path; { - eCAL::tracing::CTracingWriterJSONL writer; + eCAL::tracing::CTracingWriterJSONL writer("span_writes", trace_directory.Path()); spans_path = writer.GetSpansFilePath(); std::vector batch; @@ -69,60 +61,48 @@ TEST(TestTracingWriterJSONL, SpanWrites) writer.WriteTraceInfo(batch); } - eCAL::Finalize(); - - // Verify: every span line must be valid JSON and the total count must match. - size_t line_count = CountAndValidateJsonlLines(spans_path); - EXPECT_EQ(line_count, total_spans); - std::filesystem::remove(spans_path); + EXPECT_EQ(spans_path.filename().string(), "ecal_spans_span_writes.jsonl"); + EXPECT_EQ(CountAndValidateJsonlLines(spans_path), total_spans); } TEST(TestTracingWriterJSONL, MetadataWrites) { - constexpr size_t metadata_instances = 200; - ScopedTraceDirOverride trace_dir_override("./"); - auto ecal_config = GetTracingConfiguration(); - ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + constexpr size_t metadata_instances = 200; + ScopedTestDirectory trace_directory("ecal_tracing_writer_metadata_writes"); std::filesystem::path metadata_path; { - eCAL::tracing::CTracingWriterJSONL writer; + eCAL::tracing::CTracingWriterJSONL writer("metadata_writes", trace_directory.Path()); metadata_path = writer.GetTopicMetadataFilePath(); for (size_t i = 0; i < metadata_instances; ++i) { eCAL::tracing::STopicMetadata metadata{}; - metadata.entity_id = static_cast(i); - metadata.process_id = 1; + metadata.entity_id = static_cast(i); + metadata.process_id = 1; metadata.process_name = "my_process"; - metadata.host_name = "test_host"; - metadata.topic_name = "topic_" + std::to_string(i); - metadata.encoding = "protobuf"; - metadata.type_name = "TestType"; - metadata.direction = eCAL::tracing::topic_direction::publisher; + metadata.host_name = "test_host"; + metadata.topic_name = "topic_" + std::to_string(i); + metadata.encoding = "protobuf"; + metadata.type_name = "TestType"; + metadata.direction = eCAL::tracing::topic_direction::publisher; writer.WriteTraceInfo({metadata}); } } - eCAL::Finalize(); - - size_t line_count = CountAndValidateJsonlLines(metadata_path); - EXPECT_EQ(line_count, metadata_instances); - - std::filesystem::remove(metadata_path); + EXPECT_EQ(metadata_path.filename().string(), "ecal_metadata_metadata_writes.jsonl"); + EXPECT_EQ(CountAndValidateJsonlLines(metadata_path), metadata_instances); } TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) { - ScopedTraceDirOverride trace_dir_override("./"); - auto ecal_config = GetTracingConfiguration(); - ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + ScopedTestDirectory trace_directory("ecal_tracing_writer_publisher_fields"); std::filesystem::path spans_path; { - eCAL::tracing::CTracingWriterJSONL writer; + eCAL::tracing::CTracingWriterJSONL writer("publisher_fields", trace_directory.Path()); spans_path = writer.GetSpansFilePath(); eCAL::tracing::SPublisherSpanData span{}; @@ -139,38 +119,30 @@ TEST(TestTracingWriterJSONL, PublisherSpanJsonFields) writer.WriteTraceInfo(batch); } - eCAL::Finalize(); + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); - { - std::ifstream file(spans_path); - ASSERT_TRUE(file.is_open()); - - std::string line; - ASSERT_TRUE(std::getline(file, line)); - auto j = nlohmann::json::parse(line); - - EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); - EXPECT_EQ(j.at("entity_id"), 42u); - EXPECT_EQ(j.at("payload_size"), 256u); - EXPECT_EQ(j.at("clock"), 7); - EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); - EXPECT_EQ(j.at("start_ns"), 1000); - EXPECT_EQ(j.at("end_ns"), 2000); - } + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); - std::filesystem::remove(spans_path); + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::send)); + EXPECT_EQ(j.at("entity_id"), 42u); + EXPECT_EQ(j.at("payload_size"), 256u); + EXPECT_EQ(j.at("clock"), 7); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_udp)); + EXPECT_EQ(j.at("start_ns"), 1000); + EXPECT_EQ(j.at("end_ns"), 2000); } TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) { - ScopedTraceDirOverride trace_dir_override("./"); - auto ecal_config = GetTracingConfiguration(); - ASSERT_TRUE(eCAL::Initialize(ecal_config, "", eCAL::Init::None)); + ScopedTestDirectory trace_directory("ecal_tracing_writer_subscriber_fields"); std::filesystem::path spans_path; { - eCAL::tracing::CTracingWriterJSONL writer; + eCAL::tracing::CTracingWriterJSONL writer("subscriber_fields", trace_directory.Path()); spans_path = writer.GetSpansFilePath(); eCAL::tracing::SSubscriberSpanData span{}; @@ -188,26 +160,19 @@ TEST(TestTracingWriterJSONL, SubscriberSpanJsonFields) writer.WriteTraceInfo(batch); } - eCAL::Finalize(); - - { - std::ifstream file(spans_path); - ASSERT_TRUE(file.is_open()); - - std::string line; - ASSERT_TRUE(std::getline(file, line)); - auto j = nlohmann::json::parse(line); - - EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); - EXPECT_EQ(j.at("entity_id"), 99u); - EXPECT_EQ(j.at("topic_id"), 55u); - EXPECT_EQ(j.at("payload_size"), 512u); - EXPECT_EQ(j.at("clock"), 3); - EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); - EXPECT_EQ(j.at("start_ns"), 3000); - EXPECT_EQ(j.at("end_ns"), 4000); - } - - std::filesystem::remove(spans_path); -} - + std::ifstream file(spans_path); + ASSERT_TRUE(file.is_open()); + + std::string line; + ASSERT_TRUE(std::getline(file, line)); + auto j = nlohmann::json::parse(line); + + EXPECT_EQ(j.at("op_type"), static_cast(eCAL::tracing::operation_type::receive)); + EXPECT_EQ(j.at("entity_id"), 99u); + EXPECT_EQ(j.at("topic_id"), 55u); + EXPECT_EQ(j.at("payload_size"), 512u); + EXPECT_EQ(j.at("clock"), 3); + EXPECT_EQ(j.at("layer"), static_cast(eCAL::tracing::tl_trace_tcp)); + EXPECT_EQ(j.at("start_ns"), 3000); + EXPECT_EQ(j.at("end_ns"), 4000); +} \ No newline at end of file diff --git a/ecal/core/CMakeLists.txt b/ecal/core/CMakeLists.txt index 59d0dea3a3..a11b2c82a2 100644 --- a/ecal/core/CMakeLists.txt +++ b/ecal/core/CMakeLists.txt @@ -43,7 +43,6 @@ find_package(asio REQUIRED) find_package(Threads REQUIRED) find_package(ecaludp REQUIRED) find_package(protozero REQUIRED) -find_package(nlohmann_json REQUIRED) if (ECAL_CORE_CONFIGURATION) find_package(yaml-cpp REQUIRED) @@ -462,16 +461,14 @@ endif() # tracing ###################################### set(ecal_tracing_src - src/tracing/tracing.h + src/tracing/layer_type.cpp + src/tracing/layer_type.h src/tracing/span.cpp src/tracing/span.h src/tracing/trace_provider.cpp src/tracing/trace_provider.h src/tracing/trace_provider_default.cpp src/tracing/trace_provider_default.h - src/tracing/tracing_writer.h - src/tracing/tracing_writer_jsonl.cpp - src/tracing/tracing_writer_jsonl.h ) ###################################### @@ -726,7 +723,6 @@ target_link_libraries(ecal_core_private $<$:tcp_pubsub::tcp_pubsub> $<$:ecaltime> ecaludp::ecaludp - nlohmann_json::nlohmann_json ${CMAKE_DL_LIBS} $<$,$>,$>>:rt> $<$,$>,$>>:atomic> @@ -741,6 +737,7 @@ target_link_libraries(ecal_core_private asio::asio Threads::Threads eCAL::ecal-utils + eCAL::tracing PRIVATE # we need this so that e.g. the test can just link ecal_core_private and will get the object files of this library. ecal_core_serialization $ diff --git a/ecal/core/src/pubsub/ecal_publisher_impl.cpp b/ecal/core/src/pubsub/ecal_publisher_impl.cpp index 47ae47af7c..33e6404d13 100644 --- a/ecal/core/src/pubsub/ecal_publisher_impl.cpp +++ b/ecal/core/src/pubsub/ecal_publisher_impl.cpp @@ -46,10 +46,11 @@ #include "registration/ecal_registration_provider.h" -#include "tracing/tracing.h" #include "tracing/span.h" #include "tracing/trace_provider.h" +#include + #include #include #include diff --git a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp index 2edca61a9f..56b7dd003b 100644 --- a/ecal/core/src/pubsub/ecal_subscriber_impl.cpp +++ b/ecal/core/src/pubsub/ecal_subscriber_impl.cpp @@ -23,9 +23,10 @@ **/ #include "ecal_subscriber_impl.h" -#include "tracing/tracing.h" +#include "tracing/layer_type.h" #include "tracing/span.h" #include "tracing/trace_provider.h" + #include #include #include @@ -397,7 +398,7 @@ namespace eCAL m_subscriber_id, topic_info_, clock_, - eCAL::tracing::toTracingLayerType(layer_), + tracing::ToTracingLayerType(layer_), size_, eCAL::tracing::operation_type::receive ); @@ -484,7 +485,7 @@ namespace eCAL m_subscriber_id, topic_info_, clock_, - eCAL::tracing::toTracingLayerType(layer_), + tracing::ToTracingLayerType(layer_), size_, eCAL::tracing::operation_type::callback_execution ); diff --git a/ecal/core/src/tracing/span.h b/ecal/core/src/tracing/span.h index fcc02053f6..9d43d1fd15 100644 --- a/ecal/core/src/tracing/span.h +++ b/ecal/core/src/tracing/span.h @@ -22,7 +22,7 @@ #include #include -#include "tracing.h" +#include #include #include diff --git a/ecal/core/src/tracing/trace_provider.cpp b/ecal/core/src/tracing/trace_provider.cpp index 6f9ec62dd2..f855ce8ea0 100644 --- a/ecal/core/src/tracing/trace_provider.cpp +++ b/ecal/core/src/tracing/trace_provider.cpp @@ -28,7 +28,7 @@ namespace eCAL { if (config_.enabled) { - return CTraceProviderDefault::Create(); + return CTraceProviderDefault::CreateDefault(); } return nullptr; } diff --git a/ecal/core/src/tracing/trace_provider.h b/ecal/core/src/tracing/trace_provider.h index f48937bf13..1d9f5a545f 100644 --- a/ecal/core/src/tracing/trace_provider.h +++ b/ecal/core/src/tracing/trace_provider.h @@ -19,7 +19,7 @@ #pragma once -#include "tracing.h" +#include #include diff --git a/ecal/core/src/tracing/trace_provider_default.cpp b/ecal/core/src/tracing/trace_provider_default.cpp index d046f43c66..7a9cedf968 100644 --- a/ecal/core/src/tracing/trace_provider_default.cpp +++ b/ecal/core/src/tracing/trace_provider_default.cpp @@ -18,14 +18,32 @@ */ #include "trace_provider_default.h" -#include "tracing_writer.h" -#include "tracing_writer_jsonl.h" #include "util/single_instance_helper.h" -#include +#include +#include +#include +#include + #include +#include +#include +#include +#include #include +namespace +{ + std::string GetCurrentTimestamp() + { + std::time_t now = std::time(nullptr); + std::tm* tm_info = std::localtime(&now); + std::ostringstream oss; + oss << std::put_time(tm_info, "%Y%m%d_%H%M%S"); + return oss.str(); + } +} + namespace eCAL { namespace tracing @@ -42,6 +60,12 @@ namespace eCAL } } + std::shared_ptr CTraceProviderDefault::CreateDefault(size_t batch_size) + { + const auto file_id = std::to_string(eCAL::Process::GetProcessID()) + "_" + GetCurrentTimestamp(); + return Create(std::make_unique(file_id, std::filesystem::path(Util::GeteCALTraceDir())), batch_size); + } + CTraceProviderDefault::CTraceProviderDefault(std::unique_ptr writer, size_t batch_size) : batch_size_(batch_size), writer_(std::move(writer)) diff --git a/ecal/core/src/tracing/trace_provider_default.h b/ecal/core/src/tracing/trace_provider_default.h index 62e01689ce..a4656ba78a 100644 --- a/ecal/core/src/tracing/trace_provider_default.h +++ b/ecal/core/src/tracing/trace_provider_default.h @@ -20,11 +20,13 @@ #pragma once #include "trace_provider.h" -#include "tracing.h" -#include "tracing_writer.h" -#include "tracing_writer_jsonl.h" #include "util/single_instance_helper.h" +#include +#include +#include + +#include #include #include #include @@ -36,12 +38,15 @@ namespace eCAL { namespace tracing { + constexpr size_t kDefaultTracingBatchSize = 128; + class CTraceProviderDefault : public TraceProvider { friend class Util::CSingleInstanceHelper; public: - static std::shared_ptr Create(std::unique_ptr writer = std::make_unique(), size_t batch_size = kDefaultTracingBatchSize); + static std::shared_ptr Create(std::unique_ptr writer, size_t batch_size = kDefaultTracingBatchSize); + static std::shared_ptr CreateDefault(size_t batch_size = kDefaultTracingBatchSize); CTraceProviderDefault(const CTraceProviderDefault&) = delete; CTraceProviderDefault& operator=(const CTraceProviderDefault&) = delete; diff --git a/ecal/core/src/tracing/tracing.h b/ecal/core/src/tracing/tracing.h deleted file mode 100644 index 301a4009fd..0000000000 --- a/ecal/core/src/tracing/tracing.h +++ /dev/null @@ -1,128 +0,0 @@ -/* ========================= eCAL LICENSE ================================= - * - * Copyright 2026 AUMOVIO and subsidiaries. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * ========================= eCAL LICENSE ================================= -*/ - -/** - * @file tracing/tracing.h - * @brief Shared tracing types, enums, and constants used across the tracing subsystem. -**/ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace eCAL -{ - namespace tracing - { - // Version of the tracing implementation. - constexpr const char* kTracingVersion = "1.0.0"; - - // Default batch size for span buffering before flushing to backend - constexpr size_t kDefaultTracingBatchSize = 128; - - // Specifies the type of operation being traced - enum operation_type - { - send = 0, - receive = 1, - callback_execution = 2 - }; - - // Specifies the direction of the topic (publisher or subscriber) - enum topic_direction - { - publisher = 0, - subscriber = 1 - }; - - // Bitmask enum for active transport layers used in tracing spans. - // These use power-of-two values so combinations can be expressed with bitwise OR. - enum eTracingLayerType : uint64_t - { - tl_trace_none = 0, - tl_trace_shm = 1 << 0, // 1 - tl_trace_udp = 1 << 1, // 2 - tl_trace_tcp = 1 << 2, // 4 - tl_trace_shm_udp = tl_trace_shm | tl_trace_udp, // 3 - tl_trace_shm_tcp = tl_trace_shm | tl_trace_tcp, // 5 - tl_trace_udp_tcp = tl_trace_udp | tl_trace_tcp, // 6 - tl_trace_all = tl_trace_shm | tl_trace_udp | tl_trace_tcp, // 7 - }; - - // Convert from eTLayerType (used in core APIs) to eTracingLayerType. - inline eTracingLayerType toTracingLayerType(eTLayerType layer) - { - switch (layer) - { - case tl_ecal_shm: return tl_trace_shm; - case tl_ecal_udp: return tl_trace_udp; - case tl_ecal_tcp: return tl_trace_tcp; - case tl_all: return tl_trace_all; - default: return tl_trace_none; - } - } - - // Metadata captured when a topic is created - struct STopicMetadata - { - std::string tracing_version{kTracingVersion}; // tracing format version - uint64_t entity_id{ 0 }; // unique entity id - int32_t process_id{ 0 }; // PID of the owning process - std::string process_name; // process name of the owning process - std::string host_name; // host that created the topic - std::string topic_name; // topic name used for pub/sub matching - std::string encoding; // datatype encoding (e.g. protobuf) - std::string type_name; // datatype name - topic_direction direction{ topic_direction::publisher }; // publisher or subscriber - }; - - struct SPublisherSpanData - { - operation_type op_type; - uint64_t entity_id; - size_t payload_size; - long long clock; - uint64_t layer; - long long start_ns; // start timestamp in nanoseconds - long long end_ns; // end timestamp in nanoseconds - }; - - struct SSubscriberSpanData - { - operation_type op_type; - uint64_t entity_id; - uint64_t topic_id; - size_t payload_size; - long long clock; - uint64_t layer; - long long start_ns; // start timestamp in nanoseconds - long long end_ns; // end timestamp in nanoseconds - }; - - // Variant type for buffering heterogeneous span data. - // Extend this variant when new span types are added (e.g. client/server). - using SpanData = std::variant; - using TraceInfo = std::variant< STopicMetadata, SpanData>; - } -} diff --git a/ecal/tests/cpp/tracing_test/CMakeLists.txt b/ecal/tests/cpp/tracing_test/CMakeLists.txt index 4953253852..21a6f4546b 100644 --- a/ecal/tests/cpp/tracing_test/CMakeLists.txt +++ b/ecal/tests/cpp/tracing_test/CMakeLists.txt @@ -20,12 +20,10 @@ project(test_tracing) find_package(Threads REQUIRED) find_package(GTest REQUIRED) -find_package(nlohmann_json REQUIRED) set(tracing_test_src src/tracing_test_helpers.h src/trace_provider_test.cpp - src/tracing_writer_test.cpp ) ecal_add_gtest(${PROJECT_NAME} ${tracing_test_src}) @@ -42,7 +40,6 @@ target_link_libraries(${PROJECT_NAME} ecal_core_private eCAL::ecal-utils Threads::Threads - nlohmann_json::nlohmann_json ) target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) diff --git a/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp index 5006ce3e9e..a4bb64f686 100644 --- a/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp +++ b/ecal/tests/cpp/tracing_test/src/trace_provider_test.cpp @@ -20,7 +20,9 @@ #include "tracing_test_helpers.h" #include -#include + +#include +#include #include diff --git a/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h index 5ae705a480..a28150d618 100644 --- a/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h +++ b/ecal/tests/cpp/tracing_test/src/tracing_test_helpers.h @@ -19,63 +19,13 @@ #pragma once -#include - -#include -#include - -#include -#include +#include +#include #include -#include -#include #include -#include #include -class ScopedTraceDirOverride -{ -public: - explicit ScopedTraceDirOverride(const std::string& path) - { - if (const char* previous_value = std::getenv(kTraceDirEnvVar)) - { - had_previous_value_ = true; - previous_value_ = previous_value; - } - -#ifdef _WIN32 - _putenv_s(kTraceDirEnvVar, path.c_str()); -#else - setenv(kTraceDirEnvVar, path.c_str(), 1); -#endif - } - - ~ScopedTraceDirOverride() - { -#ifdef _WIN32 - _putenv_s(kTraceDirEnvVar, had_previous_value_ ? previous_value_.c_str() : ""); -#else - if (had_previous_value_) - setenv(kTraceDirEnvVar, previous_value_.c_str(), 1); - else - unsetenv(kTraceDirEnvVar); -#endif - } - -private: - static constexpr const char* kTraceDirEnvVar = "ECAL_TRACE_DIR"; - - bool had_previous_value_{false}; - std::string previous_value_; -}; - -inline eCAL::Configuration GetTracingConfiguration() -{ - return eCAL::Configuration{}; -} - // Mock writer that counts spans and metadata for test assertions. class MockTracingWriter : public eCAL::tracing::TracingWriter { @@ -136,22 +86,3 @@ class ProxyTracingWriter : public eCAL::tracing::TracingWriter MockTracingWriter& target_; }; -// Count the number of lines in a file and validate each line is valid JSON. -inline size_t CountAndValidateJsonlLines(const std::filesystem::path& filepath) -{ - std::ifstream file(filepath); - EXPECT_TRUE(file.is_open()) << "Failed to open: " << filepath; - - size_t count = 0; - std::string line; - while (std::getline(file, line)) - { - if (line.empty()) continue; - // Every line must be valid JSON — a corrupted/interleaved write would break parsing. - EXPECT_NO_THROW(nlohmann::json::parse(line)) - << "Invalid JSON on line " << (count + 1) << ": " << line; - ++count; - } - return count; -} -