From 177d7e3144c485f71b6aaa77d83e6d863cfc8ae3 Mon Sep 17 00:00:00 2001 From: Lukasz Towarek Date: Tue, 14 Jul 2026 19:07:33 +0000 Subject: [PATCH] Add esp-idf profiling Fixes #93 --- AGENTS.md | 2 +- CONTRIBUTING.md | 49 +++++++++++ car/components/camera/camera_metrics.cpp | 2 +- car/components/esp-opentelemetry-cpp | 2 +- .../telemetry/telemetry_metrics.cpp | 2 +- car/components/tracing/CMakeLists.txt | 3 +- car/components/tracing/Kconfig.projbuild | 32 +------- car/components/tracing/include/metrics.hpp | 9 -- car/components/tracing/include/tracing.hpp | 6 -- car/components/tracing/metrics.cpp | 82 ------------------- car/components/tracing/system_metrics.cpp | 22 ++--- car/components/tracing/tracing.cpp | 22 ----- .../web_server/test_apps/main/main.cpp | 4 +- car/components/web_server/web_server.cpp | 3 + car/dependencies.lock | 2 +- car/main/main.cpp | 11 ++- car/sdkconfig.defaults | 2 +- car/sdkconfig.defaults.all | 4 +- car/sdkconfig.defaults.profiling | 14 ++++ car/test_apps/integration/main/main.cpp | 5 +- docker-compose.yml | 10 +++ observability/docker-compose.yml | 33 +++++++- .../provisioning/datasources/pyroscope.yml | 10 +++ .../provisioning/datasources/tempo.yml | 8 ++ observability/otelcol.yml | 18 +++- observability/pyroscope.yml | 11 +++ 26 files changed, 186 insertions(+), 182 deletions(-) delete mode 100644 car/components/tracing/include/metrics.hpp delete mode 100644 car/components/tracing/metrics.cpp create mode 100644 car/sdkconfig.defaults.profiling create mode 100644 observability/grafana/provisioning/datasources/pyroscope.yml create mode 100644 observability/pyroscope.yml diff --git a/AGENTS.md b/AGENTS.md index af77ff6..a9cfd70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ This file is a lightweight navigation guide for coding agents. Run `./scripts/start_headless.sh` to build and start all services in the background. Run `./scripts/stop_headless.sh` to stop and remove all containers. -Docker Compose services: `cpp`, `python`, `js`, `otel-collector`, `tempo`, `grafana`. Use standard docker compose commands to manage them: +Docker Compose services: `cpp`, `python`, `js`, `otel-collector`, `tempo`, `victoriametrics`, `pyroscope`, `profiling-symbolizer`, `grafana`. Use standard docker compose commands to manage them: - Logs: `docker compose -f docker-compose.yml -f docker-compose.headless.yml logs [-f] ` - Restart: `docker compose -f docker-compose.yml -f docker-compose.headless.yml restart ` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bfe5471..4897fdf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,8 @@ All three services export traces via OTLP/HTTP to the OTel Collector and metrics | Python streamer | `dust-mite-streamer` | `OTEL_EXPORTER_OTLP_ENDPOINT` (default `http://otel-collector:4318`) | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` (default `http://victoriametrics:8428/opentelemetry/v1/metrics`) | | Web browser | `dust-mite-web` | `VITE_OTLP_ENDPOINT` (default `http://localhost:4318`) | `VITE_OTLP_METRICS_ENDPOINT` (default `http://localhost:4318`) via OTel Collector | +The ESP32 cannot resolve the observability stack's Docker DNS names (e.g. `otel-collector`), so its endpoints must be the host machine's LAN IP rather than a container service name. + ### Metrics The set of emitted metrics is variant-specific. See the variant documentation in [docs/variants/](docs/variants/) for the full list. When adding or removing a metric, update the metrics table in the relevant variant document. @@ -242,6 +244,53 @@ Open Grafana at `http://localhost:3000` → **Explore** → select the **Tempo** - `{resource.service.name="dust-mite-car"}` — firmware spans only - `{resource.service.name="dust-mite-streamer"}` — streamer spans only +### Profiling (firmware CPU) + +Spans and metrics tell you *which component* is slow; the statistical profiler tells you +*which code* is burning CPU on the ESP32-S3, so you can decide between optimising the code, +changing FreeRTOS scheduling (core/priority), or accepting a hardware limit. It is a sampling +profiler exported as the OpenTelemetry **profiles** signal and viewed as Grafana **Pyroscope** +flame graphs. + +For how the sampler, exporter, and symbolizer work internally, see the +[esp-opentelemetry-cpp README](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations). + +To profile a build: + +1. Bring up the stack (`./scripts/start_headless.sh`) — it includes `pyroscope` and + `profiling-symbolizer`. +2. Build/flash the firmware with the profiling overlay (enables the sampler, the + span→profile task slot, and the symbolizer endpoint; see + [car/sdkconfig.defaults.profiling](car/sdkconfig.defaults.profiling)): + `SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.defaults.profiling" idf.py build flash` + Symbols need no publishing step: the symbolizer mounts `car/` and finds the + matching ELF by hash (the profile `build_id` equals the ELF's sha256). +3. Open Grafana → **Explore** → **Pyroscope** datasource. Filter by + `{service_name="dust-mite-car"}`, optionally split by `thread_name` (FreeRTOS task). Idle + task dominance means the workload is not CPU-bound (hardware/scheduling limited); a specific + application or esp-idf frame dominating means there is code to optimise. + +QEMU is not used for profiling — its Xtensa core is not cycle-accurate, so durations would be +fictional. The sampler/span-link/exporter mechanics are covered by +[esp-opentelemetry-cpp](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations)'s +own CI (QEMU); dust-mite's CI does not build the profiling overlay at all, so it is validated +live on real hardware only. + +#### Span profiles (trace → flame graph) + +Profiles are linked to traces the same way Go/Python do it; see `esp_task_span_slot.cpp` in the +[esp-opentelemetry-cpp README](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations) +for the mechanism. No changes are needed at span call sites. + +In a Tempo trace view, spans carry a **Profiles for this span** link/flame-graph tab +(provisioned via `tracesToProfilesV2` on the Tempo datasource) showing the CPU sampled while +that exact span was active. Spans are also stamped with the `pyroscope.profile.id` attribute +Grafana uses to enable the embedded flame graph. + +Expectations: at 100 Hz/core a 5 ms span catches at most one sample — per-span flame graphs +are meaningful for spans of tens of milliseconds or when aggregated across many span +instances. Raise `CONFIG_ESP_OPENTELEMETRY_PROFILING_SAMPLE_HZ` for a session if finer coverage is needed. + ## Variants Variants use code names based on chemical elements (for example `Copper`, `Iron`). diff --git a/car/components/camera/camera_metrics.cpp b/car/components/camera/camera_metrics.cpp index e0d71c0..c1dc4ab 100644 --- a/car/components/camera/camera_metrics.cpp +++ b/car/components/camera/camera_metrics.cpp @@ -3,7 +3,7 @@ #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED #include -#include "metrics.hpp" +#include "esp_metrics.hpp" #include "opentelemetry/metrics/async_instruments.h" #include "opentelemetry/metrics/observer_result.h" #include "opentelemetry/metrics/provider.h" diff --git a/car/components/esp-opentelemetry-cpp b/car/components/esp-opentelemetry-cpp index e6559e9..b351c9f 160000 --- a/car/components/esp-opentelemetry-cpp +++ b/car/components/esp-opentelemetry-cpp @@ -1 +1 @@ -Subproject commit e6559e98fdb5a5e880e06c96f593ea1a6d7c0fd9 +Subproject commit b351c9f05603eac5bdf280e0856e835d45e9bb1d diff --git a/car/components/telemetry/telemetry_metrics.cpp b/car/components/telemetry/telemetry_metrics.cpp index c1bec0b..f8edb6f 100644 --- a/car/components/telemetry/telemetry_metrics.cpp +++ b/car/components/telemetry/telemetry_metrics.cpp @@ -1,5 +1,5 @@ #include "telemetry_metrics.hpp" -#include "metrics.hpp" +#include "esp_metrics.hpp" #include "sdkconfig.h" #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED diff --git a/car/components/tracing/CMakeLists.txt b/car/components/tracing/CMakeLists.txt index 9653600..f5bdeed 100644 --- a/car/components/tracing/CMakeLists.txt +++ b/car/components/tracing/CMakeLists.txt @@ -1,9 +1,8 @@ -idf_component_register(SRCS "system_metrics.cpp" "tracing.cpp" "metrics.cpp" +idf_component_register(SRCS "system_metrics.cpp" "tracing.cpp" INCLUDE_DIRS "include" REQUIRES esp-opentelemetry-cpp cjson - pthread heap esp_timer PRIV_REQUIRES diff --git a/car/components/tracing/Kconfig.projbuild b/car/components/tracing/Kconfig.projbuild index a6e513d..4f2bac9 100644 --- a/car/components/tracing/Kconfig.projbuild +++ b/car/components/tracing/Kconfig.projbuild @@ -1,31 +1,5 @@ -menu "Metrics" - config ESP_OPENTELEMETRY_METRICS_ENABLED - bool "Enable OpenTelemetry metrics" - depends on ESP_OPENTELEMETRY_TRACING_ENABLED - default n - help - Enables the PeriodicExportingMetricReader that exports sensor and - system metrics via OTLP/HTTP. Reuses - ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT from the Tracing menu. - - config ESP_OPENTELEMETRY_METRICS_EXPORT_INTERVAL_MS - int "Metrics export interval (ms)" - depends on ESP_OPENTELEMETRY_METRICS_ENABLED - default 500 - help - How often the PeriodicExportingMetricReader collects and exports - metrics. - - config ESP_OPENTELEMETRY_METRICS_OTLP_BASE_URL - string "Metrics OTLP base URL" - depends on ESP_OPENTELEMETRY_METRICS_ENABLED - default "" - help - Base URL for OTLP/HTTP metrics export. The path "/v1/metrics" is - appended automatically. Leave empty to reuse - ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT (the tracing endpoint). - - config ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +menu "System metrics" + config SYSTEM_METRICS_TASK_STATS_ENABLED bool "Enable per-task CPU usage / priority metrics (debug only)" depends on ESP_OPENTELEMETRY_METRICS_ENABLED default n @@ -38,7 +12,7 @@ menu "Metrics" (issue #43) - leave disabled unless actively debugging task scheduling/CPU usage, and disable again afterward. - config ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED + config SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED bool "Enable largest free heap block metric (debug only)" depends on ESP_OPENTELEMETRY_METRICS_ENABLED default n diff --git a/car/components/tracing/include/metrics.hpp b/car/components/tracing/include/metrics.hpp deleted file mode 100644 index f5085b9..0000000 --- a/car/components/tracing/include/metrics.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include - -#include "opentelemetry/metrics/observer_result.h" - -void observe_double(opentelemetry::metrics::ObserverResult& obs, double value); -void observe_int64(opentelemetry::metrics::ObserverResult& obs, int64_t value); -void metrics_setup(); diff --git a/car/components/tracing/include/tracing.hpp b/car/components/tracing/include/tracing.hpp index 75c2e09..94c91a3 100644 --- a/car/components/tracing/include/tracing.hpp +++ b/car/components/tracing/include/tracing.hpp @@ -1,13 +1,7 @@ #pragma once -#include "esp_opentelemetry.hpp" #include "opentelemetry/context/context.h" -#include "opentelemetry/trace/scope.h" -#include "opentelemetry/trace/span_startoptions.h" #include -void tracing_setup(); void tracing_inject(cJSON& obj); opentelemetry::context::Context tracing_extract(const cJSON& obj); - -void metrics_setup(); diff --git a/car/components/tracing/metrics.cpp b/car/components/tracing/metrics.cpp deleted file mode 100644 index 2cfc7bd..0000000 --- a/car/components/tracing/metrics.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "tracing.hpp" -#include "metrics.hpp" -#include "esp_pthread.h" -#include "esp_heap_caps.h" -#include "sdkconfig.h" - -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED - -#include "esp_http_client_transport.hpp" -#include "opentelemetry/nostd/shared_ptr.h" -#include "opentelemetry/nostd/variant.h" -#include "opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h" -#include "opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h" -#include "opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h" -#include "opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_options.h" -#include "opentelemetry/sdk/metrics/meter_context_factory.h" -#include "opentelemetry/sdk/metrics/meter_provider_factory.h" -#include "opentelemetry/sdk/metrics/provider.h" -#include "opentelemetry/sdk/metrics/view/view_registry_factory.h" -#include "opentelemetry/sdk/resource/resource.h" -#include -#include - -#endif // CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED - -void observe_double(opentelemetry::metrics::ObserverResult& obs, double value) { -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED - opentelemetry::nostd::get< - opentelemetry::nostd::shared_ptr>>(obs) - ->Observe(value); -#endif -} - -void observe_int64(opentelemetry::metrics::ObserverResult& obs, int64_t value) { -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED - opentelemetry::nostd::get< - opentelemetry::nostd::shared_ptr>>(obs) - ->Observe(value); -#endif -} - -void metrics_setup() { -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED - const char* metrics_base = CONFIG_ESP_OPENTELEMETRY_METRICS_OTLP_BASE_URL; - if (*metrics_base == '\0') metrics_base = CONFIG_ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT; - std::string url = std::string(metrics_base) + "/v1/metrics"; - - opentelemetry::exporter::otlp::OtlpHttpMetricExporterOptions opts; - opts.url = url; - auto exporter = opentelemetry::exporter::otlp::OtlpHttpMetricExporterFactory::Create( - opts, esp_opentelemetry::MakeEspHttpClient()); - - opentelemetry::sdk::metrics::PeriodicExportingMetricReaderOptions reader_opts; - reader_opts.export_interval_millis = - std::chrono::milliseconds(CONFIG_ESP_OPENTELEMETRY_METRICS_EXPORT_INTERVAL_MS); - reader_opts.export_timeout_millis = - std::chrono::milliseconds(CONFIG_ESP_OPENTELEMETRY_METRICS_EXPORT_INTERVAL_MS / 2); - - // The export thread (protobuf + mbedTLS) needs a large stack — route it to PSRAM. - esp_pthread_cfg_t orig_cfg = esp_pthread_get_default_config(); - esp_pthread_cfg_t cfg = orig_cfg; - cfg.stack_size = 65536; - cfg.stack_alloc_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; - cfg.prio = 1; - esp_pthread_set_cfg(&cfg); - - auto reader = opentelemetry::sdk::metrics::PeriodicExportingMetricReaderFactory::Create( - std::move(exporter), reader_opts); - - auto resource = opentelemetry::sdk::resource::Resource::Create( - {{"service.name", CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME}}); - auto context = opentelemetry::sdk::metrics::MeterContextFactory::Create( - opentelemetry::sdk::metrics::ViewRegistryFactory::Create(), resource); - context->AddMetricReader(std::move(reader)); - - esp_pthread_set_cfg(&orig_cfg); - - auto sdk_provider = opentelemetry::sdk::metrics::MeterProviderFactory::Create(std::move(context)); - std::shared_ptr api_provider = std::move(sdk_provider); - opentelemetry::sdk::metrics::Provider::SetMeterProvider(api_provider); -#endif // CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED -} diff --git a/car/components/tracing/system_metrics.cpp b/car/components/tracing/system_metrics.cpp index 2650c98..7e4262f 100644 --- a/car/components/tracing/system_metrics.cpp +++ b/car/components/tracing/system_metrics.cpp @@ -1,5 +1,5 @@ #include "system_metrics.hpp" -#include "metrics.hpp" +#include "esp_metrics.hpp" #include "sdkconfig.h" #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED @@ -24,7 +24,7 @@ namespace { static temperature_sensor_handle_t s_temp_sensor = NULL; -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED static constexpr UBaseType_t kMaxTasks = 48; struct TaskStat { @@ -96,7 +96,7 @@ static void refresh_task_stats() { s_task_count = n; s_prev_sample_time = now; } -#endif // CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#endif // CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED static void cb_free_heap(opentelemetry::metrics::ObserverResult obs, void*) { observe_int64(obs, static_cast(esp_get_free_heap_size())); @@ -104,11 +104,11 @@ static void cb_free_heap(opentelemetry::metrics::ObserverResult obs, void*) { static void cb_min_free_heap(opentelemetry::metrics::ObserverResult obs, void*) { observe_int64(obs, static_cast(esp_get_minimum_free_heap_size())); } -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED static void cb_largest_free_block(opentelemetry::metrics::ObserverResult obs, void*) { observe_int64(obs, static_cast(heap_caps_get_largest_free_block(MALLOC_CAP_8BIT))); } -#endif // CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED +#endif // CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED static void cb_internal_free_heap(opentelemetry::metrics::ObserverResult obs, void*) { observe_int64(obs, static_cast(esp_get_free_internal_heap_size())); } @@ -123,7 +123,7 @@ static void cb_temperature(opentelemetry::metrics::ObserverResult obs, void*) { if (s_temp_sensor) temperature_sensor_get_celsius(s_temp_sensor, &temp); observe_double(obs, static_cast(temp)); } -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED static void cb_task_cpu_usage(opentelemetry::metrics::ObserverResult obs, void*) { refresh_task_stats(); for (UBaseType_t i = 0; i < s_task_count; i++) @@ -136,18 +136,18 @@ static void cb_task_priority(opentelemetry::metrics::ObserverResult obs, void*) observe_task_metric(obs, static_cast(s_task_stats[i].priority), s_task_stats[i].name, s_task_stats[i].core); } -#endif // CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#endif // CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED // Base instruments (free heap, min free heap, internal free heap, free psram, // uptime, temperature) plus whichever of the two debug-only, opt-in groups // below are enabled. static constexpr size_t kBaseInstruments = 6; -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED static constexpr size_t kLargestFreeBlockInstruments = 1; #else static constexpr size_t kLargestFreeBlockInstruments = 0; #endif -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED static constexpr size_t kTaskStatsInstruments = 2; #else static constexpr size_t kTaskStatsInstruments = 0; @@ -181,7 +181,7 @@ void system_metrics_setup() { s_instruments[idx]->AddCallback(cb_min_free_heap, nullptr); idx++; -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED s_instruments[idx] = meter->CreateInt64ObservableGauge( "dust_mite.largest_free_block_bytes", "Largest contiguous free heap block", "By"); s_instruments[idx]->AddCallback(cb_largest_free_block, nullptr); @@ -208,7 +208,7 @@ void system_metrics_setup() { s_instruments[idx]->AddCallback(cb_temperature, nullptr); idx++; -#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED +#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED s_instruments[idx] = meter->CreateDoubleObservableGauge("dust_mite.task_cpu_usage", "Per-task CPU usage", "%"); s_instruments[idx]->AddCallback(cb_task_cpu_usage, nullptr); diff --git a/car/components/tracing/tracing.cpp b/car/components/tracing/tracing.cpp index 0116dad..cdaa1a9 100644 --- a/car/components/tracing/tracing.cpp +++ b/car/components/tracing/tracing.cpp @@ -1,7 +1,4 @@ #include "tracing.hpp" -#include "esp_pthread.h" -#include "esp_heap_caps.h" -#include "sdkconfig.h" #include "opentelemetry/context/runtime_context.h" #include "opentelemetry/context/propagation/global_propagator.h" @@ -39,25 +36,6 @@ class CJsonCarrier : public opentelemetry::context::propagation::TextMapCarrier } // namespace -void tracing_setup() { -#ifdef CONFIG_ESP_OPENTELEMETRY_TRACING_ENABLED - // Route BatchSpanProcessor pthread stack to PSRAM and increase its size. - // The export call chain (DoBackgroundWork → OtlpHttpExporter::Export → - // protobuf arena → mbedTLS) is ~15 frames deep and overflows 32 KB. - esp_pthread_cfg_t cfg = esp_pthread_get_default_config(); - cfg.stack_size = 65536; - cfg.stack_alloc_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT; - esp_pthread_set_cfg(&cfg); -#endif - - esp_opentelemetry_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME); - -#ifdef CONFIG_ESP_OPENTELEMETRY_TRACING_ENABLED - esp_pthread_cfg_t default_cfg = esp_pthread_get_default_config(); - esp_pthread_set_cfg(&default_cfg); -#endif -} - void tracing_inject(cJSON& obj) { auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); diff --git a/car/components/web_server/test_apps/main/main.cpp b/car/components/web_server/test_apps/main/main.cpp index 94cb4a6..6ede145 100644 --- a/car/components/web_server/test_apps/main/main.cpp +++ b/car/components/web_server/test_apps/main/main.cpp @@ -2,7 +2,7 @@ #include "freertos/queue.h" #include "web_server.hpp" #include "motor.hpp" -#include "tracing.hpp" +#include "esp_opentelemetry.hpp" #include "wifi.hpp" #include "telemetry.hpp" #include "unity.h" @@ -20,7 +20,7 @@ extern "C" void app_main(void) { #ifndef CONFIG_WEB_SERVER_TEST_QEMU_MODE motor_setup(command_queue); - tracing_setup(); + esp_opentelemetry_tracing_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME); wifi_setup(); web_server_setup(frame_queue, command_queue, telemetry_queue); #endif diff --git a/car/components/web_server/web_server.cpp b/car/components/web_server/web_server.cpp index 36fc5a7..a38f3ef 100644 --- a/car/components/web_server/web_server.cpp +++ b/car/components/web_server/web_server.cpp @@ -16,6 +16,9 @@ #include "camera.hpp" #include "telemetry.hpp" #include "tracing.hpp" +#include "esp_opentelemetry.hpp" +#include "opentelemetry/trace/scope.h" +#include "opentelemetry/trace/span_startoptions.h" #include "web_server_metrics.hpp" #include #include "mbedtls/base64.h" diff --git a/car/dependencies.lock b/car/dependencies.lock index 6511cfb..9aff759 100644 --- a/car/dependencies.lock +++ b/car/dependencies.lock @@ -41,6 +41,6 @@ direct_dependencies: - espressif/cjson - espressif/esp32-camera - idf -manifest_hash: 911421cdce94da1ed2bc6de5b3e378be9793a680401c558085611d3a71cad054 +manifest_hash: 3923910e2b6c0306067fc2b512ca70397ed38331405c74733d0a268eb626c3e7 target: esp32s3 version: 3.0.0 diff --git a/car/main/main.cpp b/car/main/main.cpp index 2b5ba3d..982e00e 100644 --- a/car/main/main.cpp +++ b/car/main/main.cpp @@ -12,9 +12,9 @@ #include "motor.hpp" #include "telemetry.hpp" #include "telemetry_metrics.hpp" -#include "tracing.hpp" #include "system_metrics.hpp" #include "wifi.hpp" +#include "esp_opentelemetry.hpp" #include "sdkconfig.h" static i2c_master_bus_handle_t i2c_bus_init() { @@ -41,14 +41,17 @@ extern "C" void app_main() { wifi_wait_for_ip(); sync_time(); + // Must run before web_server_setup() can activate a span: profiling's RuntimeContextStorage swap + // and tracing's provider both must be installed before the first span is created. + esp_opentelemetry_tracing_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME); + esp_opentelemetry_metrics_setup(); + esp_opentelemetry_profiling_setup(); + motor_setup(command_queue); camera_setup(frame_queue, i2c_bus); telemetry_setup(telemetry_queue, i2c_bus); web_server_setup(frame_queue, command_queue, telemetry_queue); - tracing_setup(); - - metrics_setup(); system_metrics_setup(); telemetry_metrics_setup(); camera_metrics_setup(); diff --git a/car/sdkconfig.defaults b/car/sdkconfig.defaults index a6c0963..4834e2d 100644 --- a/car/sdkconfig.defaults +++ b/car/sdkconfig.defaults @@ -39,7 +39,7 @@ CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" CONFIG_ESP_OPENTELEMETRY_TRACING_ENABLED=y CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED=y -CONFIG_ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT="http://192.168.50.222:4318" +CONFIG_ESP_OPENTELEMETRY_TRACING_OTLP_BASE_URL="http://192.168.50.222:4318" CONFIG_ESP_OPENTELEMETRY_METRICS_OTLP_BASE_URL="http://192.168.50.222:8428/opentelemetry" CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME="dust-mite-car" diff --git a/car/sdkconfig.defaults.all b/car/sdkconfig.defaults.all index aed81a0..0927164 100644 --- a/car/sdkconfig.defaults.all +++ b/car/sdkconfig.defaults.all @@ -2,5 +2,5 @@ # conditional-compilation bit rot in #ifdef branches not exercised by # car/sdkconfig.defaults, and usable standalone for hardware debugging (e.g. # issue #43's task-stats/heap metrics) without hand-editing sdkconfig.defaults. -CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED=y -CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED=y +CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED=y +CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED=y diff --git a/car/sdkconfig.defaults.profiling b/car/sdkconfig.defaults.profiling new file mode 100644 index 0000000..5eefbcd --- /dev/null +++ b/car/sdkconfig.defaults.profiling @@ -0,0 +1,14 @@ +# Statistical CPU profiler overlay (see CONTRIBUTING.md "Profiling"). Build with: +# SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.defaults.profiling" idf.py build +# Opt-in debug tooling: keeps its costs (sampling ISRs, per-Scope span-slot +# updates, the pyroscope.profile.id span attribute) out of normal builds. +CONFIG_ESP_OPENTELEMETRY_PROFILING_ENABLED=y + +# Profiles endpoint = the profiling-symbolizer on the host LAN, which resolves +# Xtensa addresses against the published ELF and forwards to the collector -> +# Pyroscope. +CONFIG_ESP_OPENTELEMETRY_PROFILES_OTLP_BASE_URL="http://192.168.50.222:4319" + +# TLS pointer index 1 hosts the per-task active-span slot that links profile +# samples to the span running on the interrupted task (index 0 = pthread). +CONFIG_FREERTOS_THREAD_LOCAL_STORAGE_POINTERS=2 diff --git a/car/test_apps/integration/main/main.cpp b/car/test_apps/integration/main/main.cpp index ff3fabb..7361b94 100644 --- a/car/test_apps/integration/main/main.cpp +++ b/car/test_apps/integration/main/main.cpp @@ -7,8 +7,9 @@ #include "web_server.hpp" #include "motor.hpp" #include "telemetry.hpp" -#include "tracing.hpp" #include "wifi.hpp" +#include "esp_opentelemetry.hpp" +#include "sdkconfig.h" static const char* TAG = "integration_test"; @@ -36,7 +37,7 @@ extern "C" void app_main() { wifi_setup(); wifi_wait_for_ip(); sync_time(); - tracing_setup(); + esp_opentelemetry_tracing_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME); camera_setup(frame_queue, i2c_bus); telemetry_setup(telemetry_queue, i2c_bus); web_server_setup(frame_queue, command_queue, telemetry_queue); diff --git a/docker-compose.yml b/docker-compose.yml index 24e06d5..8151f0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,16 @@ services: file: observability/docker-compose.yml service: grafana + pyroscope: + extends: + file: observability/docker-compose.yml + service: pyroscope + + profiling-symbolizer: + extends: + file: observability/docker-compose.yml + service: profiling-symbolizer + test-runner: build: context: . diff --git a/observability/docker-compose.yml b/observability/docker-compose.yml index 90e3fa9..efc91cb 100644 --- a/observability/docker-compose.yml +++ b/observability/docker-compose.yml @@ -4,7 +4,10 @@ networks: services: otel-collector: - image: otel/opentelemetry-collector-contrib:0.127.0 + image: otel/opentelemetry-collector-contrib:0.146.0 + command: + - "--config=/etc/otelcol-contrib/config.yaml" + - "--feature-gates=service.profilesSupport" ports: - "4317:4317" # OTLP gRPC - "4318:4318" # OTLP HTTP @@ -12,6 +15,34 @@ services: - ./otelcol.yml:/etc/otelcol-contrib/config.yaml:ro depends_on: - tempo + - pyroscope + networks: + - dust-mite-observability + + pyroscope: + image: grafana/pyroscope:1.18.1 + command: + - "-config.file=/etc/pyroscope/config.yml" + - "-self-profiling.disable-push=true" + volumes: + - ./pyroscope.yml:/etc/pyroscope/config.yml:ro + ports: + - "4040:4040" # HTTP API + OTLP profiles ingestion + networks: + - dust-mite-observability + + profiling-symbolizer: + build: + context: ../car/components/esp-opentelemetry-cpp/tools/symbolizer + ports: + - "4319:4319" # OTLP/HTTP JSON profiles in (device -> symbolizer) + environment: + SYMBOLIZER_LISTEN_ADDR: "0.0.0.0:4319" + SYMBOLIZER_FORWARD_URL: "http://otel-collector:4318/v1development/profiles" + volumes: + - ../car:/firmware:ro + depends_on: + - otel-collector networks: - dust-mite-observability diff --git a/observability/grafana/provisioning/datasources/pyroscope.yml b/observability/grafana/provisioning/datasources/pyroscope.yml new file mode 100644 index 0000000..f825ccc --- /dev/null +++ b/observability/grafana/provisioning/datasources/pyroscope.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Pyroscope + uid: pyroscope + type: grafana-pyroscope-datasource + access: proxy + url: http://pyroscope:4040 + isDefault: false + editable: true diff --git a/observability/grafana/provisioning/datasources/tempo.yml b/observability/grafana/provisioning/datasources/tempo.yml index 63214af..8a7a6f3 100644 --- a/observability/grafana/provisioning/datasources/tempo.yml +++ b/observability/grafana/provisioning/datasources/tempo.yml @@ -8,3 +8,11 @@ datasources: url: http://tempo:3200 isDefault: true editable: true + jsonData: + tracesToProfilesV2: + datasourceUid: pyroscope + profileTypeId: "samples:samples:count:samples:count" + customQuery: false + tags: + - key: "service.name" + value: "service_name" diff --git a/observability/otelcol.yml b/observability/otelcol.yml index 408cfeb..08b861a 100644 --- a/observability/otelcol.yml +++ b/observability/otelcol.yml @@ -10,22 +10,32 @@ receivers: - "*" exporters: - otlp/tempo: + otlp_grpc/tempo: endpoint: tempo:4317 tls: insecure: true - otlphttp/victoriametrics: + otlp_http/victoriametrics: endpoint: http://victoriametrics:8428/opentelemetry tls: insecure: true + otlp_http/pyroscope: + endpoint: http://pyroscope:4040 + compression: none + tls: + insecure: true + service: pipelines: traces: receivers: [otlp] - exporters: [otlp/tempo] + exporters: [otlp_grpc/tempo] metrics: receivers: [otlp] - exporters: [otlphttp/victoriametrics] + exporters: [otlp_http/victoriametrics] + + profiles: + receivers: [otlp] + exporters: [otlp_http/pyroscope] diff --git a/observability/pyroscope.yml b/observability/pyroscope.yml new file mode 100644 index 0000000..5101496 --- /dev/null +++ b/observability/pyroscope.yml @@ -0,0 +1,11 @@ +# Pyroscope derives the service_name label from the OTLP service.name resource +# attribute, but ALSO copies it verbatim as a "service.name" label once any +# extra attribute (e.g. thread_name for per-task split) is present. A dotted +# label name is an invalid Prometheus metric name and gets the whole series +# rejected. Drop exactly that duplicate; the normalized service_name and the +# dotless attributes (thread_name) are kept. (All attribute keys we emit other +# than service.name are already dotless, so a narrow rule is sufficient.) +limits: + ingestion_relabeling_rules: + - action: labeldrop + regex: "service\\.name"