Add esp-idf profiling

AGENTS.md

@@ -35,7 +35,7 @@ This file is a lightweight navigation guide for coding agents.
 Run `./scripts/start_headless.sh` to build and start all services in the background.
 Run `./scripts/stop_headless.sh` to stop and remove all containers.
 
-Docker Compose services: `cpp`, `python`, `js`, `otel-collector`, `tempo`, `grafana`. Use standard docker compose commands to manage them:
+Docker Compose services: `cpp`, `python`, `js`, `otel-collector`, `tempo`, `victoriametrics`, `pyroscope`, `profiling-symbolizer`, `grafana`. Use standard docker compose commands to manage them:
 
 - Logs: `docker compose -f docker-compose.yml -f docker-compose.headless.yml logs [-f] <service>`
 - Restart: `docker compose -f docker-compose.yml -f docker-compose.headless.yml restart <service>`

CONTRIBUTING.md

@@ -155,6 +155,8 @@ All three services export traces via OTLP/HTTP to the OTel Collector and metrics
 | Python streamer | `dust-mite-streamer` | `OTEL_EXPORTER_OTLP_ENDPOINT` (default `http://otel-collector:4318`) | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` (default `http://victoriametrics:8428/opentelemetry/v1/metrics`) |
 | Web browser | `dust-mite-web` | `VITE_OTLP_ENDPOINT` (default `http://localhost:4318`) | `VITE_OTLP_METRICS_ENDPOINT` (default `http://localhost:4318`) via OTel Collector |
 
+The ESP32 cannot resolve the observability stack's Docker DNS names (e.g. `otel-collector`), so its endpoints must be the host machine's LAN IP rather than a container service name.
+
 ### Metrics
 
 The set of emitted metrics is variant-specific. See the variant documentation in [docs/variants/](docs/variants/) for the full list. When adding or removing a metric, update the metrics table in the relevant variant document.
@@ -242,6 +244,53 @@ Open Grafana at `http://localhost:3000` → **Explore** → select the **Tempo**
 - `{resource.service.name="dust-mite-car"}` — firmware spans only
 - `{resource.service.name="dust-mite-streamer"}` — streamer spans only
 
+### Profiling (firmware CPU)
+
+Spans and metrics tell you *which component* is slow; the statistical profiler tells you
+*which code* is burning CPU on the ESP32-S3, so you can decide between optimising the code,
+changing FreeRTOS scheduling (core/priority), or accepting a hardware limit. It is a sampling
+profiler exported as the OpenTelemetry **profiles** signal and viewed as Grafana **Pyroscope**
+flame graphs.
+
+For how the sampler, exporter, and symbolizer work internally, see the
+[esp-opentelemetry-cpp README](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations).
+
+To profile a build:
+
+1. Bring up the stack (`./scripts/start_headless.sh`) — it includes `pyroscope` and
+   `profiling-symbolizer`.
+2. Build/flash the firmware with the profiling overlay (enables the sampler, the
+   span→profile task slot, and the symbolizer endpoint; see
+   [car/sdkconfig.defaults.profiling](car/sdkconfig.defaults.profiling)):
+   `SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.defaults.profiling" idf.py build flash`
+   Symbols need no publishing step: the symbolizer mounts `car/` and finds the
+   matching ELF by hash (the profile `build_id` equals the ELF's sha256).
+3. Open Grafana → **Explore** → **Pyroscope** datasource. Filter by
+   `{service_name="dust-mite-car"}`, optionally split by `thread_name` (FreeRTOS task). Idle
+   task dominance means the workload is not CPU-bound (hardware/scheduling limited); a specific
+   application or esp-idf frame dominating means there is code to optimise.
+
+QEMU is not used for profiling — its Xtensa core is not cycle-accurate, so durations would be
+fictional. The sampler/span-link/exporter mechanics are covered by
+[esp-opentelemetry-cpp](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations)'s
+own CI (QEMU); dust-mite's CI does not build the profiling overlay at all, so it is validated
+live on real hardware only.
+
+#### Span profiles (trace → flame graph)
+
+Profiles are linked to traces the same way Go/Python do it; see `esp_task_span_slot.cpp` in the
+[esp-opentelemetry-cpp README](https://github.com/ltowarek/esp-opentelemetry-cpp/blob/main/README.md#esp-specific-integrations)
+for the mechanism. No changes are needed at span call sites.
+
+In a Tempo trace view, spans carry a **Profiles for this span** link/flame-graph tab
+(provisioned via `tracesToProfilesV2` on the Tempo datasource) showing the CPU sampled while
+that exact span was active. Spans are also stamped with the `pyroscope.profile.id` attribute
+Grafana uses to enable the embedded flame graph.
+
+Expectations: at 100 Hz/core a 5 ms span catches at most one sample — per-span flame graphs
+are meaningful for spans of tens of milliseconds or when aggregated across many span
+instances. Raise `CONFIG_ESP_OPENTELEMETRY_PROFILING_SAMPLE_HZ` for a session if finer coverage is needed.
+
 ## Variants
 
 Variants use code names based on chemical elements (for example `Copper`, `Iron`).

car/components/camera/camera_metrics.cpp

@@ -3,7 +3,7 @@
 
 #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED
 #include <cstdint>
-#include "metrics.hpp"
+#include "esp_metrics.hpp"
 #include "opentelemetry/metrics/async_instruments.h"
 #include "opentelemetry/metrics/observer_result.h"
 #include "opentelemetry/metrics/provider.h"

car/components/telemetry/telemetry_metrics.cpp

@@ -1,5 +1,5 @@
 #include "telemetry_metrics.hpp"
-#include "metrics.hpp"
+#include "esp_metrics.hpp"
 #include "sdkconfig.h"
 
 #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED

car/components/tracing/CMakeLists.txt

@@ -1,9 +1,8 @@
-idf_component_register(SRCS "system_metrics.cpp" "tracing.cpp" "metrics.cpp"
+idf_component_register(SRCS "system_metrics.cpp" "tracing.cpp"
                     INCLUDE_DIRS "include"
                     REQUIRES
                     esp-opentelemetry-cpp
                     cjson
-                    pthread
                     heap
                     esp_timer
                     PRIV_REQUIRES

car/components/tracing/Kconfig.projbuild

@@ -1,31 +1,5 @@
-menu "Metrics"
-    config ESP_OPENTELEMETRY_METRICS_ENABLED
-        bool "Enable OpenTelemetry metrics"
-        depends on ESP_OPENTELEMETRY_TRACING_ENABLED
-        default n
-        help
-            Enables the PeriodicExportingMetricReader that exports sensor and
-            system metrics via OTLP/HTTP. Reuses
-            ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT from the Tracing menu.
-
-    config ESP_OPENTELEMETRY_METRICS_EXPORT_INTERVAL_MS
-        int "Metrics export interval (ms)"
-        depends on ESP_OPENTELEMETRY_METRICS_ENABLED
-        default 500
-        help
-            How often the PeriodicExportingMetricReader collects and exports
-            metrics.
-
-    config ESP_OPENTELEMETRY_METRICS_OTLP_BASE_URL
-        string "Metrics OTLP base URL"
-        depends on ESP_OPENTELEMETRY_METRICS_ENABLED
-        default ""
-        help
-            Base URL for OTLP/HTTP metrics export. The path "/v1/metrics" is
-            appended automatically. Leave empty to reuse
-            ESP_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT (the tracing endpoint).
-
-    config ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+menu "System metrics"
+    config SYSTEM_METRICS_TASK_STATS_ENABLED
         bool "Enable per-task CPU usage / priority metrics (debug only)"
         depends on ESP_OPENTELEMETRY_METRICS_ENABLED
         default n
@@ -38,7 +12,7 @@ menu "Metrics"
             (issue #43) - leave disabled unless actively debugging task
             scheduling/CPU usage, and disable again afterward.
 
-    config ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED
+    config SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED
         bool "Enable largest free heap block metric (debug only)"
         depends on ESP_OPENTELEMETRY_METRICS_ENABLED
         default n

car/components/tracing/include/tracing.hpp

@@ -1,13 +1,7 @@
 #pragma once
 
-#include "esp_opentelemetry.hpp"
 #include "opentelemetry/context/context.h"
-#include "opentelemetry/trace/scope.h"
-#include "opentelemetry/trace/span_startoptions.h"
 #include <cJSON.h>
 
-void tracing_setup();
 void tracing_inject(cJSON& obj);
 opentelemetry::context::Context tracing_extract(const cJSON& obj);
-
-void metrics_setup();

car/components/tracing/system_metrics.cpp

@@ -1,5 +1,5 @@
 #include "system_metrics.hpp"
-#include "metrics.hpp"
+#include "esp_metrics.hpp"
 #include "sdkconfig.h"
 
 #ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_ENABLED
@@ -24,7 +24,7 @@ namespace {
 
 static temperature_sensor_handle_t s_temp_sensor = NULL;
 
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
 static constexpr UBaseType_t kMaxTasks = 48;
 
 struct TaskStat {
@@ -96,19 +96,19 @@ static void refresh_task_stats() {
   s_task_count = n;
   s_prev_sample_time = now;
 }
-#endif  // CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#endif  // CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
 
 static void cb_free_heap(opentelemetry::metrics::ObserverResult obs, void*) {
   observe_int64(obs, static_cast<int64_t>(esp_get_free_heap_size()));
 }
 static void cb_min_free_heap(opentelemetry::metrics::ObserverResult obs, void*) {
   observe_int64(obs, static_cast<int64_t>(esp_get_minimum_free_heap_size()));
 }
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED
 static void cb_largest_free_block(opentelemetry::metrics::ObserverResult obs, void*) {
   observe_int64(obs, static_cast<int64_t>(heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)));
 }
-#endif  // CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED
+#endif  // CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED
 static void cb_internal_free_heap(opentelemetry::metrics::ObserverResult obs, void*) {
   observe_int64(obs, static_cast<int64_t>(esp_get_free_internal_heap_size()));
 }
@@ -123,7 +123,7 @@ static void cb_temperature(opentelemetry::metrics::ObserverResult obs, void*) {
   if (s_temp_sensor) temperature_sensor_get_celsius(s_temp_sensor, &temp);
   observe_double(obs, static_cast<double>(temp));
 }
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
 static void cb_task_cpu_usage(opentelemetry::metrics::ObserverResult obs, void*) {
   refresh_task_stats();
   for (UBaseType_t i = 0; i < s_task_count; i++)
@@ -136,18 +136,18 @@ static void cb_task_priority(opentelemetry::metrics::ObserverResult obs, void*)
     observe_task_metric(obs, static_cast<int64_t>(s_task_stats[i].priority), s_task_stats[i].name,
                         s_task_stats[i].core);
 }
-#endif  // CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#endif  // CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
 
 // Base instruments (free heap, min free heap, internal free heap, free psram,
 // uptime, temperature) plus whichever of the two debug-only, opt-in groups
 // below are enabled.
 static constexpr size_t kBaseInstruments = 6;
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED
 static constexpr size_t kLargestFreeBlockInstruments = 1;
 #else
 static constexpr size_t kLargestFreeBlockInstruments = 0;
 #endif
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
 static constexpr size_t kTaskStatsInstruments = 2;
 #else
 static constexpr size_t kTaskStatsInstruments = 0;
@@ -181,7 +181,7 @@ void system_metrics_setup() {
   s_instruments[idx]->AddCallback(cb_min_free_heap, nullptr);
   idx++;
 
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_LARGEST_FREE_BLOCK_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_LARGEST_FREE_BLOCK_ENABLED
   s_instruments[idx] = meter->CreateInt64ObservableGauge(
       "dust_mite.largest_free_block_bytes", "Largest contiguous free heap block", "By");
   s_instruments[idx]->AddCallback(cb_largest_free_block, nullptr);
@@ -208,7 +208,7 @@ void system_metrics_setup() {
   s_instruments[idx]->AddCallback(cb_temperature, nullptr);
   idx++;
 
-#ifdef CONFIG_ESP_OPENTELEMETRY_METRICS_TASK_STATS_ENABLED
+#ifdef CONFIG_SYSTEM_METRICS_TASK_STATS_ENABLED
   s_instruments[idx] =
       meter->CreateDoubleObservableGauge("dust_mite.task_cpu_usage", "Per-task CPU usage", "%");
   s_instruments[idx]->AddCallback(cb_task_cpu_usage, nullptr);

car/components/tracing/tracing.cpp

@@ -1,7 +1,4 @@
 #include "tracing.hpp"
-#include "esp_pthread.h"
-#include "esp_heap_caps.h"
-#include "sdkconfig.h"
 
 #include "opentelemetry/context/runtime_context.h"
 #include "opentelemetry/context/propagation/global_propagator.h"
@@ -39,25 +36,6 @@ class CJsonCarrier : public opentelemetry::context::propagation::TextMapCarrier
 
 }  // namespace
 
-void tracing_setup() {
-#ifdef CONFIG_ESP_OPENTELEMETRY_TRACING_ENABLED
-  // Route BatchSpanProcessor pthread stack to PSRAM and increase its size.
-  // The export call chain (DoBackgroundWork → OtlpHttpExporter::Export →
-  // protobuf arena → mbedTLS) is ~15 frames deep and overflows 32 KB.
-  esp_pthread_cfg_t cfg = esp_pthread_get_default_config();
-  cfg.stack_size = 65536;
-  cfg.stack_alloc_caps = MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT;
-  esp_pthread_set_cfg(&cfg);
-#endif
-
-  esp_opentelemetry_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME);
-
-#ifdef CONFIG_ESP_OPENTELEMETRY_TRACING_ENABLED
-  esp_pthread_cfg_t default_cfg = esp_pthread_get_default_config();
-  esp_pthread_set_cfg(&default_cfg);
-#endif
-}
-
 void tracing_inject(cJSON& obj) {
   auto propagator =
       opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator();

car/components/web_server/test_apps/main/main.cpp

@@ -2,7 +2,7 @@
 #include "freertos/queue.h"
 #include "web_server.hpp"
 #include "motor.hpp"
-#include "tracing.hpp"
+#include "esp_opentelemetry.hpp"
 #include "wifi.hpp"
 #include "telemetry.hpp"
 #include "unity.h"
@@ -20,7 +20,7 @@ extern "C" void app_main(void) {
 
 #ifndef CONFIG_WEB_SERVER_TEST_QEMU_MODE
   motor_setup(command_queue);
-  tracing_setup();
+  esp_opentelemetry_tracing_setup(CONFIG_ESP_OPENTELEMETRY_SERVICE_NAME);
   wifi_setup();
   web_server_setup(frame_queue, command_queue, telemetry_queue);
 #endif