From 8d22978ce06466234a870263c620fbc403e0b503 Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:46:38 +0530 Subject: [PATCH 1/6] =?UTF-8?q?feat(sdk):=20rocketride.otelbridge=20?= =?UTF-8?q?=E2=80=94=20OpenTelemetry=20export=20over=20the=20documented=20?= =?UTF-8?q?ingester=20protocol=20(#1609)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maps the engine's monitor event stream to standard OTel telemetry with zero engine/server changes — the client-side refinement invited by tracking issue #616 after the engine-invasive #498 was archived. - mapper.py: stateful FlowSpanMapper reconstructs span trees from apaevt_flow — task root span, pipe span per pipe id, component child spans paired by component identity (per the observability doc's warning against stack-position pairing); trace.error -> span status ERROR + exception event; apaevt_sse -> span events; unclosed spans at end/shutdown flagged rocketride.span.unclosed=true; per-pipe state is bounded. MetricsMapper maps apaevt_status_update snapshots to OTel instruments (object counts, rates, cpu/gpu memory + peaks). - Semantic conventions per the July 2026 semantic-conventions-genai snapshot: gen_ai.provider.name (not the deprecated gen_ai.system), gen_ai.operation.name from the well-known list only, span names '{operation} {model}' style, INTERNAL span kind for in-process component executions. - Privacy by default: lane payloads / prompt / result content never reach attributes or events unless include_content is set (8KB truncation cap when it is); enforced by a sentinel-scan test across all exported attributes and events. - setup.py: TracerProvider/MeterProvider + OTLP http/protobuf exporters; ALL opentelemetry imports are lazy so the base package imports without the extra. 53 unit tests (InMemorySpanExporter/InMemoryMetricReader; importorskip so the base CI matrix stays green), including a full replay of the captured real-engine event fixture. Co-Authored-By: Claude Fable 5 --- .../src/rocketride/otelbridge/__init__.py | 59 ++ .../src/rocketride/otelbridge/bridge.py | 279 +++++++ .../src/rocketride/otelbridge/config.py | 178 +++++ .../src/rocketride/otelbridge/mapper.py | 734 ++++++++++++++++++ .../src/rocketride/otelbridge/setup.py | 187 +++++ .../tests/fixtures/otel_bridge_events.json | 654 ++++++++++++++++ .../client-python/tests/test_otel_bridge.py | 391 ++++++++++ .../client-python/tests/test_otel_mapper.py | 711 +++++++++++++++++ .../client-python/tests/test_otel_setup.py | 263 +++++++ 9 files changed, 3456 insertions(+) create mode 100644 packages/client-python/src/rocketride/otelbridge/__init__.py create mode 100644 packages/client-python/src/rocketride/otelbridge/bridge.py create mode 100644 packages/client-python/src/rocketride/otelbridge/config.py create mode 100644 packages/client-python/src/rocketride/otelbridge/mapper.py create mode 100644 packages/client-python/src/rocketride/otelbridge/setup.py create mode 100644 packages/client-python/tests/fixtures/otel_bridge_events.json create mode 100644 packages/client-python/tests/test_otel_bridge.py create mode 100644 packages/client-python/tests/test_otel_mapper.py create mode 100644 packages/client-python/tests/test_otel_setup.py diff --git a/packages/client-python/src/rocketride/otelbridge/__init__.py b/packages/client-python/src/rocketride/otelbridge/__init__.py new file mode 100644 index 000000000..9b4e27328 --- /dev/null +++ b/packages/client-python/src/rocketride/otelbridge/__init__.py @@ -0,0 +1,59 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +RocketRide OpenTelemetry bridge ('rocketride otel'). + +Consumes the engine's documented WebSocket monitor protocol and exports +pipeline traces and metrics over OTLP -- with zero engine/server changes. + +All re-exports are lazy so that importing :mod:`rocketride.otelbridge` never +requires the optional 'otel' extra; the OpenTelemetry packages are only +imported when providers or mappers are actually constructed. +""" + +import importlib + +# Public name -> defining submodule (relative). Resolved lazily via __getattr__. +_EXPORTS = { + 'FlowSpanMapper': '.mapper', + 'MetricsMapper': '.mapper', + 'build_providers': '.setup', + 'OtelNotInstalledError': '.setup', + 'OtelConfig': '.config', + 'run_bridge': '.bridge', +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str): + """Lazily resolve public re-exports (PEP 562).""" + module_name = _EXPORTS.get(name) + if module_name is None: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + module = importlib.import_module(module_name, __name__) + return getattr(module, name) + + +def __dir__(): + return sorted(set(globals()) | set(_EXPORTS)) diff --git a/packages/client-python/src/rocketride/otelbridge/bridge.py b/packages/client-python/src/rocketride/otelbridge/bridge.py new file mode 100644 index 000000000..d737fd6f7 --- /dev/null +++ b/packages/client-python/src/rocketride/otelbridge/bridge.py @@ -0,0 +1,279 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Run loop for the RocketRide OpenTelemetry bridge. + +This module connects a RocketRideClient to the engine's WebSocket monitor +protocol and forwards the event stream to the OTel mappers: + + - apaevt_task / apaevt_flow / apaevt_sse -> FlowSpanMapper.handle_event + - apaevt_status_update -> MetricsMapper.handle_status + +The bridge subscribes with the wildcard monitor key ``{'token': '*'}`` (the +documented scope for an ingestion service) for the TASK, SUMMARY, FLOW and +SSE event types. Monitor subscriptions are per-connection and not durable, +but the SDK's EventMixin replays every ``add_monitor`` subscription on each +(re)connect, so the bridge subscribes exactly once and relies on the SDK +for resubscription. Reconnection after a connection loss uses capped +exponential backoff. + +Lifecycle: + - Startup connection failure -> clean stderr message, exit code 2 + - SIGINT/SIGTERM -> close all open spans, flush exporters, exit code 0 + +The event dispatcher wraps every mapper call in its own try/except because +the SDK swallows exceptions raised by user event handlers; without local +logging, mapper failures would be invisible. + +This module imports ``opentelemetry`` only lazily (via the mapper/setup +modules) so it is importable without the ``rocketride[otel]`` extra. + +Components: + run_bridge: Async entry point running the bridge until stopped +""" + +import asyncio +import signal +import sys +from typing import Any, Callable, Dict, Optional + +from .config import OtelConfig + +# Wildcard monitor subscription: all tasks visible to this API key +MONITOR_KEY: Dict[str, str] = {'token': '*'} + +# Event types the bridge consumes (case-insensitive on the server) +MONITOR_TYPES = ('TASK', 'SUMMARY', 'FLOW', 'SSE') + +# Events routed to FlowSpanMapper.handle_event +_SPAN_EVENTS = ('apaevt_task', 'apaevt_flow', 'apaevt_sse') + +# Event routed to MetricsMapper.handle_status +_STATUS_EVENT = 'apaevt_status_update' + + +def _log(message: str) -> None: + """Write a bridge diagnostic line to stderr (never stdout).""" + print(f'otel-bridge: {message}', file=sys.stderr) + + +async def _wait_for_stop(stop_event: asyncio.Event, timeout: float) -> None: + """Wait until stop_event is set or timeout elapses, whichever is first.""" + try: + await asyncio.wait_for(stop_event.wait(), timeout) + except asyncio.TimeoutError: + pass + + +async def run_bridge( + client: Any, + config: OtelConfig, + mapper_factory: Optional[Callable[[], Any]] = None, + metrics_factory: Optional[Callable[[], Any]] = None, + *, + shutdown_fn: Optional[Callable[[], None]] = None, + stop_event: Optional[asyncio.Event] = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + poll_interval: float = 0.5, +) -> int: + """ + Run the OpenTelemetry bridge until interrupted. + + Subscribes to the engine's monitor event stream and dispatches events to + the span and metrics mappers, reconnecting with capped exponential + backoff on connection loss. When no factories are provided, the OTLP + providers are built from ``config`` via + :func:`rocketride.otelbridge.setup.build_providers` (this is the only + point that imports ``opentelemetry`` and therefore requires the + ``rocketride[otel]`` extra). + + Args: + client: RocketRideClient (or compatible) instance. May already be + connected; otherwise the bridge connects it. + config: Resolved bridge configuration. + mapper_factory: Optional zero-arg factory returning a span mapper + with ``handle_event(event_name, body)`` and ``close_all()``. + Injectable for tests; defaults to FlowSpanMapper on the real + OTLP tracer. + metrics_factory: Optional zero-arg factory returning a metrics + mapper with ``handle_status(body)``. Ignored (never called) + when ``config.no_metrics`` is set; defaults to MetricsMapper + on the real OTLP meter. + shutdown_fn: Optional callable flushing/shutting down exporters at + exit. Defaults to the shutdown function returned by + ``build_providers`` when providers are built here. + stop_event: Optional externally controlled stop signal (used by + tests); created internally when absent. SIGINT/SIGTERM set it. + initial_backoff: First reconnect delay in seconds. + max_backoff: Reconnect delay cap in seconds. + poll_interval: Connection health poll interval in seconds. + + Returns: + int: 0 on graceful shutdown (signal or stop_event), 2 when the + startup connection or the initial monitor subscription fails. + + Shutdown order: + 1. Signal handlers removed (previous handlers restored) + 2. Event dispatcher detached from the client + 3. ``mapper.close_all()`` — open spans are ended + 4. ``shutdown_fn()`` — exporters flush pending telemetry + """ + # --------------------------------------------------------------------- + # Build mappers (and OTLP providers when no factories are injected) + # --------------------------------------------------------------------- + mapper = mapper_factory() if mapper_factory is not None else None + metrics = None + if not config.no_metrics and metrics_factory is not None: + metrics = metrics_factory() + + needs_tracer = mapper is None + needs_meter = metrics is None and not config.no_metrics + + if needs_tracer or needs_meter: + # Lazy imports: these modules require the 'rocketride[otel]' extra + from .mapper import FlowSpanMapper, MetricsMapper + from .setup import build_providers + + tracer, meter, provider_shutdown = build_providers(config) + if shutdown_fn is None: + shutdown_fn = provider_shutdown + if needs_tracer: + mapper = FlowSpanMapper(tracer, include_content=config.include_content) + if needs_meter: + metrics = MetricsMapper(meter) + + if stop_event is None: + stop_event = asyncio.Event() + + # --------------------------------------------------------------------- + # Attach the event dispatcher, chaining any pre-existing handler + # --------------------------------------------------------------------- + previous_handler = getattr(client, '_caller_on_event', None) + + async def _dispatch(message: Dict[str, Any]) -> None: + """Route one monitor event to the mappers, logging (not raising) failures.""" + event_name = message.get('event', '') + body = message.get('body') or {} + + # The SDK swallows handler exceptions, so log locally or lose them. + try: + if event_name == _STATUS_EVENT: + if metrics is not None: + metrics.handle_status(body) + elif event_name in _SPAN_EVENTS: + mapper.handle_event(event_name, body) + except Exception as exc: + _log(f'error handling {event_name}: {exc}') + + # Preserve any handler installed before the bridge (e.g. CLI forwarding) + if previous_handler is not None: + try: + await previous_handler(message) + except Exception as exc: + _log(f'error in chained event handler for {event_name}: {exc}') + + client._caller_on_event = _dispatch + + # --------------------------------------------------------------------- + # Signal handling: SIGINT/SIGTERM trigger a graceful stop + # --------------------------------------------------------------------- + loop = asyncio.get_running_loop() + registered_signals = [] + previous_signal_handlers: Dict[int, Any] = {} + for sig in (signal.SIGINT, signal.SIGTERM): + try: + previous_signal_handlers[sig] = signal.getsignal(sig) + loop.add_signal_handler(sig, stop_event.set) + registered_signals.append(sig) + except (NotImplementedError, RuntimeError, ValueError, OSError): + # Not supported on this platform/thread; Ctrl+C falls back to + # KeyboardInterrupt handling in the caller. + pass + + try: + # ----------------------------------------------------------------- + # Startup: connect and subscribe once. The SDK resubscribes all + # add_monitor keys automatically on every reconnect. + # ----------------------------------------------------------------- + if not client.is_connected(): + try: + await client.connect() + except Exception as exc: + _log(f'unable to connect to RocketRide server: {exc}') + return 2 + + try: + await client.add_monitor(dict(MONITOR_KEY), list(MONITOR_TYPES)) + except Exception as exc: + _log(f'monitor subscription failed: {exc}') + return 2 + + # ----------------------------------------------------------------- + # Main loop: watch connection health, reconnect with capped backoff + # ----------------------------------------------------------------- + backoff = initial_backoff + was_connected = True + while not stop_event.is_set(): + if client.is_connected(): + if not was_connected: + _log('reconnected') + was_connected = True + backoff = initial_backoff + await _wait_for_stop(stop_event, poll_interval) + else: + was_connected = False + try: + await client.connect() + except Exception as exc: + _log(f'reconnect failed, retrying in {backoff:.0f}s: {exc}') + await _wait_for_stop(stop_event, backoff) + backoff = min(backoff * 2, max_backoff) + + return 0 + + finally: + # Remove signal handlers and restore whatever was installed before + for sig in registered_signals: + try: + loop.remove_signal_handler(sig) + previous = previous_signal_handlers.get(sig) + if previous is not None: + signal.signal(sig, previous) + except (NotImplementedError, RuntimeError, ValueError, OSError): + pass + + # Detach the dispatcher, restoring any chained handler + client._caller_on_event = previous_handler + + # Close open spans first, then flush exporters + try: + mapper.close_all() + except Exception as exc: + _log(f'error closing spans: {exc}') + + if shutdown_fn is not None: + try: + shutdown_fn() + except Exception as exc: + _log(f'error during exporter shutdown: {exc}') diff --git a/packages/client-python/src/rocketride/otelbridge/config.py b/packages/client-python/src/rocketride/otelbridge/config.py new file mode 100644 index 000000000..1b6d7bba4 --- /dev/null +++ b/packages/client-python/src/rocketride/otelbridge/config.py @@ -0,0 +1,178 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Configuration for the RocketRide OpenTelemetry bridge. + +This module provides the OtelConfig dataclass consumed by the bridge run loop +(``rocketride.otelbridge.bridge``) and the OTLP provider setup +(``rocketride.otelbridge.setup``). Configuration is resolved with the +precedence: explicit CLI arguments > standard ``OTEL_*`` environment +variables > built-in defaults. + +Environment variables honored when the matching CLI argument is absent: + - OTEL_EXPORTER_OTLP_ENDPOINT: OTLP base URL (signal paths such as + ``/v1/traces`` are appended by the exporter setup, matching the + OpenTelemetry SDK's base-endpoint semantics) + - OTEL_EXPORTER_OTLP_HEADERS: comma-separated ``key=value`` pairs sent + with every OTLP export request (e.g. ``Authorization=Basic `` for + Langfuse or ``x-api-key=,Langsmith-Project=`` for LangSmith) + - OTEL_SERVICE_NAME: the ``service.name`` resource attribute + +When neither ``--endpoint`` nor ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set, the +resolved ``endpoint`` stays ``None`` and the OTLP exporters are constructed +bare, so the SDK's own environment semantics apply: the signal-specific +``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` / ``OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`` +variables are honored, falling back to the SDK defaults +(``http://localhost:4318`` for http/protobuf, ``localhost:4317`` for gRPC). +An explicit ``--endpoint`` or ``OTEL_EXPORTER_OTLP_ENDPOINT`` overrides the +signal-specific variables (CLI-arg-over-env precedence, documented here +because the OTLP spec orders the two env vars the other way around). + +This module intentionally imports nothing from ``opentelemetry`` so it is +importable without the ``rocketride[otel]`` extra installed. + +Components: + OtelConfig: Bridge configuration dataclass with from_args_env resolution + parse_headers: Parser for comma-separated ``key=value`` header strings +""" + +import os +from dataclasses import dataclass, field +from typing import Any, Dict, Mapping, Optional + +# Built-in defaults (OTLP over HTTP/protobuf; endpoint defaults are the +# OTLP exporters' own, so signal-specific OTEL_EXPORTER_OTLP_*_ENDPOINT +# environment variables stay effective when no endpoint is given here) +DEFAULT_PROTOCOL = 'http' +DEFAULT_SERVICE_NAME = 'rocketride-engine' + +# Standard OpenTelemetry environment variable names +ENV_OTLP_ENDPOINT = 'OTEL_EXPORTER_OTLP_ENDPOINT' +ENV_OTLP_HEADERS = 'OTEL_EXPORTER_OTLP_HEADERS' +ENV_SERVICE_NAME = 'OTEL_SERVICE_NAME' + + +def parse_headers(headers_str: Optional[str]) -> Dict[str, str]: + """ + Parse a comma-separated ``key=value`` header string into a dict. + + Follows the OTEL_EXPORTER_OTLP_HEADERS wire format: pairs are separated + by commas and each pair is split on the FIRST ``=`` only, so values that + themselves contain ``=`` (e.g. base64 padding in + ``Authorization=Basic cGs6c2s=``) survive intact. Whitespace around keys + and values is stripped; empty pairs and pairs without ``=`` are skipped. + Values are passed through verbatim (no URL decoding). + + Args: + headers_str: Raw header string, e.g. ``'a=1,x-api-key=abc'``. + None or empty returns an empty dict. + + Returns: + Dict[str, str]: Parsed header name/value pairs. + """ + headers: Dict[str, str] = {} + if not headers_str: + return headers + + for pair in headers_str.split(','): + pair = pair.strip() + if not pair or '=' not in pair: + continue + key, value = pair.split('=', 1) + key = key.strip() + if key: + headers[key] = value.strip() + + return headers + + +@dataclass +class OtelConfig: + """ + Configuration for the OpenTelemetry bridge. + + Attributes: + endpoint: OTLP endpoint base URL, or None to defer to the OTLP + exporters' own environment/default semantics (including the + signal-specific ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` / + ``OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`` variables). When set, + signal paths (``/v1/traces``, ``/v1/metrics``) are appended by + the exporter setup. + protocol: OTLP transport, ``'http'`` (http/protobuf, default) or + ``'grpc'`` (requires the optional grpc exporter package). + service_name: Value for the ``service.name`` resource attribute. + include_content: When True, pipeline payload content (trace data, + lane payloads, results) is included in span attributes/events, + subject to the mapper's size cap. Default False: no payload + content reaches any span. + no_metrics: When True, only traces are exported; apaevt_status_update + snapshots are not mapped to OTel metrics. + headers: Extra headers sent with every OTLP export request. + """ + + endpoint: Optional[str] = None + protocol: str = DEFAULT_PROTOCOL + service_name: str = DEFAULT_SERVICE_NAME + include_content: bool = False + no_metrics: bool = False + headers: Dict[str, str] = field(default_factory=dict) + + @classmethod + def from_args_env(cls, args: Any, env: Optional[Mapping[str, str]] = None) -> 'OtelConfig': + """ + Build an OtelConfig from parsed CLI arguments and the environment. + + Resolution precedence for each field: CLI argument (when present and + non-empty) > standard ``OTEL_*`` environment variable > default. + Missing attributes on ``args`` are treated as absent, so partial + namespaces (e.g. in tests) work. + + Args: + args: Parsed argparse namespace (or any object with optional + ``endpoint``, ``protocol``, ``service_name``, ``headers``, + ``include_content`` and ``no_metrics`` attributes). + env: Environment mapping override; defaults to ``os.environ``. + + Returns: + OtelConfig: Fully resolved bridge configuration. + """ + environ: Mapping[str, str] = os.environ if env is None else env + + # None (not a hardcoded default) when unset, so the SDK exporters' + # env semantics — incl. OTEL_EXPORTER_OTLP_TRACES/METRICS_ENDPOINT — + # and their protocol-appropriate defaults stay in effect. + endpoint = getattr(args, 'endpoint', None) or environ.get(ENV_OTLP_ENDPOINT) or None + protocol = getattr(args, 'protocol', None) or DEFAULT_PROTOCOL + service_name = getattr(args, 'service_name', None) or environ.get(ENV_SERVICE_NAME) or DEFAULT_SERVICE_NAME + + headers_arg = getattr(args, 'headers', None) + headers = parse_headers(headers_arg if headers_arg else environ.get(ENV_OTLP_HEADERS)) + + return cls( + endpoint=endpoint, + protocol=protocol, + service_name=service_name, + include_content=bool(getattr(args, 'include_content', False)), + no_metrics=bool(getattr(args, 'no_metrics', False)), + headers=headers, + ) diff --git a/packages/client-python/src/rocketride/otelbridge/mapper.py b/packages/client-python/src/rocketride/otelbridge/mapper.py new file mode 100644 index 000000000..3e31c17af --- /dev/null +++ b/packages/client-python/src/rocketride/otelbridge/mapper.py @@ -0,0 +1,734 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +RocketRide monitor-event to OpenTelemetry mappers. + +This module translates the engine's WebSocket monitor events (``apaevt_flow``, +``apaevt_task``, ``apaevt_sse``, ``apaevt_status_update``) into OpenTelemetry +spans and metrics. It is a pure consumer of the OpenTelemetry API: no network +I/O and no asyncio live here, so the mappers can be unit-tested with the SDK's +in-memory exporters. + +Span hierarchy (documented contract): + - ``apaevt_task`` ``begin`` (or a seeded ``running`` snapshot) opens one + task-level root span per run. Runs are keyed primarily by the wire + correlation id ``body['__id']`` (``'.'``), falling + back to ``(project_id, source)``. + - ``apaevt_flow`` ``op='begin'`` opens a pipe root span for that + ``(run, pipe id)`` segment, parented to the run's task span when one is + open (else it is a standalone root). + - ``op='enter'`` opens a child component span whose parent is the nearest + open component span on that pipe's stack (else the pipe root); + ``op='leave'`` closes the span paired BY COMPONENT IDENTITY, never by + stack position. + - ``op='end'`` closes the pipe root. Component spans still open are closed + first with status UNSET and ``rocketride.span.unclosed=true``. + - ``apaevt_sse`` becomes a span event on the innermost open span of the + target pipe (or its root). Events for runs/pipes that were never + announced (e.g. the bridge attached mid-run) open implicit roots marked + ``rocketride.span.implicit=true``. + +Bounded state (long-lived bridge guarantee): a run whose ``end`` is never +observed (missed during a disconnect, or an implicit run that never +terminates) cannot hold spans open forever. Open runs are closed — spans +flagged ``rocketride.span.unclosed=true`` — by whichever comes first: + - the seeded ``running`` snapshot on (re)subscribe, which is authoritative + for which tasks are still alive: tracked runs it does not announce are + closed and dropped; + - the ``MAX_TRACKED_RUNS`` cap, which evicts the least-recently-eventful + run when a new one would exceed it; + - :meth:`FlowSpanMapper.close_all` at shutdown. +A deliberately idle-but-alive run (e.g. a webhook service task) is never +expired by a clock: it stays announced in every snapshot, so no TTL heuristic +is used. ``MetricsMapper`` similarly caps its per-run delta bookkeeping at +``MAX_TRACKED_METRIC_RUNS`` entries (least-recently-updated evicted first). + +Timestamps: the monitor wire protocol carries no event timestamps, so spans +use event ARRIVAL time (the OpenTelemetry SDK's clock at ``start``/``end``). +Durations are therefore bridge-observed, not engine-measured. + +Privacy: payload content (``trace.data``, lane payloads, the run-segment +result delivered in ``end.trace``, and SSE ``data``) never reaches span +attributes or events unless the mapper is built with ``include_content=True``, +and is then truncated to ``MAX_CONTENT_LENGTH`` characters. + +GenAI semantic conventions: component spans for ``llm_*`` / ``embedding_*`` / +``agent_*`` / ``tool_*`` nodes carry ``gen_ai.operation.name`` (and +``gen_ai.provider.name`` / ``gen_ai.tool.name`` when derivable from the +component id). Attribute names follow the open-telemetry/semantic-conventions-genai +repository snapshot of July 2026 (Development stability, no tagged release); +deprecated names (``gen_ai.system``, ``gen_ai.usage.prompt_tokens``, ...) are +never emitted. Flow events at trace level ``summary`` carry no model names or +token counts, so ``gen_ai.request.model`` / ``gen_ai.usage.*`` are omitted +rather than invented. NOTE: ``apaevt_status_update`` ``tokens.*`` are compute +credits, NOT LLM tokens, and are deliberately never mapped to ``gen_ai.usage.*``. +""" + +import json +import logging +import re +from dataclasses import dataclass, field +from typing import Any, Dict, List, Tuple + +from .setup import OtelNotInstalledError, missing_otel_message + +logger = logging.getLogger(__name__) + +# Maximum characters of payload content copied into a span attribute or event +# attribute when include_content is enabled. Content beyond this is truncated. +MAX_CONTENT_LENGTH = 8192 + +# Upper bound on concurrently tracked runs (open span state). When a new run +# would exceed it, the least-recently-eventful run is closed (spans flagged +# rocketride.span.unclosed=true) and dropped, so a bridge that never observes +# some runs' 'end' events still has bounded memory. Generous: the engine does +# not run thousands of concurrent tasks per API key. +MAX_TRACKED_RUNS = 1024 + +# Upper bound on MetricsMapper's per-run last-count entries (delta +# bookkeeping); the least-recently-updated entry is evicted first. Evicting a +# run that later reports again re-counts its totals once, so the cap is +# generous relative to realistic concurrent-run counts. +MAX_TRACKED_METRIC_RUNS = 4096 + +# --------------------------------------------------------------------------- +# Attribute name constants. +# +# rocketride.* names are bridge-defined. gen_ai.* names are centralized here +# and follow the open-telemetry/semantic-conventions-genai registry snapshot +# of July 2026 (all gen_ai.* attributes are Development stability; error.type +# is the only Stable attribute referenced). +# --------------------------------------------------------------------------- +ATTR_PROJECT_ID = 'rocketride.project_id' +ATTR_SOURCE = 'rocketride.source' +ATTR_RUN_ID = 'rocketride.run_id' +ATTR_COMPONENT = 'rocketride.component' +ATTR_PIPE_ID = 'rocketride.pipe_id' +ATTR_LANE = 'rocketride.lane' +ATTR_TASK_NAME = 'rocketride.task.name' +ATTR_TASK_RESTARTED = 'rocketride.task.restarted' +ATTR_OBJECT_NAME = 'rocketride.object' +ATTR_FLOW_RESULT = 'rocketride.flow.result' +ATTR_TRACE_DATA = 'rocketride.trace.data' +ATTR_RESULT_CONTENT = 'rocketride.result' +ATTR_SPAN_UNCLOSED = 'rocketride.span.unclosed' +ATTR_SPAN_IMPLICIT = 'rocketride.span.implicit' +ATTR_UNMATCHED_LEAVES = 'rocketride.flow.unmatched_leaves' +ATTR_SSE_TYPE = 'rocketride.sse.type' +ATTR_SSE_DATA = 'rocketride.sse.data' +ATTR_ERROR_TYPE = 'error.type' + +GEN_AI_OPERATION_NAME = 'gen_ai.operation.name' +GEN_AI_PROVIDER_NAME = 'gen_ai.provider.name' +GEN_AI_TOOL_NAME = 'gen_ai.tool.name' + +# error.type is low-cardinality by spec; the wire only gives a free-form error +# string, so the semconv fallback value '_OTHER' is used. +ERROR_TYPE_FALLBACK = '_OTHER' + +# Well-known gen_ai.provider.name values, keyed by RocketRide node provider +# (the component id minus its trailing '_' instance suffix). Only values +# from the semconv well-known list are emitted; unmapped providers omit the +# attribute rather than inventing a value. +_GENAI_PROVIDERS: Dict[str, str] = { + 'llm_openai': 'openai', + 'llm_openai_api': 'openai', + 'llm_vision_openai': 'openai', + 'llm_anthropic': 'anthropic', + 'llm_bedrock': 'aws.bedrock', + 'llm_gemini': 'gcp.gemini', + 'llm_vision_gemini': 'gcp.gemini', + 'llm_mistral': 'mistral_ai', + 'llm_vision_mistral': 'mistral_ai', + 'llm_deepseek': 'deepseek', + 'llm_perplexity': 'perplexity', + 'llm_xai': 'x_ai', + 'llm_ibm_watson': 'ibm.watsonx.ai', + 'embedding_openai': 'openai', +} + +_INSTANCE_SUFFIX = re.compile(r'_\d+$') + + +def _serialize_content(value: Any) -> str: + """Serialize a payload value to a JSON string capped at MAX_CONTENT_LENGTH.""" + if isinstance(value, str): + text = value + else: + try: + text = json.dumps(value, default=str) + except (TypeError, ValueError): + text = str(value) + return text[:MAX_CONTENT_LENGTH] + + +@dataclass +class _ComponentSpan: + """One open component span on a pipe's stack.""" + + component: str + span: Any + + +@dataclass +class _PipeState: + """Open span state for one (run, pipe id).""" + + root: Any + stack: List[_ComponentSpan] = field(default_factory=list) + unmatched_leaves: int = 0 + + +@dataclass +class _RunState: + """Open span state for one engine run (task).""" + + key: str + project_id: str = '' + source: str = '' + name: str = '' + task_span: Any = None + pipes: Dict[int, _PipeState] = field(default_factory=dict) + + +class FlowSpanMapper: + """ + Maps apaevt_flow / apaevt_task / apaevt_sse monitor events to OTel spans. + + Pure consumer of an injected OpenTelemetry ``Tracer``: no network and no + asyncio. Feed it raw DAP event bodies via :meth:`handle_event` and close + any remaining open spans with :meth:`close_all` at shutdown. Tracked-run + state is bounded (see "Bounded state" in the module docstring): runs whose + 'end' is never observed are closed by seeded-snapshot reconciliation or + by the ``MAX_TRACKED_RUNS`` least-recently-eventful eviction. + + Args: + tracer: An ``opentelemetry.trace.Tracer`` (SDK or API no-op). + include_content: When True, payload content (trace.data, end-of-run + results, SSE data) is copied into span attributes, truncated to + ``MAX_CONTENT_LENGTH`` characters. Default False: no payload text + ever reaches a span. + """ + + def __init__(self, tracer: Any, *, include_content: bool = False) -> None: + try: + from opentelemetry import trace as trace_api + from opentelemetry.trace import SpanKind, Status, StatusCode + except ImportError as exc: # pragma: no cover - exercised via subprocess test + raise OtelNotInstalledError(missing_otel_message()) from exc + + self._tracer = tracer + self._include_content = include_content + self._trace_api = trace_api + self._span_kind = SpanKind + self._status = Status + self._status_code = StatusCode + self._runs: Dict[str, _RunState] = {} + # (project_id, source) -> run key, for events that carry no __id. + self._aliases: Dict[Tuple[str, str], str] = {} + + # ------------------------------------------------------------------ + # Public interface + # ------------------------------------------------------------------ + + def handle_event(self, event_name: str, body: dict) -> None: + """Dispatch one monitor event body by its DAP event name.""" + if not isinstance(body, dict): + return + if event_name == 'apaevt_flow': + self._handle_flow(body) + elif event_name == 'apaevt_task': + self._handle_task(body) + elif event_name == 'apaevt_sse': + self._handle_sse(body) + # Other event types (status updates, output, ...) are not span-shaped. + + def close_all(self) -> None: + """Close every open span (components, pipe roots, task spans) as unclosed.""" + for run in list(self._runs.values()): + self._close_run_spans(run, unclosed_task=True) + self._runs.clear() + self._aliases.clear() + + def open_span_count(self) -> int: + """Return the number of currently open spans (for tests/diagnostics).""" + count = 0 + for run in self._runs.values(): + if run.task_span is not None: + count += 1 + for pipe in run.pipes.values(): + count += 1 + len(pipe.stack) + return count + + # ------------------------------------------------------------------ + # Run correlation + # ------------------------------------------------------------------ + + @staticmethod + def _identity(body: dict) -> Tuple[str, str, str]: + """Extract (run_id, project_id, source) tolerating the wire's naming split.""" + run_id = body.get('__id') or '' + # apaevt_task uses camelCase projectId; flow/status use project_id. + project_id = body.get('project_id') or body.get('projectId') or '' + source = body.get('source') or '' + return run_id, project_id, source + + def _resolve_key(self, run_id: str, project_id: str, source: str) -> str: + if run_id: + return run_id + alias = self._aliases.get((project_id, source)) + if alias: + return alias + return f'{project_id}|{source}' + + def _get_run(self, run_id: str, project_id: str, source: str) -> _RunState: + key = self._resolve_key(run_id, project_id, source) + run = self._runs.pop(key, None) + if run is None: + # Evict before inserting so the new run itself is never a victim. + self._evict_runs_above(MAX_TRACKED_RUNS - 1) + run = _RunState(key=key, project_id=project_id, source=source) + # (Re)insert last: dict order stays least-recently-eventful first, + # so cap eviction always hits the longest-idle run. + self._runs[key] = run + if project_id or source: + run.project_id = run.project_id or project_id + run.source = run.source or source + self._aliases[(run.project_id, run.source)] = key + return run + + def _evict_runs_above(self, max_size: int) -> None: + """Close and drop least-recently-eventful runs until at most max_size remain.""" + while len(self._runs) > max_size: + oldest = next(iter(self._runs.values())) + logger.debug('tracked-run cap reached; closing idle run %s as unclosed', oldest.key) + self._close_run_spans(oldest, unclosed_task=True) + self._drop_run(oldest) + + def _drop_run(self, run: _RunState) -> None: + self._runs.pop(run.key, None) + for alias_key, target in list(self._aliases.items()): + if target == run.key: + del self._aliases[alias_key] + + # ------------------------------------------------------------------ + # apaevt_task: task lifecycle -> task-level root spans + # ------------------------------------------------------------------ + + def _handle_task(self, body: dict) -> None: + action = body.get('action') + if action == 'running': + # Seeded snapshot (re-sent on every reconnect): idempotently ensure + # a task span per announced task, never duplicating open ones. + announced_keys = set() + announced_pairs = set() + for task in body.get('tasks') or []: + if not isinstance(task, dict): + continue + run_id, project_id, source = self._identity(task) + run_id = task.get('id') or run_id + if not (run_id or project_id or source): + continue + run = self._get_run(run_id, project_id, source) + self._ensure_task_span(run, task.get('name') or '') + announced_keys.add(run.key) + announced_pairs.add((run.project_id, run.source)) + # The snapshot is authoritative for which tasks are still alive: + # any tracked run it does not announce (by key or by its + # project/source identity) ended while its 'end' event was missed + # (e.g. during a disconnect). Close it as unclosed and drop it so + # a long-lived bridge does not accrue zombie runs. + for run in list(self._runs.values()): + if run.key in announced_keys or (run.project_id, run.source) in announced_pairs: + continue + logger.debug('run %s missing from seeded running snapshot; closing as unclosed', run.key) + self._close_run_spans(run, unclosed_task=True) + self._drop_run(run) + return + + run_id, project_id, source = self._identity(body) + if not (run_id or project_id or source): + return + run = self._get_run(run_id, project_id, source) + name = body.get('name') or '' + + if action == 'begin': + self._ensure_task_span(run, name) + elif action == 'end': + self._close_run_spans(run, unclosed_task=False) + self._drop_run(run) + elif action == 'restart': + # The engine restarted the task: close the previous segment's open + # spans and open a fresh task span flagged as a restart. + self._close_run_spans(run, unclosed_task=False) + self._ensure_task_span(run, name) + run.task_span.set_attribute(ATTR_TASK_RESTARTED, True) + + def _ensure_task_span(self, run: _RunState, name: str) -> None: + if run.task_span is not None: + return # already announced (seeded 'running' snapshots repeat on reconnect) + run.name = name or run.name + attributes = { + ATTR_PROJECT_ID: run.project_id, + ATTR_SOURCE: run.source, + ATTR_RUN_ID: run.key, + } + if run.name: + attributes[ATTR_TASK_NAME] = run.name + run.task_span = self._tracer.start_span( + name=f'task {run.name}' if run.name else 'task', + kind=self._span_kind.INTERNAL, + attributes=attributes, + ) + + def _close_run_spans(self, run: _RunState, *, unclosed_task: bool) -> None: + """ + Close all open pipe/component spans of a run, then its task span. + + Pipe/component spans still open here always count as unclosed (their + flow leave/end never arrived). The task span is only flagged unclosed + when no lifecycle signal justified the close (bridge shutdown, + seeded-snapshot reconciliation, or tracked-run cap eviction); a task + 'end'/'restart' event is an authoritative close. + """ + for pipe_id in list(run.pipes): + self._close_pipe(run, pipe_id, unclosed=True) + if run.task_span is not None: + if unclosed_task: + run.task_span.set_attribute(ATTR_SPAN_UNCLOSED, True) + run.task_span.end() + run.task_span = None + + # ------------------------------------------------------------------ + # apaevt_flow: pipe segments and component spans + # ------------------------------------------------------------------ + + def _handle_flow(self, body: dict) -> None: + run_id, project_id, source = self._identity(body) + run = self._get_run(run_id, project_id, source) + pipe_id = body.get('id') + if not isinstance(pipe_id, int): + return + op = body.get('op') + trace = body.get('trace') if isinstance(body.get('trace'), dict) else {} + component = body.get('component') or '' + + if op == 'begin': + self._flow_begin(run, pipe_id, component, body) + elif op == 'enter': + self._flow_enter(run, pipe_id, component, trace) + elif op == 'leave': + self._flow_leave(run, pipe_id, component, trace) + elif op == 'end': + self._flow_end(run, pipe_id, trace) + + def _pipe_attributes(self, run: _RunState, pipe_id: int) -> Dict[str, Any]: + return { + ATTR_PROJECT_ID: run.project_id, + ATTR_SOURCE: run.source, + ATTR_RUN_ID: run.key, + ATTR_PIPE_ID: pipe_id, + } + + def _context_for(self, parent_span: Any) -> Any: + if parent_span is None: + return None + return self._trace_api.set_span_in_context(parent_span) + + def _open_root(self, run: _RunState, pipe_id: int, name: str, *, implicit: bool = False) -> _PipeState: + attributes = self._pipe_attributes(run, pipe_id) + if implicit: + attributes[ATTR_SPAN_IMPLICIT] = True + root = self._tracer.start_span( + name=name, + kind=self._span_kind.INTERNAL, + context=self._context_for(run.task_span), + attributes=attributes, + ) + state = _PipeState(root=root) + run.pipes[pipe_id] = state + return state + + def _get_pipe(self, run: _RunState, pipe_id: int) -> _PipeState: + """Return the pipe state, opening an implicit root for never-begun pipes.""" + state = run.pipes.get(pipe_id) + if state is None: + logger.debug('flow event for pipe %s of run %s before begin; opening implicit root', pipe_id, run.key) + state = self._open_root(run, pipe_id, f'pipe {pipe_id}', implicit=True) + return state + + def _flow_begin(self, run: _RunState, pipe_id: int, component: str, body: dict) -> None: + existing = run.pipes.get(pipe_id) + if existing is not None: + # A new object segment began before the previous one ended. + self._close_pipe(run, pipe_id, unclosed=True) + # On begin/end 'component' is the OBJECT name (== pipes[0]), not a + # pipeline component id. + pipes = body.get('pipes') or [] + object_name = component or (pipes[0] if pipes else f'pipe {pipe_id}') + state = self._open_root(run, pipe_id, object_name) + state.root.set_attribute(ATTR_OBJECT_NAME, object_name) + + def _flow_enter(self, run: _RunState, pipe_id: int, component: str, trace: dict) -> None: + state = self._get_pipe(run, pipe_id) + parent = state.stack[-1].span if state.stack else state.root + name, kind, genai_attrs = self._classify_component(component) + attributes = self._pipe_attributes(run, pipe_id) + attributes[ATTR_COMPONENT] = component + attributes.update(genai_attrs) + lane = trace.get('lane') + if isinstance(lane, str): + attributes[ATTR_LANE] = lane + if self._include_content and trace.get('data') is not None: + attributes[ATTR_TRACE_DATA] = _serialize_content(trace['data']) + span = self._tracer.start_span( + name=name, + kind=kind, + context=self._context_for(parent), + attributes=attributes, + ) + state.stack.append(_ComponentSpan(component=component, span=span)) + + def _flow_leave(self, run: _RunState, pipe_id: int, component: str, trace: dict) -> None: + state = self._get_pipe(run, pipe_id) + # Pair by component identity (innermost matching entry), never by + # stack position: leave's 'pipes' is already popped on the wire. + index = None + for i in range(len(state.stack) - 1, -1, -1): + if state.stack[i].component == component: + index = i + break + if index is None: + state.unmatched_leaves += 1 + logger.debug('unmatched leave for component %r on pipe %s of run %s', component, pipe_id, run.key) + return + entry = state.stack.pop(index) + span = entry.span + result = trace.get('result') + if isinstance(result, str): + # Flow-control verdict ('continue', ...), not payload content. + span.set_attribute(ATTR_FLOW_RESULT, result) + if self._include_content and trace.get('data') is not None: + span.set_attribute(ATTR_TRACE_DATA, _serialize_content(trace['data'])) + error = trace.get('error') + if error: + error_text = error if isinstance(error, str) else _serialize_content(error) + span.set_attribute(ATTR_ERROR_TYPE, ERROR_TYPE_FALLBACK) + # exception.type is omitted: the wire carries no exception class + # name, and the '_OTHER' fallback is defined only for error.type. + span.add_event('exception', {'exception.message': error_text}) + span.set_status(self._status(self._status_code.ERROR, error_text)) + span.end() + + def _flow_end(self, run: _RunState, pipe_id: int, trace: dict) -> None: + state = run.pipes.get(pipe_id) + if state is None: + logger.debug('flow end for unknown pipe %s of run %s; ignoring', pipe_id, run.key) + return + # WIRE TRUTH: at level=summary the PIPELINE_RESULT arrives INSIDE + # body.trace (there is no top-level 'result' field), and it contains + # actual content -- it is gated behind include_content. + if self._include_content and trace: + state.root.set_attribute(ATTR_RESULT_CONTENT, _serialize_content(trace)) + self._close_pipe(run, pipe_id, unclosed=False) + + def _close_pipe(self, run: _RunState, pipe_id: int, *, unclosed: bool) -> None: + state = run.pipes.pop(pipe_id, None) + if state is None: + return + # Missing leave at end/shutdown: status stays UNSET, flagged unclosed. + while state.stack: + entry = state.stack.pop() + entry.span.set_attribute(ATTR_SPAN_UNCLOSED, True) + entry.span.end() + if state.unmatched_leaves: + state.root.set_attribute(ATTR_UNMATCHED_LEAVES, state.unmatched_leaves) + if unclosed: + state.root.set_attribute(ATTR_SPAN_UNCLOSED, True) + state.root.end() + + # ------------------------------------------------------------------ + # apaevt_sse: node-emitted custom messages -> span events + # ------------------------------------------------------------------ + + def _handle_sse(self, body: dict) -> None: + pipe_id = body.get('pipe_id') + sse_type = body.get('type') or 'sse' + run_id = body.get('__id') or '' + # SSE bodies carry no project_id/source; __id presence is unverified + # on the wire. Route by __id when present (creating an implicit run if + # the bridge attached mid-run), fall back to the sole active run, else + # drop with a debug log. + if run_id: + run = self._get_run(run_id, '', '') + elif len(self._runs) == 1: + run = next(iter(self._runs.values())) + else: + logger.debug('cannot route apaevt_sse (type=%r) to a run; dropping', sse_type) + return + + attributes: Dict[str, Any] = {ATTR_SSE_TYPE: sse_type} + # 'data' is OMITTED (not null) on the wire when empty. + if self._include_content and body.get('data') is not None: + attributes[ATTR_SSE_DATA] = _serialize_content(body['data']) + + if isinstance(pipe_id, int): + state = self._get_pipe(run, pipe_id) + target = state.stack[-1].span if state.stack else state.root + elif run.task_span is not None: + target = run.task_span + else: + logger.debug('apaevt_sse without pipe_id and no task span for run %s; dropping', run.key) + return + target.add_event(sse_type, attributes) + + # ------------------------------------------------------------------ + # GenAI semantic conventions (Development stability, July 2026 snapshot) + # ------------------------------------------------------------------ + + def _classify_component(self, component: str) -> Tuple[str, Any, Dict[str, Any]]: + """ + Derive (span name, span kind, gen_ai attributes) for a component id. + + RocketRide component ids are '_' (e.g. 'llm_openai_1'). + LLM/embedding nodes call remote model services -> SpanKind CLIENT with + the spec span name '{operation} {model}' degraded to the bare operation + because flow events carry no model name. Tool/agent nodes execute + in-process -> INTERNAL. Everything else is a plain INTERNAL span named + by the component id. + """ + base = _INSTANCE_SUFFIX.sub('', component) + genai: Dict[str, Any] = {} + if base.startswith('llm_'): + genai[GEN_AI_OPERATION_NAME] = 'chat' + provider = _GENAI_PROVIDERS.get(base) + if provider: + genai[GEN_AI_PROVIDER_NAME] = provider + return 'chat', self._span_kind.CLIENT, genai + if base.startswith('embedding_'): + genai[GEN_AI_OPERATION_NAME] = 'embeddings' + provider = _GENAI_PROVIDERS.get(base) + if provider: + genai[GEN_AI_PROVIDER_NAME] = provider + return 'embeddings', self._span_kind.CLIENT, genai + if base.startswith('agent_'): + # In-process agent frameworks are INTERNAL per gen-ai-agent-spans; + # bare 'invoke_agent' because no agent name is available. + genai[GEN_AI_OPERATION_NAME] = 'invoke_agent' + return 'invoke_agent', self._span_kind.INTERNAL, genai + if base.startswith('tool_'): + tool_name = base[len('tool_') :] + genai[GEN_AI_OPERATION_NAME] = 'execute_tool' + genai[GEN_AI_TOOL_NAME] = tool_name + return f'execute_tool {tool_name}', self._span_kind.INTERNAL, genai + return component or 'component', self._span_kind.INTERNAL, genai + + +class MetricsMapper: + """ + Maps apaevt_status_update snapshots to OpenTelemetry metrics. + + Pure consumer of an injected OpenTelemetry ``Meter``. Object counts are + emitted as non-monotonic up-down counters fed with per-run deltas between + snapshots (counts reset on restart, so deltas may be negative); rates and + resource metrics are instantaneous gauges. All instruments carry + ``rocketride.project_id`` / ``rocketride.source`` attributes. + + NOTE: the snapshot's ``tokens.*`` block is compute credits (100 = $1), NOT + LLM tokens; it is intentionally not exported as ``gen_ai.usage.*``. + Zero-valued metrics snapshots are exported as-is (all-zero is legitimate). + + Delta bookkeeping is bounded: at most ``MAX_TRACKED_METRIC_RUNS`` per-run + entries are kept, evicting the least-recently-updated first, so a + long-lived bridge does not grow with the number of runs ever observed. + """ + + def __init__(self, meter: Any) -> None: + self._meter = meter + self._last_counts: Dict[str, Tuple[int, int, int]] = {} + self._objects_total = meter.create_up_down_counter( + 'rocketride.objects.total', unit='{object}', description='Objects seen by the pipeline run' + ) + self._objects_completed = meter.create_up_down_counter( + 'rocketride.objects.completed', unit='{object}', description='Objects completed by the pipeline run' + ) + self._objects_failed = meter.create_up_down_counter( + 'rocketride.objects.failed', unit='{object}', description='Objects failed by the pipeline run' + ) + self._rate_count = meter.create_gauge( + 'rocketride.rate.count', unit='{object}/s', description='Instantaneous object processing rate' + ) + self._rate_size = meter.create_gauge( + 'rocketride.rate.size', unit='By/s', description='Instantaneous byte processing rate' + ) + self._cpu_percent = meter.create_gauge('rocketride.cpu.percent', unit='%', description='Engine CPU utilization') + self._cpu_percent_peak = meter.create_gauge( + 'rocketride.cpu.percent.peak', unit='%', description='Peak engine CPU utilization' + ) + self._cpu_memory = meter.create_gauge( + 'rocketride.memory.cpu_mb', unit='MBy', description='Engine CPU memory usage' + ) + self._cpu_memory_peak = meter.create_gauge( + 'rocketride.memory.cpu_mb.peak', unit='MBy', description='Peak engine CPU memory usage' + ) + self._gpu_memory = meter.create_gauge( + 'rocketride.memory.gpu_mb', unit='MBy', description='Engine GPU memory usage' + ) + self._gpu_memory_peak = meter.create_gauge( + 'rocketride.memory.gpu_mb.peak', unit='MBy', description='Peak engine GPU memory usage' + ) + + def handle_status(self, body: dict) -> None: + """Record metrics from one apaevt_status_update (TASK_STATUS) snapshot.""" + if not isinstance(body, dict): + return + project_id = body.get('project_id') or body.get('projectId') or '' + source = body.get('source') or '' + attributes = {ATTR_PROJECT_ID: project_id, ATTR_SOURCE: source} + run_key = body.get('__id') or f'{project_id}|{source}' + + total = int(body.get('totalCount') or 0) + completed = int(body.get('completedCount') or 0) + failed = int(body.get('failedCount') or 0) + # pop + reinsert keeps dict order least-recently-updated first, so the + # cap below always evicts the run that has been silent the longest. + # (A snapshot can arrive after the task's 'end' event, so entries are + # NOT dropped on task end — that would re-count the final totals.) + last_total, last_completed, last_failed = self._last_counts.pop(run_key, (0, 0, 0)) + self._objects_total.add(total - last_total, attributes) + self._objects_completed.add(completed - last_completed, attributes) + self._objects_failed.add(failed - last_failed, attributes) + self._last_counts[run_key] = (total, completed, failed) + while len(self._last_counts) > MAX_TRACKED_METRIC_RUNS: + del self._last_counts[next(iter(self._last_counts))] + + self._rate_count.set(float(body.get('rateCount') or 0), attributes) + self._rate_size.set(float(body.get('rateSize') or 0), attributes) + + metrics = body.get('metrics') if isinstance(body.get('metrics'), dict) else {} + self._cpu_percent.set(float(metrics.get('cpu_percent') or 0.0), attributes) + self._cpu_percent_peak.set(float(metrics.get('peak_cpu_percent') or 0.0), attributes) + self._cpu_memory.set(float(metrics.get('cpu_memory_mb') or 0.0), attributes) + self._cpu_memory_peak.set(float(metrics.get('peak_cpu_memory_mb') or 0.0), attributes) + self._gpu_memory.set(float(metrics.get('gpu_memory_mb') or 0.0), attributes) + self._gpu_memory_peak.set(float(metrics.get('peak_gpu_memory_mb') or 0.0), attributes) diff --git a/packages/client-python/src/rocketride/otelbridge/setup.py b/packages/client-python/src/rocketride/otelbridge/setup.py new file mode 100644 index 000000000..b5340e52e --- /dev/null +++ b/packages/client-python/src/rocketride/otelbridge/setup.py @@ -0,0 +1,187 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +OpenTelemetry provider construction for the RocketRide OTel bridge. + +This module builds the TracerProvider / MeterProvider pair (with OTLP +exporters) that the ``rocketride otel`` bridge feeds. ALL imports of the +``opentelemetry`` packages are lazy (function-local): importing this module +never requires the 'otel' extra, and a missing extra surfaces as +:class:`OtelNotInstalledError` with the exact install command. + +Endpoint semantics (matching the OTLP exporter spec): + - ``config.endpoint`` set: treated as a BASE url; the per-signal paths + ``/v1/traces`` / ``/v1/metrics`` are appended unless already present, + so pasting Langfuse's ``https:///api/public/otel`` or LangSmith's + ``https://api.smith.langchain.com/otel`` just works (http protocol). + - ``config.endpoint`` unset: the exporters are constructed without an + explicit endpoint so the standard ``OTEL_EXPORTER_OTLP_ENDPOINT`` / + ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` / ``OTEL_EXPORTER_OTLP_HEADERS`` + environment variables (and the localhost:4318 default) apply. + - ``protocol='grpc'``: requires the optional + ``opentelemetry-exporter-otlp-proto-grpc`` package (not part of the + 'otel' extra); gRPC endpoints are used verbatim (no per-signal path). +""" + +from typing import Any, Callable, Dict, Optional, Tuple + +_INSTALL_HINT = "pip install 'rocketride[otel]'" +_GRPC_INSTALL_HINT = 'pip install opentelemetry-exporter-otlp-proto-grpc' + +# Instrumentation scope name for tracer/meter lookups. +SCOPE_NAME = 'rocketride.otelbridge' + + +class OtelNotInstalledError(RuntimeError): + """Raised when an OpenTelemetry dependency needed by the bridge is missing.""" + + +def missing_otel_message() -> str: + """Return the user-facing message for a missing 'otel' extra.""" + return f"The 'rocketride otel' bridge requires the OpenTelemetry SDK. Install it with: {_INSTALL_HINT}" + + +def _resolve_endpoint(base: str, signal_path: str) -> str: + """ + Append a per-signal OTLP path to a base endpoint unless already present. + + Args: + base: User-supplied base URL (e.g. ``http://localhost:4318`` or + ``https://cloud.langfuse.com/api/public/otel``). + signal_path: Signal path without leading slash (``v1/traces``). + """ + trimmed = base.rstrip('/') + if trimmed.endswith('/' + signal_path): + return trimmed + return f'{trimmed}/{signal_path}' + + +def _build_resource(config: Any) -> Any: + """Build the OTel Resource carrying service.name (+ service.version when known).""" + from opentelemetry.sdk.resources import Resource + + attributes: Dict[str, Any] = {'service.name': config.service_name} + try: + from importlib.metadata import version + + attributes['service.version'] = version('rocketride') + except Exception: # noqa: BLE001 - version is best-effort metadata only + pass + return Resource.create(attributes) + + +def _exporter_headers(config: Any) -> Optional[Dict[str, str]]: + """Return explicit headers, or None so OTEL_EXPORTER_OTLP_HEADERS applies.""" + headers = getattr(config, 'headers', None) + return dict(headers) if headers else None + + +def _build_span_exporter(config: Any) -> Any: + """Build the OTLP span exporter for config.protocol ('http' or 'grpc').""" + protocol = getattr(config, 'protocol', 'http') or 'http' + headers = _exporter_headers(config) + if protocol == 'grpc': + try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + except ImportError as exc: + raise OtelNotInstalledError( + f'--protocol grpc requires the OTLP gRPC exporter. Install it with: {_GRPC_INSTALL_HINT}' + ) from exc + if config.endpoint: + return OTLPSpanExporter(endpoint=config.endpoint, headers=headers) + return OTLPSpanExporter(headers=headers) + + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + + if config.endpoint: + return OTLPSpanExporter(endpoint=_resolve_endpoint(config.endpoint, 'v1/traces'), headers=headers) + return OTLPSpanExporter(headers=headers) + + +def _build_metric_exporter(config: Any) -> Any: + """Build the OTLP metric exporter for config.protocol ('http' or 'grpc').""" + protocol = getattr(config, 'protocol', 'http') or 'http' + headers = _exporter_headers(config) + if protocol == 'grpc': + try: + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + except ImportError as exc: + raise OtelNotInstalledError( + f'--protocol grpc requires the OTLP gRPC exporter. Install it with: {_GRPC_INSTALL_HINT}' + ) from exc + if config.endpoint: + return OTLPMetricExporter(endpoint=config.endpoint, headers=headers) + return OTLPMetricExporter(headers=headers) + + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + + if config.endpoint: + return OTLPMetricExporter(endpoint=_resolve_endpoint(config.endpoint, 'v1/metrics'), headers=headers) + return OTLPMetricExporter(headers=headers) + + +def build_providers(config: Any) -> Tuple[Any, Any, Callable[[], None]]: + """ + Build (tracer, meter, shutdown_fn) from an OtelConfig. + + The providers are kept local (the global OpenTelemetry tracer/meter + providers are never touched) so tests and embedders stay isolated. + ``shutdown_fn`` flushes and shuts down both providers; call it once on + exit (SIGINT/SIGTERM handling lives in the bridge loop). + + Args: + config: An object with ``endpoint``, ``protocol``, ``service_name``, + ``include_content``, ``no_metrics`` and ``headers`` attributes + (see ``rocketride.otelbridge.config.OtelConfig``). + + Raises: + OtelNotInstalledError: The 'otel' extra (or the gRPC exporter for + ``protocol='grpc'``) is not installed. + """ + try: + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError as exc: + raise OtelNotInstalledError(missing_otel_message()) from exc + + resource = _build_resource(config) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor(BatchSpanProcessor(_build_span_exporter(config))) + tracer = tracer_provider.get_tracer(SCOPE_NAME) + + if getattr(config, 'no_metrics', False): + meter_provider = MeterProvider(resource=resource, metric_readers=[]) + else: + reader = PeriodicExportingMetricReader(_build_metric_exporter(config)) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader]) + meter = meter_provider.get_meter(SCOPE_NAME) + + def shutdown() -> None: + """Flush pending telemetry and shut down both providers.""" + tracer_provider.shutdown() + meter_provider.shutdown() + + return tracer, meter, shutdown diff --git a/packages/client-python/tests/fixtures/otel_bridge_events.json b/packages/client-python/tests/fixtures/otel_bridge_events.json new file mode 100644 index 000000000..e3a010e94 --- /dev/null +++ b/packages/client-python/tests/fixtures/otel_bridge_events.json @@ -0,0 +1,654 @@ +[ + { + "event": "apaevt_task", + "body": { + "action": "running", + "tasks": [], + "__id": "" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": false, + "state": 1, + "startTime": 1784207709.2029505, + "endTime": 0.0, + "debuggerAttached": false, + "status": "", + "warnings": [], + "errors": [], + "currentObject": "", + "currentSize": 0, + "notes": [], + "totalSize": 0, + "totalCount": 0, + "completedSize": 0, + "completedCount": 0, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": false, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 0, + "byPipe": {} + }, + "metrics": { + "cpu_percent": 0.0, + "cpu_memory_mb": 0.0, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 0.0, + "peak_cpu_memory_mb": 0.0, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 0.0, + "avg_cpu_memory_mb": 0.0, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_task", + "body": { + "action": "begin", + "name": "otel-scout-echo.My webhook", + "projectId": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": false, + "state": 2, + "startTime": 1784207709.2029505, + "endTime": 0.0, + "debuggerAttached": false, + "status": "", + "warnings": [], + "errors": [], + "currentObject": "", + "currentSize": 0, + "notes": [], + "totalSize": 0, + "totalCount": 0, + "completedSize": 0, + "completedCount": 0, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": false, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 0, + "byPipe": {} + }, + "metrics": { + "cpu_percent": 0.0, + "cpu_memory_mb": 0.0, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 0.0, + "peak_cpu_memory_mb": 0.0, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 0.0, + "avg_cpu_memory_mb": 0.0, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": false, + "state": 3, + "startTime": 1784207709.2029505, + "endTime": 0.0, + "debuggerAttached": false, + "status": " + versioneer==0.29", + "warnings": [], + "errors": [], + "currentObject": "", + "currentSize": 0, + "notes": [], + "totalSize": 0, + "totalCount": 0, + "completedSize": 0, + "completedCount": 0, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": true, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 0, + "byPipe": {} + }, + "metrics": { + "cpu_percent": 3.9, + "cpu_memory_mb": 203.28515625, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 61.45, + "peak_cpu_memory_mb": 322.24609375, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 0.0, + "avg_cpu_memory_mb": 0.0, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "begin", + "pipes": [ + "probe.txt" + ], + "component": "probe.txt", + "trace": {}, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "enter", + "pipes": [ + "probe.txt", + "response_1" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "open" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "leave", + "pipes": [ + "probe.txt" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "open", + "result": "continue" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "enter", + "pipes": [ + "probe.txt", + "response_1" + ], + "component": "response_1", + "trace": { + "data": { + "length": 17 + }, + "lane": "text" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "leave", + "pipes": [ + "probe.txt" + ], + "component": "response_1", + "trace": { + "data": { + "length": 17 + }, + "lane": "text", + "result": "continue" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "enter", + "pipes": [ + "probe.txt", + "response_1" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "closing" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "leave", + "pipes": [ + "probe.txt" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "closing", + "result": "continue" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "enter", + "pipes": [ + "probe.txt", + "response_1" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "close" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "leave", + "pipes": [ + "probe.txt" + ], + "component": "response_1", + "trace": { + "data": null, + "lane": "close", + "result": "continue" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "end", + "pipes": [], + "component": "probe.txt", + "trace": { + "name": "probe.txt", + "path": "", + "result_types": { + "text": "text" + }, + "text": [ + "hello otel bridge\n\n" + ] + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": false, + "state": 3, + "startTime": 1784207709.2029505, + "endTime": 0.0, + "debuggerAttached": false, + "status": "Webhook ready - system is ready to accept requests", + "warnings": [], + "errors": [], + "currentObject": "webhook://WebHook", + "currentSize": 0, + "notes": [ + { + "url-text": "Webhook interface URL", + "url-link": "{host}/webhook", + "auth-text": "Public Authorization Key", + "auth-key": "pk_EXAMPLEPUBLICKEYEXAMPLEPUBLICKEY", + "token-text": "Private Token", + "token-key": "tk_EXAMPLEPRIVATETOKENEXAMPLETOKEN" + } + ], + "totalSize": 17, + "totalCount": 1, + "completedSize": 17, + "completedCount": 1, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": true, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 1, + "byPipe": { + "0": [] + } + }, + "metrics": { + "cpu_percent": 0.0, + "cpu_memory_mb": 301.2265625, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 61.45, + "peak_cpu_memory_mb": 322.24609375, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 34.43921109881824, + "avg_cpu_memory_mb": 299.94935548579383, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": false, + "state": 4, + "startTime": 1784207709.2029505, + "endTime": 0.0, + "debuggerAttached": false, + "status": "Stopping", + "warnings": [], + "errors": [], + "currentObject": "webhook://WebHook", + "currentSize": 0, + "notes": [], + "totalSize": 17, + "totalCount": 1, + "completedSize": 17, + "completedCount": 1, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": false, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 1, + "byPipe": { + "0": [] + } + }, + "metrics": { + "cpu_percent": 0.0, + "cpu_memory_mb": 302.08203125, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 61.45, + "peak_cpu_memory_mb": 322.24609375, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 19.712562463845757, + "avg_cpu_memory_mb": 301.23662025002017, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_task", + "body": { + "action": "end", + "name": "otel-scout-echo.My webhook", + "projectId": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_status_update", + "body": { + "name": "otel-scout-echo.My webhook", + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "completed": true, + "state": 6, + "startTime": 1784207709.2029505, + "endTime": 1784207736.5963075, + "debuggerAttached": false, + "status": "Stopped", + "warnings": [], + "errors": [], + "currentObject": "webhook://WebHook", + "currentSize": 0, + "notes": [], + "totalSize": 17, + "totalCount": 1, + "completedSize": 17, + "completedCount": 1, + "failedSize": 0, + "failedCount": 0, + "wordsSize": 0, + "wordsCount": 0, + "rateSize": 0, + "rateCount": 0, + "serviceUp": false, + "exitCode": 0, + "exitMessage": "", + "pipeflow": { + "totalPipes": 1, + "byPipe": { + "0": [] + } + }, + "metrics": { + "cpu_percent": 0.0, + "cpu_memory_mb": 302.08203125, + "gpu_memory_mb": 0.0, + "peak_cpu_percent": 61.45, + "peak_cpu_memory_mb": 322.24609375, + "peak_gpu_memory_mb": 0.0, + "avg_cpu_percent": 19.712562463845757, + "avg_cpu_memory_mb": 301.23662025002017, + "avg_gpu_memory_mb": 0.0 + }, + "tokens": { + "cpu_utilization": 0.0, + "cpu_memory": 0.0, + "gpu_memory": 0.0, + "gpu_inference": 0.0, + "custom": {}, + "total": 0.0 + }, + "__id": "1b4bbac0.webhook_1" + } + }, + { + "event": "apaevt_task", + "body": { + "action": "running", + "tasks": [ + { + "id": "29d98820.webhook_1", + "name": "otel-scout-running.My webhook", + "projectId": "7f1c2ab0-1234-4c3d-9a8b-aaaaaaaaaaaa", + "source": "webhook_1" + } + ], + "__id": "" + } + }, + { + "event": "apaevt_sse", + "body": { + "pipe_id": 0, + "type": "thinking", + "data": { + "message": "Analyzing your request..." + }, + "__id": "1b4bbac0.webhook_1" + }, + "_synthesized": true + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "enter", + "pipes": [ + "probe.txt", + "llm_openai_1" + ], + "component": "llm_openai_1", + "trace": { + "data": { + "length": 17 + }, + "lane": "questions" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + }, + "_synthesized": true + }, + { + "event": "apaevt_flow", + "body": { + "id": 0, + "op": "leave", + "pipes": [ + "probe.txt" + ], + "component": "llm_openai_1", + "trace": { + "data": null, + "lane": "questions", + "error": "OpenAI API error: 401 Unauthorized" + }, + "project_id": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + }, + "_synthesized": true + }, + { + "event": "apaevt_task", + "body": { + "action": "restart", + "name": "otel-scout-echo.My webhook", + "projectId": "e612b741-748c-4b35-a8b7-186797a8ea42", + "source": "webhook_1", + "__id": "1b4bbac0.webhook_1" + }, + "_synthesized": true + } +] diff --git a/packages/client-python/tests/test_otel_bridge.py b/packages/client-python/tests/test_otel_bridge.py new file mode 100644 index 000000000..ab9a033b7 --- /dev/null +++ b/packages/client-python/tests/test_otel_bridge.py @@ -0,0 +1,391 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Unit tests for the OpenTelemetry bridge run loop and configuration. + +These tests use in-process fakes for the client and the mappers, so they run +without the 'rocketride[otel]' extra installed and without a live server +(unlike the integration tests that use the shared conftest client fixture). + +Covered here: + - OtelConfig precedence: CLI args > OTEL_* env vars > defaults + - OTEL_EXPORTER_OTLP_HEADERS parsing (first-'=' split, whitespace, padding) + - run_bridge event routing (task/flow/sse -> mapper, status -> metrics) + - Wildcard monitor subscription (token '*', TASK/SUMMARY/FLOW/SSE) + - Startup connection / subscription failure -> exit code 2 + - Reconnect loop with capped backoff (no hand-rolled resubscription) + - Shutdown order: close_all() before exporter shutdown_fn() + - --no-metrics: metrics factory never invoked, status events dropped + - Dispatcher isolation: a raising mapper does not kill the bridge + - Replay of the recorded wire fixture (tests/fixtures/otel_bridge_events.json) +""" + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from rocketride.otelbridge.bridge import MONITOR_TYPES, run_bridge +from rocketride.otelbridge.config import ( + DEFAULT_PROTOCOL, + DEFAULT_SERVICE_NAME, + ENV_OTLP_ENDPOINT, + ENV_OTLP_HEADERS, + ENV_SERVICE_NAME, + OtelConfig, + parse_headers, +) + +FIXTURE_PATH = Path(__file__).parent / 'fixtures' / 'otel_bridge_events.json' + + +# ========================================================================= +# FAKES +# ========================================================================= + + +class FakeClient: + """Minimal stand-in for RocketRideClient as seen by run_bridge.""" + + def __init__(self, connected: bool = False, connect_failures: int = 0): + self._connected = connected + self.connect_failures = connect_failures + self.connect_calls = 0 + self.monitor_calls = [] + self._caller_on_event = None + + def is_connected(self) -> bool: + return self._connected + + async def connect(self): + self.connect_calls += 1 + if self.connect_failures > 0: + self.connect_failures -= 1 + raise ConnectionError('connection refused') + self._connected = True + + async def add_monitor(self, key, types): + self.monitor_calls.append((dict(key), list(types))) + + async def emit(self, event: str, body: dict): + """Deliver one DAP event envelope to the attached handler.""" + handler = self._caller_on_event + if handler is not None: + await handler({'type': 'event', 'event': event, 'seq': 0, 'body': body}) + + +class FakeMapper: + """Records handle_event/close_all calls; optional shared order log.""" + + def __init__(self, order_log=None): + self.events = [] + self.closed = False + self._order_log = order_log + + def handle_event(self, event_name, body): + self.events.append((event_name, body)) + + def close_all(self): + self.closed = True + if self._order_log is not None: + self._order_log.append('close_all') + + +class FakeMetrics: + """Records handle_status calls.""" + + def __init__(self): + self.statuses = [] + + def handle_status(self, body): + self.statuses.append(body) + + +async def _wait_until(predicate, timeout: float = 2.0): + """Poll until predicate() is truthy or fail the test after timeout.""" + deadline = asyncio.get_running_loop().time() + timeout + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + pytest.fail('timed out waiting for condition') + await asyncio.sleep(0.005) + + +def _start_bridge(client, config=None, mapper=None, metrics=None, **kwargs): + """Start run_bridge as a task; returns (task, stop_event, mapper, metrics).""" + config = config or OtelConfig() + mapper = mapper if mapper is not None else FakeMapper() + metrics = metrics if metrics is not None else FakeMetrics() + stop_event = kwargs.pop('stop_event', asyncio.Event()) + task = asyncio.ensure_future( + run_bridge( + client, + config, + lambda: mapper, + lambda: metrics, + stop_event=stop_event, + poll_interval=kwargs.pop('poll_interval', 0.01), + **kwargs, + ) + ) + return task, stop_event, mapper, metrics + + +# ========================================================================= +# CONFIG: precedence and header parsing +# ========================================================================= + + +class TestOtelConfig: + def test_defaults_when_no_args_no_env(self): + config = OtelConfig.from_args_env(SimpleNamespace(), env={}) + # endpoint stays None so the OTLP exporters' own env/default + # semantics apply (http://localhost:4318 for http, 4317 for grpc). + assert config.endpoint is None + assert config.protocol == DEFAULT_PROTOCOL == 'http' + assert config.service_name == DEFAULT_SERVICE_NAME == 'rocketride-engine' + assert config.include_content is False + assert config.no_metrics is False + assert config.headers == {} + + def test_signal_specific_env_left_to_the_sdk_exporters(self): + # OTEL_EXPORTER_OTLP_TRACES/METRICS_ENDPOINT must reach the SDK + # exporters: the bridge keeps endpoint None rather than clobbering + # them with a hardcoded default (the exporters read them directly; + # see test_otel_setup.py for the exporter-level proof). + env = {'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT': 'http://traces.example:4318/v1/traces'} + config = OtelConfig.from_args_env(SimpleNamespace(), env=env) + assert config.endpoint is None + + def test_env_used_when_args_absent(self, monkeypatch): + monkeypatch.setenv(ENV_OTLP_ENDPOINT, 'https://collector.example:4318') + monkeypatch.setenv(ENV_OTLP_HEADERS, 'Authorization=Basic cGs6c2s=') + monkeypatch.setenv(ENV_SERVICE_NAME, 'env-service') + config = OtelConfig.from_args_env(SimpleNamespace()) + assert config.endpoint == 'https://collector.example:4318' + assert config.headers == {'Authorization': 'Basic cGs6c2s='} + assert config.service_name == 'env-service' + + def test_args_override_env(self): + env = { + ENV_OTLP_ENDPOINT: 'https://env.example:4318', + ENV_OTLP_HEADERS: 'a=env', + ENV_SERVICE_NAME: 'env-service', + } + args = SimpleNamespace( + endpoint='https://args.example:4318', + protocol='grpc', + service_name='args-service', + headers='x-api-key=key123,Langsmith-Project=proj', + include_content=True, + no_metrics=True, + ) + config = OtelConfig.from_args_env(args, env=env) + assert config.endpoint == 'https://args.example:4318' + assert config.protocol == 'grpc' + assert config.service_name == 'args-service' + assert config.headers == {'x-api-key': 'key123', 'Langsmith-Project': 'proj'} + assert config.include_content is True + assert config.no_metrics is True + + def test_header_value_keeps_base64_padding(self): + # Basic auth values end in '=' padding; split must be on the FIRST '=' + headers = parse_headers('Authorization=Basic cGstbGY6c2stbGY=,x-api-key=k') + assert headers == {'Authorization': 'Basic cGstbGY6c2stbGY=', 'x-api-key': 'k'} + + def test_header_parsing_edge_cases(self): + assert parse_headers(None) == {} + assert parse_headers('') == {} + assert parse_headers(' a = 1 , , no-equals , b=2, =nokey ') == {'a': '1', 'b': '2'} + + +# ========================================================================= +# BRIDGE: subscription, routing, resilience, shutdown +# ========================================================================= + + +class TestRunBridge: + async def test_subscribes_wildcard_and_routes_events(self): + client = FakeClient(connected=True) + task, stop_event, mapper, metrics = _start_bridge(client) + await _wait_until(lambda: client.monitor_calls) + + await client.emit('apaevt_task', {'action': 'begin', 'projectId': 'p', 'source': 's'}) + await client.emit('apaevt_flow', {'id': 0, 'op': 'begin', 'pipes': ['x'], 'component': 'x'}) + await client.emit('apaevt_sse', {'pipe_id': 1, 'type': 'thinking'}) + await client.emit('apaevt_status_update', {'state': 3}) + await client.emit('apaevt_unrelated', {'ignored': True}) + + stop_event.set() + assert await task == 0 + + # Wildcard token scope with the four monitor event types, exactly once + assert client.monitor_calls == [({'token': '*'}, list(MONITOR_TYPES))] + assert list(MONITOR_TYPES) == ['TASK', 'SUMMARY', 'FLOW', 'SSE'] + + # task/flow/sse -> span mapper; status -> metrics; unrelated dropped + assert [name for name, _ in mapper.events] == ['apaevt_task', 'apaevt_flow', 'apaevt_sse'] + assert metrics.statuses == [{'state': 3}] + assert mapper.closed is True + + async def test_startup_connect_failure_returns_2(self, capsys): + client = FakeClient(connected=False, connect_failures=99) + exit_code = await run_bridge(client, OtelConfig(), lambda: FakeMapper(), lambda: FakeMetrics()) + assert exit_code == 2 + assert client.connect_calls == 1 # no retry storm at startup + assert client.monitor_calls == [] + assert 'unable to connect' in capsys.readouterr().err + + async def test_subscription_failure_returns_2(self, capsys): + class FailingMonitorClient(FakeClient): + async def add_monitor(self, key, types): + raise RuntimeError('subscribe denied') + + client = FailingMonitorClient(connected=True) + exit_code = await run_bridge(client, OtelConfig(), lambda: FakeMapper(), lambda: FakeMetrics()) + assert exit_code == 2 + assert 'monitor subscription failed' in capsys.readouterr().err + + async def test_reconnects_with_backoff_without_resubscribing(self): + client = FakeClient(connected=True) + task, stop_event, mapper, metrics = _start_bridge(client, initial_backoff=0.01, max_backoff=0.02) + await _wait_until(lambda: client.monitor_calls) + + # Simulate a dropped connection whose first two reconnects fail + client._connected = False + client.connect_failures = 2 + await _wait_until(lambda: client.is_connected()) + + stop_event.set() + assert await task == 0 + + # Two failed attempts + one success + assert client.connect_calls == 3 + # The SDK owns resubscription (EventMixin replays monitors on + # reconnect); the bridge must not hand-roll a second add_monitor. + assert len(client.monitor_calls) == 1 + + async def test_shutdown_closes_spans_before_flushing_exporters(self): + order = [] + client = FakeClient(connected=True) + mapper = FakeMapper(order_log=order) + task, stop_event, _, _ = _start_bridge(client, mapper=mapper, shutdown_fn=lambda: order.append('shutdown')) + await _wait_until(lambda: client.monitor_calls) + + stop_event.set() + assert await task == 0 + assert order == ['close_all', 'shutdown'] + + async def test_no_metrics_never_builds_metrics_and_drops_status(self): + invoked = [] + client = FakeClient(connected=True) + mapper = FakeMapper() + + def metrics_factory(): + invoked.append(True) + return FakeMetrics() + + stop_event = asyncio.Event() + task = asyncio.ensure_future( + run_bridge( + client, + OtelConfig(no_metrics=True), + lambda: mapper, + metrics_factory, + stop_event=stop_event, + poll_interval=0.01, + ) + ) + await _wait_until(lambda: client.monitor_calls) + + await client.emit('apaevt_status_update', {'state': 3}) + await client.emit('apaevt_flow', {'id': 0, 'op': 'begin'}) + + stop_event.set() + assert await task == 0 + assert invoked == [] # metrics factory must not be called + assert [name for name, _ in mapper.events] == ['apaevt_flow'] + + async def test_mapper_exception_is_logged_not_fatal(self, capsys): + class ExplodingMapper(FakeMapper): + def handle_event(self, event_name, body): + super().handle_event(event_name, body) + raise ValueError('mapper boom') + + client = FakeClient(connected=True) + mapper = ExplodingMapper() + task, stop_event, _, metrics = _start_bridge(client, mapper=mapper) + await _wait_until(lambda: client.monitor_calls) + + # Both events are still delivered despite the first one raising + await client.emit('apaevt_flow', {'id': 0, 'op': 'begin'}) + await client.emit('apaevt_flow', {'id': 0, 'op': 'end'}) + await client.emit('apaevt_status_update', {'state': 3}) + + stop_event.set() + assert await task == 0 + assert len(mapper.events) == 2 + assert metrics.statuses == [{'state': 3}] + assert 'mapper boom' in capsys.readouterr().err + + async def test_chains_and_restores_previous_event_handler(self): + client = FakeClient(connected=True) + seen = [] + + async def previous_handler(message): + seen.append(message['event']) + + client._caller_on_event = previous_handler + task, stop_event, mapper, _ = _start_bridge(client) + await _wait_until(lambda: client.monitor_calls) + + await client.emit('apaevt_flow', {'id': 0, 'op': 'begin'}) + + stop_event.set() + assert await task == 0 + # The pre-existing handler kept receiving events while bridged... + assert seen == ['apaevt_flow'] + # ...and is restored once the bridge exits + assert client._caller_on_event is previous_handler + + async def test_fixture_replay_routes_all_recorded_events(self): + records = json.loads(FIXTURE_PATH.read_text(encoding='utf-8')) + assert len(records) == 24 + + client = FakeClient(connected=True) + task, stop_event, mapper, metrics = _start_bridge(client) + await _wait_until(lambda: client.monitor_calls) + + for record in records: + await client.emit(record['event'], record['body']) + + stop_event.set() + assert await task == 0 + + expected_span_events = [r['event'] for r in records if r['event'] != 'apaevt_status_update'] + expected_status_count = sum(1 for r in records if r['event'] == 'apaevt_status_update') + + assert [name for name, _ in mapper.events] == expected_span_events + assert len(metrics.statuses) == expected_status_count + assert len(mapper.events) + len(metrics.statuses) == 24 diff --git a/packages/client-python/tests/test_otel_mapper.py b/packages/client-python/tests/test_otel_mapper.py new file mode 100644 index 000000000..786a4cd09 --- /dev/null +++ b/packages/client-python/tests/test_otel_mapper.py @@ -0,0 +1,711 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Unit tests for rocketride.otelbridge.mapper (FlowSpanMapper / MetricsMapper). + +Pure in-memory tests: monitor event bodies (from the live-captured fixture +file and synthetic edge cases) are fed to the mappers and the resulting span +forest / metric points are asserted via the OpenTelemetry SDK's in-memory +exporters. Skipped gracefully when the optional 'otel' extra is absent (the +base CI matrix does not install it). +""" + +import json +from pathlib import Path + +import pytest + +pytest.importorskip('opentelemetry') +pytest.importorskip('opentelemetry.sdk') + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, StatusCode + +import rocketride.otelbridge.mapper as mapper_module +from rocketride.otelbridge.mapper import ( + ATTR_COMPONENT, + ATTR_FLOW_RESULT, + ATTR_LANE, + ATTR_PIPE_ID, + ATTR_PROJECT_ID, + ATTR_SOURCE, + ATTR_SPAN_IMPLICIT, + ATTR_SPAN_UNCLOSED, + ATTR_TASK_RESTARTED, + ATTR_UNMATCHED_LEAVES, + MAX_CONTENT_LENGTH, + FlowSpanMapper, + MetricsMapper, +) + +FIXTURE_PATH = Path(__file__).parent / 'fixtures' / 'otel_bridge_events.json' + +RUN_ID = '1b4bbac0.webhook_1' +PROJECT_ID = 'e612b741-748c-4b35-a8b7-186797a8ea42' +SOURCE = 'webhook_1' + +DEPRECATED_GENAI_ATTRS = ( + 'gen_ai.system', + 'gen_ai.usage.prompt_tokens', + 'gen_ai.usage.completion_tokens', + 'gen_ai.prompt', + 'gen_ai.completion', +) + + +# ========================================================================= +# HELPERS +# ========================================================================= + + +def make_mapper(**kwargs): + """Build a FlowSpanMapper wired to an InMemorySpanExporter.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + mapper = FlowSpanMapper(provider.get_tracer('test'), **kwargs) + return mapper, exporter + + +def make_metrics_mapper(): + """Build a MetricsMapper wired to an InMemoryMetricReader.""" + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + return MetricsMapper(provider.get_meter('test')), reader + + +def flow(op, component, pipe=0, trace=None, pipes=None, run_id=RUN_ID): + """Build an apaevt_flow body matching the captured wire shape.""" + return { + 'id': pipe, + 'op': op, + 'pipes': pipes if pipes is not None else [], + 'component': component, + 'trace': trace if trace is not None else {}, + 'project_id': PROJECT_ID, + 'source': SOURCE, + '__id': run_id, + } + + +def task(action, run_id=RUN_ID, **extra): + """Build an apaevt_task body (camelCase projectId per the wire).""" + body = {'action': action, 'projectId': PROJECT_ID, 'source': SOURCE, '__id': run_id} + body.update(extra) + return body + + +def spans_by_name(exporter, name): + return [span for span in exporter.get_finished_spans() if span.name == name] + + +def span_contains(span, needle): + """Scan every user-visible surface of a finished span for a payload string.""" + if needle in span.name: + return True + for key, value in (span.attributes or {}).items(): + if needle in str(key) or needle in str(value): + return True + for event in span.events: + if needle in event.name: + return True + for key, value in (event.attributes or {}).items(): + if needle in str(key) or needle in str(value): + return True + if span.status is not None and span.status.description and needle in span.status.description: + return True + return False + + +def any_span_contains(exporter, needle): + return any(span_contains(span, needle) for span in exporter.get_finished_spans()) + + +def collect_metrics(reader): + """ + Collect all metric data points ONCE, keyed by metric name. + + Gauge last-value aggregations are consumed by a collection cycle, so each + test must collect a single snapshot and assert against it. + """ + data = reader.get_metrics_data() + points = {} + if data is None: + return points + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + points.setdefault(metric.name, []).extend(metric.data.data_points) + return points + + +def metric_value(points, name): + assert points.get(name), f'no data points for metric {name!r}' + return points[name][-1].value + + +def load_fixture(): + return json.loads(FIXTURE_PATH.read_text()) + + +# ========================================================================= +# FLOW SPAN PAIRING +# ========================================================================= + + +def test_begin_enter_leave_end_cycle(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'response_1', trace={'data': None, 'lane': 'open'})) + mapper.handle_event( + 'apaevt_flow', flow('leave', 'response_1', trace={'data': None, 'lane': 'open', 'result': 'continue'}) + ) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt', trace={'name': 'probe.txt'})) + + spans = exporter.get_finished_spans() + assert [span.name for span in spans] == ['response_1', 'probe.txt'] + child, root = spans + assert root.parent is None + assert child.parent is not None and child.parent.span_id == root.context.span_id + assert child.attributes[ATTR_COMPONENT] == 'response_1' + assert child.attributes[ATTR_LANE] == 'open' + assert child.attributes[ATTR_FLOW_RESULT] == 'continue' + assert child.attributes[ATTR_PIPE_ID] == 0 + assert child.attributes[ATTR_PROJECT_ID] == PROJECT_ID + assert child.attributes[ATTR_SOURCE] == SOURCE + assert mapper.open_span_count() == 0 + + +def test_real_fixture_flow_cycle(): + """The verbatim wire-captured cycle: begin + one enter/leave per lane + end.""" + mapper, exporter = make_mapper() + for record in load_fixture(): + if record['event'] == 'apaevt_flow' and not record.get('_synthesized'): + mapper.handle_event('apaevt_flow', record['body']) + + spans = exporter.get_finished_spans() + assert len(spans) == 5 + roots = spans_by_name(exporter, 'probe.txt') + assert len(roots) == 1 + children = spans_by_name(exporter, 'response_1') + assert len(children) == 4 + assert {span.attributes[ATTR_LANE] for span in children} == {'open', 'text', 'closing', 'close'} + for child in children: + assert child.parent.span_id == roots[0].context.span_id + assert mapper.open_span_count() == 0 + + +def test_nested_enters_stack_parentage(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'obj.txt', pipes=['obj.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'outer_1', trace={'lane': 'text'})) + mapper.handle_event('apaevt_flow', flow('enter', 'inner_1', trace={'lane': 'text'})) + mapper.handle_event('apaevt_flow', flow('leave', 'inner_1', trace={'lane': 'text', 'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('leave', 'outer_1', trace={'lane': 'text', 'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'obj.txt')) + + inner = spans_by_name(exporter, 'inner_1')[0] + outer = spans_by_name(exporter, 'outer_1')[0] + root = spans_by_name(exporter, 'obj.txt')[0] + assert inner.parent.span_id == outer.context.span_id + assert outer.parent.span_id == root.context.span_id + + +def test_leave_pairs_by_component_identity_not_stack_position(): + """An out-of-order leave must close the span with the matching component.""" + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'obj.txt', pipes=['obj.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'comp_a_1')) + mapper.handle_event('apaevt_flow', flow('enter', 'comp_b_1')) + # comp_a leaves while comp_b is on top of the stack. + mapper.handle_event('apaevt_flow', flow('leave', 'comp_a_1', trace={'result': 'continue'})) + assert [span.name for span in exporter.get_finished_spans()] == ['comp_a_1'] + mapper.handle_event('apaevt_flow', flow('leave', 'comp_b_1', trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'obj.txt')) + + for span in exporter.get_finished_spans(): + assert ATTR_SPAN_UNCLOSED not in (span.attributes or {}) + root = spans_by_name(exporter, 'obj.txt')[0] + assert ATTR_UNMATCHED_LEAVES not in (root.attributes or {}) + + +def test_unknown_leave_is_tolerated_and_counted(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'obj.txt', pipes=['obj.txt'])) + mapper.handle_event('apaevt_flow', flow('leave', 'ghost_1', trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'obj.txt')) + + root = spans_by_name(exporter, 'obj.txt')[0] + assert root.attributes[ATTR_UNMATCHED_LEAVES] == 1 + + +def test_end_closes_open_components_as_unclosed(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'obj.txt', pipes=['obj.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'comp_a_1')) + mapper.handle_event('apaevt_flow', flow('end', 'obj.txt')) + + dangling = spans_by_name(exporter, 'comp_a_1')[0] + assert dangling.attributes[ATTR_SPAN_UNCLOSED] is True + assert dangling.status.status_code == StatusCode.UNSET + root = spans_by_name(exporter, 'obj.txt')[0] + assert ATTR_SPAN_UNCLOSED not in (root.attributes or {}) + + +def test_enter_without_begin_opens_implicit_root(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('enter', 'comp_a_1')) + mapper.handle_event('apaevt_flow', flow('leave', 'comp_a_1', trace={'result': 'continue'})) + mapper.close_all() + + root = spans_by_name(exporter, 'pipe 0')[0] + assert root.attributes[ATTR_SPAN_IMPLICIT] is True + child = spans_by_name(exporter, 'comp_a_1')[0] + assert child.parent.span_id == root.context.span_id + + +def test_begin_for_already_open_pipe_recycles_root(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'first.txt', pipes=['first.txt'])) + mapper.handle_event('apaevt_flow', flow('begin', 'second.txt', pipes=['second.txt'])) + mapper.close_all() + + first = spans_by_name(exporter, 'first.txt')[0] + assert first.attributes[ATTR_SPAN_UNCLOSED] is True + assert len(spans_by_name(exporter, 'second.txt')) == 1 + + +def test_error_leave_sets_error_status_and_exception_event(): + mapper, exporter = make_mapper() + error_text = 'OpenAI API error: 401 Unauthorized' + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'llm_openai_1', trace={'lane': 'questions'})) + mapper.handle_event( + 'apaevt_flow', flow('leave', 'llm_openai_1', trace={'data': None, 'lane': 'questions', 'error': error_text}) + ) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + span = spans_by_name(exporter, 'chat')[0] + assert span.status.status_code == StatusCode.ERROR + assert error_text in span.status.description + assert span.attributes['error.type'] == '_OTHER' + exception_events = [event for event in span.events if event.name == 'exception'] + assert exception_events and exception_events[0].attributes['exception.message'] == error_text + + +# ========================================================================= +# SSE EVENTS +# ========================================================================= + + +def test_sse_becomes_event_on_innermost_span(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'agent_rocketride_1')) + mapper.handle_event('apaevt_sse', {'pipe_id': 0, 'type': 'thinking', 'data': {'message': 'hmm'}, '__id': RUN_ID}) + mapper.handle_event('apaevt_flow', flow('leave', 'agent_rocketride_1', trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + agent_span = spans_by_name(exporter, 'invoke_agent')[0] + events = [event for event in agent_span.events if event.name == 'thinking'] + assert events and events[0].attributes['rocketride.sse.type'] == 'thinking' + root = spans_by_name(exporter, 'probe.txt')[0] + assert not [event for event in root.events if event.name == 'thinking'] + + +def test_sse_without_id_routes_to_sole_active_run(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + # No __id, 'data' key omitted entirely (wire truth): must not KeyError. + mapper.handle_event('apaevt_sse', {'pipe_id': 0, 'type': 'tool_call'}) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + root = spans_by_name(exporter, 'probe.txt')[0] + assert [event for event in root.events if event.name == 'tool_call'] + + +def test_sse_unroutable_is_dropped(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_sse', {'pipe_id': 0, 'type': 'thinking'}) # no runs at all + mapper.close_all() + assert exporter.get_finished_spans() == () + + +# ========================================================================= +# PRIVACY GATE +# ========================================================================= + +SENTINEL = 'SECRET_PAYLOAD_a4f1c2' + + +def _feed_content_events(mapper): + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event( + 'apaevt_flow', flow('enter', 'llm_openai_1', trace={'lane': 'text', 'data': {'text': SENTINEL}}) + ) + mapper.handle_event('apaevt_sse', {'pipe_id': 0, 'type': 'thinking', 'data': {'message': SENTINEL}, '__id': RUN_ID}) + mapper.handle_event( + 'apaevt_flow', + flow('leave', 'llm_openai_1', trace={'lane': 'text', 'data': {'text': SENTINEL}, 'result': 'continue'}), + ) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt', trace={'name': 'probe.txt', 'text': [SENTINEL]})) + + +def test_privacy_default_no_payload_reaches_spans(): + """Grep-provable default: without include_content no payload text is exported.""" + mapper, exporter = make_mapper() + _feed_content_events(mapper) + assert exporter.get_finished_spans() + assert not any_span_contains(exporter, SENTINEL) + + +def test_include_content_exposes_payloads(): + mapper, exporter = make_mapper(include_content=True) + _feed_content_events(mapper) + assert any_span_contains(exporter, SENTINEL) + + +def test_include_content_truncates_to_cap(): + mapper, exporter = make_mapper(include_content=True) + big = 'x' * (MAX_CONTENT_LENGTH * 2) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'comp_a_1', trace={'lane': 'text', 'data': {'text': big}})) + mapper.handle_event('apaevt_flow', flow('leave', 'comp_a_1', trace={'lane': 'text', 'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + span = spans_by_name(exporter, 'comp_a_1')[0] + assert len(span.attributes['rocketride.trace.data']) <= MAX_CONTENT_LENGTH + + +# ========================================================================= +# TASK LIFECYCLE +# ========================================================================= + + +def test_task_span_wraps_pipe_roots(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + mapper.handle_event('apaevt_task', task('end', name='demo.My webhook')) + + task_span = spans_by_name(exporter, 'task demo.My webhook')[0] + root = spans_by_name(exporter, 'probe.txt')[0] + assert root.parent.span_id == task_span.context.span_id + assert task_span.parent is None + assert task_span.attributes['rocketride.task.name'] == 'demo.My webhook' + assert mapper.open_span_count() == 0 + + +def test_running_snapshot_is_idempotent(): + """Reconnects re-seed the running snapshot; spans must not duplicate.""" + mapper, exporter = make_mapper() + running = { + 'action': 'running', + 'tasks': [{'id': RUN_ID, 'name': 'demo.My webhook', 'projectId': PROJECT_ID, 'source': SOURCE}], + '__id': '', + } + mapper.handle_event('apaevt_task', running) + mapper.handle_event('apaevt_task', running) + mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) + mapper.close_all() + + assert len(spans_by_name(exporter, 'task demo.My webhook')) == 1 + + +def test_task_restart_closes_and_reopens(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_task', task('restart', name='demo.My webhook')) + mapper.close_all() + + root = spans_by_name(exporter, 'probe.txt')[0] + assert root.attributes[ATTR_SPAN_UNCLOSED] is True + task_spans = spans_by_name(exporter, 'task demo.My webhook') + assert len(task_spans) == 2 + assert sum(1 for span in task_spans if (span.attributes or {}).get(ATTR_TASK_RESTARTED)) == 1 + + +def test_close_all_closes_everything_and_is_reentrant(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', name='demo')) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'comp_a_1')) + assert mapper.open_span_count() == 3 + mapper.close_all() + assert mapper.open_span_count() == 0 + spans = exporter.get_finished_spans() + assert len(spans) == 3 + assert all(span.end_time is not None for span in spans) + assert all((span.attributes or {}).get(ATTR_SPAN_UNCLOSED) for span in spans) + mapper.close_all() # re-entrant no-op + assert len(exporter.get_finished_spans()) == 3 + + +# ========================================================================= +# BOUNDED STATE (missed 'end' events must not leak open spans forever) +# ========================================================================= + + +def test_running_snapshot_reconciles_runs_with_missed_ends(): + """A seeded snapshot that no longer announces a tracked run closes it.""" + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + # Bridge reconnects; the run ended while disconnected, so the re-seeded + # snapshot only announces a different, still-running task. + mapper.handle_event( + 'apaevt_task', + { + 'action': 'running', + 'tasks': [{'id': 'aa11bb22.other_1', 'name': 'other', 'projectId': 'p-other', 'source': 'other_1'}], + '__id': '', + }, + ) + + stale_root = spans_by_name(exporter, 'probe.txt')[0] + assert stale_root.attributes[ATTR_SPAN_UNCLOSED] is True + stale_task = spans_by_name(exporter, 'task demo.My webhook')[0] + assert stale_task.attributes[ATTR_SPAN_UNCLOSED] is True + # Only the announced task remains tracked/open. + assert mapper.open_span_count() == 1 + mapper.close_all() + assert spans_by_name(exporter, 'task other') + + +def test_running_snapshot_keeps_announced_runs_open(): + """Reconciliation must not touch runs the snapshot still announces.""" + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) + mapper.handle_event( + 'apaevt_task', + { + 'action': 'running', + 'tasks': [{'id': RUN_ID, 'name': 'demo.My webhook', 'projectId': PROJECT_ID, 'source': SOURCE}], + '__id': '', + }, + ) + + assert exporter.get_finished_spans() == () + assert mapper.open_span_count() == 1 + + +def test_tracked_run_cap_evicts_least_recently_eventful_run(monkeypatch): + monkeypatch.setattr(mapper_module, 'MAX_TRACKED_RUNS', 2) + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', run_id='run1.s', name='one')) + mapper.handle_event('apaevt_task', task('begin', run_id='run2.s', name='two')) + # Touch run1 so run2 becomes the least recently eventful... + mapper.handle_event('apaevt_flow', flow('begin', 'obj.txt', pipes=['obj.txt'], run_id='run1.s')) + # ...then a third run must evict run2, closing its span as unclosed. + mapper.handle_event('apaevt_task', task('begin', run_id='run3.s', name='three')) + + evicted = spans_by_name(exporter, 'task two')[0] + assert evicted.attributes[ATTR_SPAN_UNCLOSED] is True + assert not spans_by_name(exporter, 'task one') + assert not spans_by_name(exporter, 'task three') + assert mapper.open_span_count() == 3 # run1 task + pipe root, run3 task + + +def test_metrics_last_counts_bounded_lru(monkeypatch): + monkeypatch.setattr(mapper_module, 'MAX_TRACKED_METRIC_RUNS', 2) + mapper, reader = make_metrics_mapper() + template = next(body for body in _status_records() if body['totalCount'] == 1) + for run in ('a.s', 'b.s', 'a.s', 'c.s'): # refresh 'a.s' before 'c.s' arrives + mapper.handle_status(dict(template, __id=run)) + + assert set(mapper._last_counts) == {'a.s', 'c.s'} # 'b.s' was the LRU entry + collect_metrics(reader) # drain so other tests see a clean reader + + +def test_genai_llm_component_attributes(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + mapper.handle_event('apaevt_flow', flow('enter', 'llm_openai_1', trace={'lane': 'questions'})) + mapper.handle_event('apaevt_flow', flow('leave', 'llm_openai_1', trace={'lane': 'questions', 'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + span = spans_by_name(exporter, 'chat')[0] + assert span.kind == SpanKind.CLIENT + assert span.attributes['gen_ai.operation.name'] == 'chat' + assert span.attributes['gen_ai.provider.name'] == 'openai' + assert span.attributes[ATTR_COMPONENT] == 'llm_openai_1' + for deprecated in DEPRECATED_GENAI_ATTRS: + assert deprecated not in span.attributes + + +def test_genai_unknown_provider_is_omitted_not_invented(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('enter', 'llm_ollama_1')) + mapper.close_all() + + span = spans_by_name(exporter, 'chat')[0] + assert span.attributes['gen_ai.operation.name'] == 'chat' + assert 'gen_ai.provider.name' not in span.attributes + + +def test_genai_tool_agent_and_embedding_components(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) + for component in ('tool_python_1', 'agent_rocketride_1', 'embedding_openai_1'): + mapper.handle_event('apaevt_flow', flow('enter', component)) + mapper.handle_event('apaevt_flow', flow('leave', component, trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + + tool = spans_by_name(exporter, 'execute_tool python')[0] + assert tool.kind == SpanKind.INTERNAL + assert tool.attributes['gen_ai.operation.name'] == 'execute_tool' + assert tool.attributes['gen_ai.tool.name'] == 'python' + + agent = spans_by_name(exporter, 'invoke_agent')[0] + assert agent.kind == SpanKind.INTERNAL + assert agent.attributes['gen_ai.operation.name'] == 'invoke_agent' + + embedding = spans_by_name(exporter, 'embeddings')[0] + assert embedding.kind == SpanKind.CLIENT + assert embedding.attributes['gen_ai.operation.name'] == 'embeddings' + assert embedding.attributes['gen_ai.provider.name'] == 'openai' + + +def test_plain_component_has_no_genai_attributes(): + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_flow', flow('enter', 'response_1')) + mapper.close_all() + + span = spans_by_name(exporter, 'response_1')[0] + assert span.kind == SpanKind.INTERNAL + assert not [key for key in span.attributes if key.startswith('gen_ai.')] + + +# ========================================================================= +# METRICS +# ========================================================================= + + +def _status_records(): + return [record['body'] for record in load_fixture() if record['event'] == 'apaevt_status_update'] + + +def test_metrics_from_status_fixture(): + mapper, reader = make_metrics_mapper() + counted = next(body for body in _status_records() if body['totalCount'] == 1) + mapper.handle_status(counted) + + points = collect_metrics(reader) + assert metric_value(points, 'rocketride.objects.total') == 1 + assert metric_value(points, 'rocketride.objects.completed') == 1 + assert metric_value(points, 'rocketride.objects.failed') == 0 + assert metric_value(points, 'rocketride.memory.cpu_mb') == pytest.approx(301.2265625) + assert metric_value(points, 'rocketride.cpu.percent.peak') == pytest.approx(61.45) + point = points['rocketride.objects.total'][-1] + assert point.attributes['rocketride.project_id'] == PROJECT_ID + assert point.attributes['rocketride.source'] == SOURCE + + +def test_metrics_counts_use_deltas_not_resums(): + mapper, reader = make_metrics_mapper() + counted = next(body for body in _status_records() if body['totalCount'] == 1) + mapper.handle_status(counted) + mapper.handle_status(counted) # identical snapshot: cumulative sum must not double + assert metric_value(collect_metrics(reader), 'rocketride.objects.total') == 1 + + grown = dict(counted, totalCount=3, completedCount=2, failedCount=1) + mapper.handle_status(grown) + points = collect_metrics(reader) + assert metric_value(points, 'rocketride.objects.total') == 3 + assert metric_value(points, 'rocketride.objects.completed') == 2 + assert metric_value(points, 'rocketride.objects.failed') == 1 + + +def test_metrics_all_zero_snapshot_still_emits_gauges(): + """Wire truth: metrics can legitimately be all zero; gauges must still export.""" + mapper, reader = make_metrics_mapper() + zero = next(body for body in _status_records() if body['state'] == 1) + mapper.handle_status(zero) + + points = collect_metrics(reader) + for name in ( + 'rocketride.rate.count', + 'rocketride.rate.size', + 'rocketride.cpu.percent', + 'rocketride.memory.cpu_mb', + 'rocketride.memory.gpu_mb', + ): + assert metric_value(points, name) == 0.0 + + +# ========================================================================= +# FULL FIXTURE REPLAY +# ========================================================================= + + +def test_full_fixture_replay_produces_coherent_span_forest(): + """Feed all 24 captured/synthesized records end to end; no crashes, all spans closed.""" + mapper, exporter = make_mapper() + metrics_mapper, reader = make_metrics_mapper() + + for record in load_fixture(): + if record['event'] == 'apaevt_status_update': + metrics_mapper.handle_status(record['body']) + else: + mapper.handle_event(record['event'], record['body']) + mapper.close_all() + + spans = exporter.get_finished_spans() + assert mapper.open_span_count() == 0 + assert all(span.end_time is not None for span in spans) + assert len(spans) == 10 + + # First run: task span wrapping the probe.txt pipe root and 4 lane spans. + task_spans = [span for span in spans if span.name.startswith('task ')] + assert len(task_spans) == 3 # begin/end run, seeded running task, post-restart task + root = spans_by_name(exporter, 'probe.txt')[0] + assert root.parent is not None + assert len(spans_by_name(exporter, 'response_1')) == 4 + + # Post-task-end SSE + flow re-attach on an implicit root; error maps to status. + implicit = spans_by_name(exporter, 'pipe 0')[0] + assert implicit.attributes[ATTR_SPAN_IMPLICIT] is True + assert [event for event in implicit.events if event.name == 'thinking'] + chat = spans_by_name(exporter, 'chat')[0] + assert chat.status.status_code == StatusCode.ERROR + assert chat.parent.span_id == implicit.context.span_id + + # Privacy default: real result payload and SSE text never exported. + assert not any_span_contains(exporter, 'hello otel bridge') + assert not any_span_contains(exporter, 'Analyzing your request') + + # No deprecated GenAI attribute anywhere. + for span in spans: + for deprecated in DEPRECATED_GENAI_ATTRS: + assert deprecated not in (span.attributes or {}) + + # Metrics ingested from the same replay. + points = collect_metrics(reader) + assert metric_value(points, 'rocketride.objects.total') == 1 + assert metric_value(points, 'rocketride.memory.cpu_mb.peak') == pytest.approx(322.24609375) diff --git a/packages/client-python/tests/test_otel_setup.py b/packages/client-python/tests/test_otel_setup.py new file mode 100644 index 000000000..39b3c5062 --- /dev/null +++ b/packages/client-python/tests/test_otel_setup.py @@ -0,0 +1,263 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Unit tests for rocketride.otelbridge.setup (provider/exporter construction). + +Covers OTLP endpoint semantics (base URL + per-signal path appending, env var +fallbacks), header pass-through, provider construction, the lazy-import +guarantee (module import never requires the 'otel' extra, verified in a +subprocess with opentelemetry blocked), and the gRPC-extra error path. +Skipped gracefully when the optional 'otel' extra is absent. +""" + +import os +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Optional + +import pytest + +pytest.importorskip('opentelemetry') +pytest.importorskip('opentelemetry.sdk') + +from rocketride.otelbridge.setup import ( + OtelNotInstalledError, + _build_metric_exporter, + _build_span_exporter, + _resolve_endpoint, + build_providers, + missing_otel_message, +) + +SRC_DIR = Path(__file__).parent.parent / 'src' + +try: + import opentelemetry.exporter.otlp.proto.grpc # noqa: F401 + + HAS_GRPC_EXPORTER = True +except ImportError: + HAS_GRPC_EXPORTER = False + + +@dataclass +class BridgeConfigStub: + """Minimal stand-in matching the OtelConfig attribute surface.""" + + endpoint: Optional[str] = None + protocol: str = 'http' + service_name: str = 'rocketride-engine' + include_content: bool = False + no_metrics: bool = False + headers: Dict[str, str] = field(default_factory=dict) + + +# ========================================================================= +# ENDPOINT SEMANTICS +# ========================================================================= + + +def test_resolve_endpoint_appends_signal_path(): + assert _resolve_endpoint('http://localhost:4318', 'v1/traces') == 'http://localhost:4318/v1/traces' + + +def test_resolve_endpoint_tolerates_trailing_slash(): + assert _resolve_endpoint('http://localhost:4318/', 'v1/traces') == 'http://localhost:4318/v1/traces' + + +def test_resolve_endpoint_keeps_full_signal_path_verbatim(): + full = 'http://localhost:4318/v1/traces' + assert _resolve_endpoint(full, 'v1/traces') == full + + +def test_resolve_endpoint_supports_vendor_base_paths(): + """Langfuse/LangSmith users paste base ingest URLs; the signal path is appended.""" + assert ( + _resolve_endpoint('https://cloud.langfuse.com/api/public/otel', 'v1/traces') + == 'https://cloud.langfuse.com/api/public/otel/v1/traces' + ) + assert ( + _resolve_endpoint('https://api.smith.langchain.com/otel', 'v1/traces') + == 'https://api.smith.langchain.com/otel/v1/traces' + ) + + +def test_span_exporter_uses_config_endpoint_and_headers(): + config = BridgeConfigStub(endpoint='http://collector:4318', headers={'x-api-key': 'k1'}) + exporter = _build_span_exporter(config) + assert exporter._endpoint == 'http://collector:4318/v1/traces' + assert dict(exporter._headers)['x-api-key'] == 'k1' + + +def test_metric_exporter_uses_config_endpoint(): + config = BridgeConfigStub(endpoint='http://collector:4318') + exporter = _build_metric_exporter(config) + assert exporter._endpoint == 'http://collector:4318/v1/metrics' + + +def test_env_endpoint_honored_when_config_endpoint_absent(monkeypatch): + """OTEL_EXPORTER_OTLP_ENDPOINT is a base URL; the exporter appends the signal path.""" + monkeypatch.setenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://envhost:4318') + monkeypatch.delenv('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', raising=False) + exporter = _build_span_exporter(BridgeConfigStub(endpoint=None)) + assert exporter._endpoint == 'http://envhost:4318/v1/traces' + + +def test_signal_specific_env_endpoint_honored_when_config_endpoint_absent(monkeypatch): + """The standard signal-specific env var is used verbatim by the SDK exporter.""" + monkeypatch.delenv('OTEL_EXPORTER_OTLP_ENDPOINT', raising=False) + monkeypatch.setenv('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', 'http://traces.example:4318/v1/traces') + exporter = _build_span_exporter(BridgeConfigStub(endpoint=None)) + assert exporter._endpoint == 'http://traces.example:4318/v1/traces' + + +def test_env_headers_honored_when_config_headers_absent(monkeypatch): + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=abc123') + exporter = _build_span_exporter(BridgeConfigStub(endpoint='http://collector:4318')) + assert dict(exporter._headers).get('x-api-key') == 'abc123' + + +# ========================================================================= +# PROVIDER CONSTRUCTION +# ========================================================================= + + +def test_build_providers_returns_usable_tracer_meter_shutdown(): + config = BridgeConfigStub(endpoint='http://localhost:4318', no_metrics=True) + tracer, meter, shutdown = build_providers(config) + assert callable(shutdown) + assert hasattr(tracer, 'start_span') + # The meter must accept the instruments MetricsMapper creates. + meter.create_up_down_counter('test.updown', unit='{object}') + meter.create_gauge('test.gauge', unit='%') + # no_metrics=True and no ended spans: shutdown flushes nothing and returns. + shutdown() + + +def test_build_providers_sets_service_name_resource(): + config = BridgeConfigStub(endpoint='http://localhost:4318', service_name='my-bridge', no_metrics=True) + tracer, _meter, shutdown = build_providers(config) + resource = tracer.resource # SDK tracer exposes its provider resource + assert resource.attributes['service.name'] == 'my-bridge' + shutdown() + + +def test_build_providers_with_metrics_wires_a_periodic_reader(monkeypatch): + """With no_metrics=False a periodic reader exports recorded measurements at shutdown.""" + import io + + from opentelemetry.sdk.metrics.export import ConsoleMetricExporter + + import rocketride.otelbridge.setup as setup_module + + buffer = io.StringIO() + monkeypatch.setattr(setup_module, '_build_metric_exporter', lambda config: ConsoleMetricExporter(out=buffer)) + config = BridgeConfigStub(endpoint='http://localhost:4318', no_metrics=False) + _tracer, meter, shutdown = build_providers(config) + counter = meter.create_up_down_counter('test.counter', unit='{object}') + counter.add(1, {'k': 'v'}) + shutdown() + assert 'test.counter' in buffer.getvalue() + + +# ========================================================================= +# MISSING-DEPENDENCY PATHS +# ========================================================================= + + +def test_missing_otel_message_names_the_extra(): + assert "pip install 'rocketride[otel]'" in missing_otel_message() + + +@pytest.mark.skipif(HAS_GRPC_EXPORTER, reason='OTLP gRPC exporter is installed') +def test_grpc_protocol_without_grpc_exporter_raises_install_hint(): + config = BridgeConfigStub(endpoint='http://localhost:4317', protocol='grpc') + with pytest.raises(OtelNotInstalledError, match='grpc'): + _build_span_exporter(config) + with pytest.raises(OtelNotInstalledError, match='grpc'): + _build_metric_exporter(config) + + +_IMPORT_SAFETY_SCRIPT = """ +import sys +from importlib.abc import MetaPathFinder + + +class _Blocker(MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'opentelemetry' or fullname.startswith('opentelemetry.'): + raise ImportError('opentelemetry blocked for import-safety test') + return None + + +sys.meta_path.insert(0, _Blocker()) +for name in list(sys.modules): + if name.startswith('opentelemetry'): + del sys.modules[name] + +import rocketride.otelbridge # must import without the 'otel' extra +from rocketride.otelbridge.mapper import FlowSpanMapper +from rocketride.otelbridge.setup import OtelNotInstalledError, build_providers + + +class Cfg: + endpoint = None + protocol = 'http' + service_name = 's' + include_content = False + no_metrics = True + headers = {} + + +try: + build_providers(Cfg()) +except OtelNotInstalledError as exc: + assert "rocketride[otel]" in str(exc), str(exc) +else: + raise SystemExit('build_providers did not raise OtelNotInstalledError') + +try: + FlowSpanMapper(None) +except OtelNotInstalledError: + pass +else: + raise SystemExit('FlowSpanMapper did not raise OtelNotInstalledError') + +print('IMPORT_SAFE_OK') +""" + + +def test_module_import_is_safe_without_otel_extra(): + """The otelbridge package must import (and fail helpfully) without the extra.""" + env = dict(os.environ) + env['PYTHONPATH'] = str(SRC_DIR) + result = subprocess.run( + [sys.executable, '-c', _IMPORT_SAFETY_SCRIPT], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + assert result.returncode == 0, f'stdout={result.stdout!r} stderr={result.stderr!r}' + assert 'IMPORT_SAFE_OK' in result.stdout From dd78d46403d83c5d60431051745f02b4856f6ac2 Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:47:02 +0530 Subject: [PATCH 2/6] feat(cli): add 'rocketride otel' subcommand + optional [otel] extra (#1609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rocketride otel [--endpoint URL] [--protocol http|grpc] [--service-name] [--headers k=v,...] [--include-content] [--no-metrics] + the standard connection args. Subscribes via rrext_monitor with token '*' (the observability doc's recommended ingestion scope, types TASK|SUMMARY| FLOW|SSE), dispatches to the mappers, reconnects with capped backoff and resubscribes (subscriptions are per-connection and not durable), and flushes exporters on SIGINT/SIGTERM. Endpoint resolution honors the standard OTLP env contract: explicit --endpoint > OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (verbatim) > OTEL_EXPORTER_OTLP_ENDPOINT (base URL + /v1/traces|/v1/metrics) > default. Packaging: opentelemetry-sdk + OTLP http exporter live behind an optional extra — the base install gains zero dependencies, and running the subcommand without the extra exits 2 with the exact "pip install 'rocketride[otel]'" hint before any connection attempt. 25 tests: config precedence matrix, dispatch/reconnect/resubscribe with a fake client, shutdown flush ordering, missing-extra exit path. Co-Authored-By: Claude Fable 5 --- packages/client-python/pyproject.toml | 5 + .../src/rocketride/cli/commands/__init__.py | 3 + .../src/rocketride/cli/commands/otel.py | 220 +++++++++++ .../client-python/src/rocketride/cli/main.py | 67 ++++ packages/client-python/tests/test_otel_cli.py | 368 ++++++++++++++++++ 5 files changed, 663 insertions(+) create mode 100644 packages/client-python/src/rocketride/cli/commands/otel.py create mode 100644 packages/client-python/tests/test_otel_cli.py diff --git a/packages/client-python/pyproject.toml b/packages/client-python/pyproject.toml index c2a31a2fe..06ae6ced9 100644 --- a/packages/client-python/pyproject.toml +++ b/packages/client-python/pyproject.toml @@ -61,6 +61,11 @@ test = [ "python-dotenv>=1.0.0", ] +otel = [ + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-http>=1.27.0", +] + [project.urls] Homepage = "https://github.com/rocketride-ai/rocketride-server" Documentation = "https://github.com/rocketride-ai/rocketride-server#readme" diff --git a/packages/client-python/src/rocketride/cli/commands/__init__.py b/packages/client-python/src/rocketride/cli/commands/__init__.py index 7edf47528..0b7c7f50c 100644 --- a/packages/client-python/src/rocketride/cli/commands/__init__.py +++ b/packages/client-python/src/rocketride/cli/commands/__init__.py @@ -34,6 +34,7 @@ EventsCommand: Monitor real-time pipeline events ListCommand: List all active tasks StoreCommand: Project and template storage operations + OtelCommand: Export pipeline traces and metrics over OpenTelemetry """ from .start import StartCommand @@ -43,6 +44,7 @@ from .events import EventsCommand from .list import ListCommand from .store import StoreCommand +from .otel import OtelCommand __all__ = [ 'StartCommand', @@ -52,4 +54,5 @@ 'EventsCommand', 'ListCommand', 'StoreCommand', + 'OtelCommand', ] diff --git a/packages/client-python/src/rocketride/cli/commands/otel.py b/packages/client-python/src/rocketride/cli/commands/otel.py new file mode 100644 index 000000000..fdded5f4b --- /dev/null +++ b/packages/client-python/src/rocketride/cli/commands/otel.py @@ -0,0 +1,220 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +RocketRide CLI OpenTelemetry Bridge Command Implementation. + +This module provides the OtelCommand class for exporting live pipeline traces +and metrics to any OpenTelemetry collector over OTLP through the RocketRide +CLI. Use this command to observe pipeline executions in backends such as +Jaeger, Grafana Tempo, Datadog, Langfuse, or LangSmith without any engine or +server changes: the bridge is a pure consumer of the engine's documented +WebSocket monitor protocol. + +The command subscribes to the TASK, SUMMARY, FLOW, and SSE monitor event +types with the wildcard token scope and maps them to OTel spans (pipeline +flow) and metrics (task status snapshots), exporting both over the same OTLP +endpoint. + +Trace level: the bridge can only observe runs; it cannot change the trace +level of a run it did not start. Pipeline FLOW spans appear only for runs +started with the ``pipelineTraceLevel`` execute argument, e.g. +``client.use(pipelineTraceLevel='summary')``. Without it, the bridge still +exports task lifecycle spans and status metrics. + +Key Features: + - OTLP export of pipeline flow spans and task status metrics + - Wildcard monitor subscription covering all tasks for the API key + - Standard OTEL_EXPORTER_OTLP_ENDPOINT/_HEADERS/OTEL_SERVICE_NAME support + (with no endpoint configured at all, the exporters' own env semantics + apply, including OTEL_EXPORTER_OTLP_TRACES/METRICS_ENDPOINT) + - Privacy by default: payload content excluded unless --include-content + - Automatic reconnection with capped exponential backoff + - Graceful shutdown on SIGINT/SIGTERM (spans closed, exporters flushed) + +Usage: + rocketride otel --apikey + rocketride otel --endpoint http://localhost:4318 --service-name my-engine + rocketride otel --headers 'Authorization=Basic ' --no-metrics + rocketride otel --include-content --apikey + +Requires the optional OpenTelemetry dependencies: + pip install 'rocketride[otel]' + +Components: + OtelCommand: Main command implementation for the OpenTelemetry bridge +""" + +import importlib.util +import sys +from typing import TYPE_CHECKING + +from .base import BaseCommand +from ...otelbridge.bridge import run_bridge +from ...otelbridge.config import OtelConfig + +# Safe without the 'otel' extra: setup.py keeps all opentelemetry imports lazy. +from ...otelbridge.setup import OtelNotInstalledError + +if TYPE_CHECKING: + from ..main import RocketRideClient + +# Exact install hint for the missing-extra error path (contract: exit code 2) +OTEL_INSTALL_HINT = "pip install 'rocketride[otel]'" + + +def _otel_available() -> bool: + """ + Check whether the optional OpenTelemetry SDK is installed. + + Probes for the ``opentelemetry`` and ``opentelemetry.sdk`` modules + without importing them, so the check itself never pays import cost and + never fails on a partially installed distribution. + + Returns: + bool: True when the 'rocketride[otel]' extra appears importable. + """ + try: + return ( + importlib.util.find_spec('opentelemetry') is not None + and importlib.util.find_spec('opentelemetry.sdk') is not None + ) + except (ImportError, ValueError): + return False + + +class OtelCommand(BaseCommand): + """ + Command implementation for the OpenTelemetry bridge. + + Exports live pipeline traces and metrics over OTLP by consuming the + engine's WebSocket monitor protocol. Runs until interrupted (Ctrl+C or + SIGTERM), closing open spans and flushing exporters on shutdown. + + Example: + ```python + # Initialize and execute the OTel bridge + command = OtelCommand(cli, args) + exit_code = await command.execute(client) + ``` + + Key Features: + - Lazy dependency guard: clear exit code 2 with install hint when + the 'rocketride[otel]' extra is missing (checked before connecting) + - Configuration precedence: CLI args > OTEL_* env vars > defaults + - Traces and metrics over one OTLP endpoint (http/protobuf default) + - Content privacy gate via --include-content + """ + + def __init__(self, cli, args): + """ + Initialize OtelCommand with CLI context and parsed arguments. + + Args: + cli: CLI instance providing cancellation state and event handling + args: Parsed command line arguments containing exporter options + and connection configuration + """ + super().__init__(cli, args) + + async def execute(self, client: 'RocketRideClient') -> int: + """ + Execute the OpenTelemetry bridge command. + + Verifies the optional OpenTelemetry dependencies are installed, + resolves the exporter configuration, and runs the bridge loop until + interrupted. + + Args: + client: RocketRideClient instance for server communication + (connected by the bridge if not already connected) + + Returns: + Exit code: 0 for graceful shutdown, 1 for unexpected errors, + 2 when dependencies are missing or the startup connection fails + + Process Flow: + 1. Guard: missing 'rocketride[otel]' extra -> exit 2 with hint + (before any connection attempt) + 2. Resolve OtelConfig (CLI args > OTEL_* env vars > defaults) + 3. Print the effective bridge configuration + 4. Run the bridge loop (connect, subscribe, dispatch, reconnect) + 5. Graceful shutdown on SIGINT/SIGTERM: spans closed, exporters + flushed, exit 0 + """ + # Dependency guard MUST run before any connection attempt + if not _otel_available(): + print( + f"Error: 'rocketride otel' requires the OpenTelemetry extra. Install it with: {OTEL_INSTALL_HINT}", + file=sys.stderr, + ) + return 2 + + # Resolve configuration: CLI args > OTEL_* env vars > defaults + config = OtelConfig.from_args_env(self.args) + + # --trace-level is documentation-surface only: the monitor protocol + # has no way to change the trace level of runs the bridge didn't start + trace_level = getattr(self.args, 'trace_level', None) + if trace_level: + print( + f'Note: --trace-level={trace_level} is informational only. The bridge cannot change the ' + f'trace level of runs it did not start; start runs with ' + f"pipelineTraceLevel='{trace_level}' (e.g. client.use(pipelineTraceLevel='{trace_level}')) " + f'to emit FLOW events at that level.' + ) + + # Show the effective configuration before entering the run loop. + # With no endpoint configured the OTLP exporters resolve their own: + # OTEL_EXPORTER_OTLP_TRACES/METRICS_ENDPOINT, else the SDK default. + if config.endpoint: + endpoint_display = config.endpoint + else: + sdk_default = 'localhost:4317' if config.protocol == 'grpc' else 'http://localhost:4318' + endpoint_display = f'exporter default (OTEL_EXPORTER_OTLP_*_ENDPOINT or {sdk_default})' + print(f'OpenTelemetry bridge starting (endpoint: {endpoint_display}, protocol: {config.protocol})') + print(f' service.name: {config.service_name}') + print(f' metrics: {"disabled" if config.no_metrics else "enabled"}') + print(f' payload content: {"included (size-capped)" if config.include_content else "excluded"}') + print( + " FLOW spans require runs started with pipelineTraceLevel (e.g. client.use(pipelineTraceLevel='summary'))" + ) + + try: + # Run the bridge until stopped (returns 0) or startup fails (2) + return await run_bridge(client, config) + + except (ImportError, OtelNotInstalledError) as e: + # Missing optional dependency surfaced after the guard, e.g. + # --protocol grpc without the OTLP gRPC exporter package + # (build_providers raises OtelNotInstalledError for that path). + print(f'Error: {e}', file=sys.stderr) + return 2 + + except KeyboardInterrupt: + # Fallback when signal handlers could not be installed + print('\nStopping OpenTelemetry bridge...') + return 0 + + except Exception as e: + print(f'Error: OpenTelemetry bridge failed: {e}', file=sys.stderr) + return 1 diff --git a/packages/client-python/src/rocketride/cli/main.py b/packages/client-python/src/rocketride/cli/main.py index daff72ed6..75ebf4d93 100644 --- a/packages/client-python/src/rocketride/cli/main.py +++ b/packages/client-python/src/rocketride/cli/main.py @@ -67,6 +67,7 @@ from .commands.events import EventsCommand from .commands.list import ListCommand from .commands.store import StoreCommand +from .commands.otel import OtelCommand try: # Try importing from installed package first @@ -327,6 +328,7 @@ def setup_parser(self) -> argparse.ArgumentParser: - status: Monitor task execution status - stop: Terminate running task - events: Monitor task events with filtering + - otel: Export pipeline traces and metrics over OpenTelemetry """ # Create main parser with description and help text parser = argparse.ArgumentParser( @@ -452,6 +454,70 @@ def add_common_args(subparser): help='Optional log file to write all events (e.g., --log=events.log)', ) + # Otel command - exports pipeline traces and metrics over OTLP + otel_parser = subparsers.add_parser( + 'otel', + help='Export pipeline traces and metrics to an OpenTelemetry collector', + ) + add_common_args(otel_parser) + + # OTLP endpoint base URL (signal paths are appended automatically) + otel_parser.add_argument( + '--endpoint', + default=None, + help='OTLP endpoint base URL (default: OTEL_EXPORTER_OTLP_ENDPOINT or http://localhost:4318)', + ) + + # OTLP transport protocol + otel_parser.add_argument( + '--protocol', + choices=['http', 'grpc'], + default='http', + help='OTLP transport protocol (default: %(default)s)', + ) + + # service.name resource attribute + otel_parser.add_argument( + '--service-name', + dest='service_name', + default=None, + help='service.name resource attribute (default: OTEL_SERVICE_NAME or rocketride-engine)', + ) + + # OTLP headers for collector authentication (Langfuse, LangSmith, ...) + otel_parser.add_argument( + '--headers', + default=None, + help='OTLP headers as comma-separated key=value pairs (default: OTEL_EXPORTER_OTLP_HEADERS)', + ) + + # Privacy gate: payload content is excluded from spans by default + otel_parser.add_argument( + '--include-content', + dest='include_content', + action='store_true', + help='Include pipeline payload content in spans, size-capped (default: content excluded)', + ) + + # Metrics toggle: traces-only export + otel_parser.add_argument( + '--no-metrics', + dest='no_metrics', + action='store_true', + help='Disable metrics export; export traces only', + ) + + # Informational only: the bridge cannot set the trace level of runs + # it did not start (pipelineTraceLevel is an execute-time argument) + otel_parser.add_argument( + '--trace-level', + dest='trace_level', + choices=['none', 'metadata', 'summary', 'full'], + default=None, + help='Informational only: FLOW spans require runs started with pipelineTraceLevel; ' + 'the bridge cannot change the trace level of running tasks', + ) + # List command - lists all active tasks list_parser = subparsers.add_parser('list', help='List all active tasks') add_common_args(list_parser) @@ -605,6 +671,7 @@ async def run(self) -> int: 'events': EventsCommand, 'list': ListCommand, 'store': StoreCommand, + 'otel': OtelCommand, } if self.args.command in command_map: diff --git a/packages/client-python/tests/test_otel_cli.py b/packages/client-python/tests/test_otel_cli.py new file mode 100644 index 000000000..5eec3d44c --- /dev/null +++ b/packages/client-python/tests/test_otel_cli.py @@ -0,0 +1,368 @@ +# MIT License +# +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Unit tests for the 'rocketride otel' CLI command. + +These tests exercise argument parsing, command registration, the +missing-extra guard (exit code 2 with the install hint, before any +connection attempt), and the wiring from parsed args through OtelConfig to +run_bridge. The RocketRide client and the bridge loop are faked, so no +server and no 'rocketride[otel]' extra are required. +""" + +import importlib +import importlib.util +import signal +import sys +from types import SimpleNamespace + +import pytest + +from rocketride.cli.commands import OtelCommand +from rocketride.cli.commands import otel as otel_module + +# Importable without the 'otel' extra: setup.py keeps all otel imports lazy. +from rocketride.otelbridge.setup import OtelNotInstalledError + +HAS_OTEL_SDK = ( + importlib.util.find_spec('opentelemetry') is not None and importlib.util.find_spec('opentelemetry.sdk') is not None +) +try: + HAS_GRPC_EXPORTER = importlib.util.find_spec('opentelemetry.exporter.otlp.proto.grpc') is not None +except (ImportError, ValueError): + HAS_GRPC_EXPORTER = False + +# The rocketride.cli package re-exports the main() FUNCTION under the name +# 'main', shadowing the main module; import the module explicitly. +cli_main = importlib.import_module('rocketride.cli.main') + + +class FakeCli: + """Minimal CLI context for constructing commands directly.""" + + def __init__(self): + self.uri = 'http://localhost:5565' + + def is_cancelled(self): + return True + + +class FakeClient: + """Connection recorder; the guard must exit before any connect call.""" + + def __init__(self): + self.connect_calls = 0 + self._caller_on_event = None + + def is_connected(self): + return False + + async def connect(self): + self.connect_calls += 1 + + async def add_monitor(self, key, types): + pass + + async def disconnect(self): + pass + + +def make_args(**overrides): + """Build a parsed-args namespace with the otel command's attributes.""" + base = dict( + command='otel', + uri='http://localhost:5565', + apikey='', + token=None, + endpoint=None, + protocol='http', + service_name=None, + headers=None, + include_content=False, + no_metrics=False, + trace_level=None, + ) + base.update(overrides) + return SimpleNamespace(**base) + + +@pytest.fixture +def cli_parser(): + """RocketRideCLI parser, restoring the SIGINT handler it replaces.""" + previous = signal.getsignal(signal.SIGINT) + try: + yield cli_main.RocketRideCLI().setup_parser() + finally: + signal.signal(signal.SIGINT, previous) + + +# ========================================================================= +# REGISTRATION AND ARGUMENT PARSING +# ========================================================================= + + +class TestOtelParser: + def test_command_is_exported(self): + from rocketride.cli import commands + + assert 'OtelCommand' in commands.__all__ + assert commands.OtelCommand is OtelCommand + + def test_defaults(self, cli_parser): + args = cli_parser.parse_args(['otel']) + assert args.command == 'otel' + assert args.endpoint is None + assert args.protocol == 'http' + assert args.service_name is None + assert args.headers is None + assert args.include_content is False + assert args.no_metrics is False + assert args.trace_level is None + + def test_all_flags_parse(self, cli_parser): + args = cli_parser.parse_args( + [ + 'otel', + '--endpoint', + 'http://collector:4318', + '--protocol', + 'grpc', + '--service-name', + 'my-engine', + '--headers', + 'x-api-key=abc,Langsmith-Project=proj', + '--include-content', + '--no-metrics', + '--trace-level', + 'summary', + '--uri', + 'http://server:5565', + '--apikey', + 'KEY', + ] + ) + assert args.endpoint == 'http://collector:4318' + assert args.protocol == 'grpc' + assert args.service_name == 'my-engine' + assert args.headers == 'x-api-key=abc,Langsmith-Project=proj' + assert args.include_content is True + assert args.no_metrics is True + assert args.trace_level == 'summary' + assert args.uri == 'http://server:5565' + assert args.apikey == 'KEY' + + def test_invalid_protocol_rejected(self, cli_parser): + with pytest.raises(SystemExit): + cli_parser.parse_args(['otel', '--protocol', 'carrier-pigeon']) + + def test_invalid_trace_level_rejected(self, cli_parser): + with pytest.raises(SystemExit): + cli_parser.parse_args(['otel', '--trace-level', 'verbose']) + + +# ========================================================================= +# MISSING-EXTRA GUARD +# ========================================================================= + + +class TestMissingExtraGuard: + async def test_exits_2_with_hint_before_connecting(self, monkeypatch, capsys): + monkeypatch.setattr(otel_module, '_otel_available', lambda: False) + client = FakeClient() + + exit_code = await OtelCommand(FakeCli(), make_args()).execute(client) + + assert exit_code == 2 + err = capsys.readouterr().err + assert "pip install 'rocketride[otel]'" in err + # Guard must fire before any connection attempt + assert client.connect_calls == 0 + + def test_otel_available_reflects_find_spec(self, monkeypatch): + import importlib.util + + monkeypatch.setattr(importlib.util, 'find_spec', lambda name: None) + assert otel_module._otel_available() is False + + def test_otel_available_handles_import_machinery_errors(self, monkeypatch): + import importlib.util + + def raising_find_spec(name): + raise ModuleNotFoundError(name) + + monkeypatch.setattr(importlib.util, 'find_spec', raising_find_spec) + assert otel_module._otel_available() is False + + +# ========================================================================= +# EXECUTION WIRING +# ========================================================================= + + +class TestOtelExecute: + async def test_passes_config_built_from_args_to_run_bridge(self, monkeypatch, capsys): + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + recorded = {} + + async def fake_run_bridge(client, config): + recorded['client'] = client + recorded['config'] = config + return 0 + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + client = FakeClient() + args = make_args( + endpoint='http://collector:4318', + service_name='my-engine', + headers='x-api-key=abc', + include_content=True, + no_metrics=True, + ) + + exit_code = await OtelCommand(FakeCli(), args).execute(client) + + assert exit_code == 0 + assert recorded['client'] is client + config = recorded['config'] + assert config.endpoint == 'http://collector:4318' + assert config.service_name == 'my-engine' + assert config.headers == {'x-api-key': 'abc'} + assert config.include_content is True + assert config.no_metrics is True + out = capsys.readouterr().out + assert 'OpenTelemetry bridge starting' in out + + async def test_trace_level_prints_informational_note(self, monkeypatch, capsys): + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + + async def fake_run_bridge(client, config): + return 0 + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + + await OtelCommand(FakeCli(), make_args(trace_level='summary')).execute(FakeClient()) + + out = capsys.readouterr().out + assert 'informational only' in out + assert 'pipelineTraceLevel' in out + + @pytest.mark.parametrize('exc_type', [OtelNotInstalledError, ImportError]) + async def test_missing_dependency_from_bridge_exits_2(self, monkeypatch, capsys, exc_type): + # OtelNotInstalledError is what build_providers actually raises for + # --protocol grpc without opentelemetry-exporter-otlp-proto-grpc; + # a raw ImportError from a lazy import must map the same way. + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + + async def fake_run_bridge(client, config): + raise exc_type('--protocol grpc requires: pip install opentelemetry-exporter-otlp-proto-grpc') + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + + exit_code = await OtelCommand(FakeCli(), make_args(protocol='grpc')).execute(FakeClient()) + + assert exit_code == 2 + assert 'opentelemetry-exporter-otlp-proto-grpc' in capsys.readouterr().err + + @pytest.mark.skipif(not HAS_OTEL_SDK, reason="requires the 'otel' extra") + @pytest.mark.skipif(HAS_GRPC_EXPORTER, reason='OTLP gRPC exporter is installed') + async def test_real_grpc_protocol_without_exporter_exits_2(self, capsys): + # Nothing faked: the real run_bridge -> build_providers path must + # surface the missing gRPC exporter as exit code 2 with the hint. + exit_code = await OtelCommand(FakeCli(), make_args(protocol='grpc')).execute(FakeClient()) + + assert exit_code == 2 + assert 'opentelemetry-exporter-otlp-proto-grpc' in capsys.readouterr().err + + async def test_unexpected_error_exits_1(self, monkeypatch, capsys): + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + + async def fake_run_bridge(client, config): + raise RuntimeError('exporter blew up') + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + + exit_code = await OtelCommand(FakeCli(), make_args()).execute(FakeClient()) + + assert exit_code == 1 + assert 'exporter blew up' in capsys.readouterr().err + + async def test_bridge_exit_code_propagates(self, monkeypatch): + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + + async def fake_run_bridge(client, config): + return 2 + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + + assert await OtelCommand(FakeCli(), make_args()).execute(FakeClient()) == 2 + + +# ========================================================================= +# END-TO-END CLI ROUTING (parser -> validation -> command_map -> execute) +# ========================================================================= + + +class TestCliRouting: + async def test_run_routes_otel_without_requiring_token(self, monkeypatch): + # 'otel' must not trip the token validation applied to status/stop/events + monkeypatch.delenv('ROCKETRIDE_TOKEN', raising=False) + monkeypatch.setattr(sys, 'argv', ['rocketride', 'otel']) + + created = {} + + class FakeRRClient: + def __init__(self, **kwargs): + created.update(kwargs) + + async def disconnect(self): + pass + + async def fake_execute(self, client): + return 7 + + monkeypatch.setattr(cli_main, 'RocketRideClient', FakeRRClient) + monkeypatch.setattr(cli_main.OtelCommand, 'execute', fake_execute) + + previous = signal.getsignal(signal.SIGINT) + try: + exit_code = await cli_main.RocketRideCLI().run() + finally: + signal.signal(signal.SIGINT, previous) + + # The command executed (returning our sentinel), so no token error path + assert exit_code == 7 + assert created['uri'] + + async def test_events_still_requires_token_but_otel_does_not(self, monkeypatch, capsys): + # Guard against the otel command being ensnared by the events check + monkeypatch.delenv('ROCKETRIDE_TOKEN', raising=False) + monkeypatch.setattr(sys, 'argv', ['rocketride', 'events', 'ALL']) + + previous = signal.getsignal(signal.SIGINT) + try: + exit_code = await cli_main.RocketRideCLI().run() + finally: + signal.signal(signal.SIGINT, previous) + + assert exit_code == 1 + assert 'Token is required' in capsys.readouterr().out From 8c9bc3fba064f6cfe23258fb3df5259853e0ce14 Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:47:02 +0530 Subject: [PATCH 3/6] docs: OpenTelemetry bridge guide with Jaeger, Langfuse, and Datadog recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setup guide covering the span model (task -> pipe -> component tree), the privacy default and --include-content, the FLOW-requires-trace-level callout, reconnect semantics, and copy-paste recipes with exact OTLP endpoints/auth for Jaeger all-in-one, Langfuse (/api/public/otel, Basic auth), and Datadog. Notes that Jaeger does not ingest OTLP metrics — pair it with --no-metrics or point metrics at a collector. Co-Authored-By: Claude Fable 5 --- docs/README-clients.md | 2 + docs/README-otel-bridge.md | 353 +++++++++++++++++++++++++++ docs/README-python-client.md | 18 ++ packages/docs/content-static/cli.mdx | 52 +++- 4 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 docs/README-otel-bridge.md diff --git a/docs/README-clients.md b/docs/README-clients.md index 3e1e6c671..031ef9d56 100644 --- a/docs/README-clients.md +++ b/docs/README-clients.md @@ -26,6 +26,8 @@ URIs: clients accept `http`/`https` or `ws`/`wss` and convert to WebSocket (`htt Each document lists every constructor option, method, type, and usage example for that client. +The Python client also ships the OpenTelemetry bridge (`rocketride otel`), which exports live pipeline traces and metrics over OTLP — see [README-otel-bridge.md](README-otel-bridge.md). + --- ## Installation diff --git a/docs/README-otel-bridge.md b/docs/README-otel-bridge.md new file mode 100644 index 000000000..76aa814f7 --- /dev/null +++ b/docs/README-otel-bridge.md @@ -0,0 +1,353 @@ +# OpenTelemetry Bridge (`rocketride otel`) + +The OpenTelemetry bridge exports **live pipeline traces and metrics** from a running +RocketRide engine to any OpenTelemetry collector over OTLP — Jaeger, Grafana, Datadog, +Langfuse, LangSmith, or anything else that ingests OTLP. It ships with the Python client +as the `rocketride otel` CLI command and requires **zero engine or server changes**: the +bridge is a pure consumer of the engine's documented +[WebSocket monitor protocol](../packages/server/docs/observability.md), subscribing to the +`TASK`, `SUMMARY`, `FLOW`, and `SSE` event streams with the wildcard token scope (the +documented scope for an ingestion service) and translating them into OTel spans and +metrics on the fly. Point it at your engine and your collector, and every pipeline run +visible to your API key shows up as a trace. + +```text +RocketRide engine ──(WebSocket monitor events)──▶ rocketride otel ──(OTLP)──▶ your backend +``` + +- [Install](#install) +- [Quickstart: Jaeger end-to-end](#quickstart-jaeger-end-to-end) +- [CLI reference](#cli-reference) +- [Backend recipes](#backend-recipes) +- [The span model](#the-span-model) +- [Attributes](#attributes) +- [Privacy: content is excluded by default](#privacy-content-is-excluded-by-default) +- [Metrics](#metrics) +- [Reconnection and shutdown](#reconnection-and-shutdown) +- [Troubleshooting](#troubleshooting) +- [Limitations](#limitations) + +--- + +## Install + +The bridge's OpenTelemetry dependencies are an optional extra — the base `rocketride` +package gains no new dependencies: + +```bash +pip install 'rocketride[otel]' +``` + +Running `rocketride otel` without the extra exits with code 2 and prints the install +command above. + +## Quickstart: Jaeger end-to-end + +Run Jaeger all-in-one (UI + OTLP receivers, in-memory storage): + +```bash +docker run --rm -d --name jaeger \ + -p 16686:16686 -p 4317:4317 -p 4318:4318 \ + jaegertracing/all-in-one:1.76.0 +``` + +Start the bridge — the defaults already point at `http://localhost:4318` (OTLP over +HTTP/protobuf): + +```bash +export ROCKETRIDE_URI=ws://localhost:5565 # or wss://api.rocketride.ai + ROCKETRIDE_APIKEY +rocketride otel +``` + +Now start a pipeline run **with a trace level** so it emits `FLOW` events (see +[the callout below](#flow-spans-need-a-trace-level) — this is the step everyone misses): + +```python +import asyncio +from rocketride import RocketRideClient + +async def main(): + # Uses ROCKETRIDE_URI / ROCKETRIDE_APIKEY from the environment + async with RocketRideClient() as client: + result = await client.use(filepath='pipeline.pipe', pipelineTraceLevel='summary') + token = result['token'] + await client.send(token, 'hello traces', objinfo={'name': 'input.txt'}, mimetype='text/plain') + await client.terminate(token) + +asyncio.run(main()) +``` + +Open [http://localhost:16686](http://localhost:16686), select the **rocketride-engine** +service, and click **Find Traces**. You'll see one trace per run: a task root span, a pipe +span per object, and component child spans underneath. + +## CLI reference + +```bash +rocketride otel [--endpoint URL] [--protocol http|grpc] [--service-name NAME] + [--headers k=v,k=v] [--include-content] [--no-metrics] + [--trace-level none|metadata|summary|full] +``` + +| Flag | Default | Description | +| ------------------- | --------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `--endpoint ` | `OTEL_EXPORTER_OTLP_ENDPOINT`, else the exporter default (`http://localhost:4318` for http, `localhost:4317` for grpc) | OTLP **base** URL. The signal paths `/v1/traces` / `/v1/metrics` are appended automatically unless already present, so pasting Langfuse's or LangSmith's ingest URL just works. With neither the flag nor `OTEL_EXPORTER_OTLP_ENDPOINT` set, the signal-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` variables are honored too. | +| `--protocol

` | `http` | OTLP transport: `http` (http/protobuf) or `grpc`. The gRPC exporter is **not** part of the `otel` extra — see [Troubleshooting](#troubleshooting). | +| `--service-name `| `OTEL_SERVICE_NAME` or `rocketride-engine` | The `service.name` resource attribute your backend groups by. | +| `--headers ` | `OTEL_EXPORTER_OTLP_HEADERS` | Comma-separated `key=value` pairs sent with every OTLP request (collector auth). Values are split on the *first* `=` only, so base64 padding survives. | +| `--include-content` | off | Include pipeline payload content in spans, truncated to 8 KB per attribute. **By default no payload text reaches any span.** | +| `--no-metrics` | off | Export traces only; task status snapshots are not mapped to metrics. | +| `--trace-level ` | — | **Informational only.** The bridge cannot change the trace level of runs it did not start; this flag just prints a reminder of how to start traced runs. | + +Plus the standard connection arguments shared by all subcommands: `--uri` +(`ROCKETRIDE_URI`) and `--apikey` (`ROCKETRIDE_APIKEY`). No task token is needed — the +bridge subscribes to every task your API key owns. + +**Configuration precedence:** explicit CLI flags > standard `OTEL_*` environment +variables (`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, +`OTEL_SERVICE_NAME`) > built-in defaults. + +**Exit codes:** `0` graceful shutdown (Ctrl+C / SIGTERM), `1` unexpected runtime error, +`2` missing dependency (the `otel` extra, or the gRPC exporter with `--protocol grpc`) +or startup connection/subscribe failure. + +### FLOW spans need a trace level + +`apaevt_flow` events — and therefore per-component spans — fire **only for runs started +with a `pipelineTraceLevel`** (an argument of the `execute` request, i.e. +`client.use(..., pipelineTraceLevel='summary')`). The bridge can only observe; it cannot +turn tracing on for a run it did not start. Without a trace level you still get task +lifecycle spans and all metrics, just no component breakdown. `summary` is the practical +level: lane writes and final results without per-call noise. + +## Backend recipes + +### Jaeger / any OTLP collector + +Covered by the [quickstart](#quickstart-jaeger-end-to-end): OTLP/HTTP on port 4318 +(or `--protocol grpc` against 4317 with the gRPC exporter installed). The same shape works +for the OpenTelemetry Collector, Grafana stacks, and any other standard OTLP receiver. + +### Langfuse + +Langfuse ingests OTLP **traces over HTTP only** (no gRPC, no metrics) at +`/api/public/otel`, with HTTP Basic auth built from your project keys: + +```bash +# Build the Basic auth value from your Langfuse project keys (pk-lf-... : sk-lf-...) +export LANGFUSE_AUTH=$(echo -n 'pk-lf-your-public-key:sk-lf-your-secret-key' | base64) + +rocketride otel \ + --endpoint https://cloud.langfuse.com/api/public/otel \ + --headers "Authorization=Basic ${LANGFUSE_AUTH}" \ + --no-metrics +``` + +Use `https://us.cloud.langfuse.com/api/public/otel` for the US region, or +`https:///api/public/otel` for self-hosted (Langfuse v3.22.0+). Pass +`--no-metrics`: Langfuse's OTLP ingest is traces-only. Spans from LLM components carry +`gen_ai.*` attributes, which Langfuse maps into its own data model. + +### LangSmith + +LangSmith ingests OTLP traces over HTTP at `/otel`, authenticated with an `x-api-key` +header (optional `Langsmith-Project` header to pick the project): + +```bash +rocketride otel \ + --endpoint https://api.smith.langchain.com/otel \ + --headers 'x-api-key=your-langsmith-api-key,Langsmith-Project=your-project-name' \ + --no-metrics +``` + +Regional hosts: `eu.api.smith.langchain.com` (EU); self-hosted follows +`https:///api/v1/otel`. + +### Datadog + +Send OTLP to a Datadog Agent (or an OpenTelemetry Collector with the Datadog exporter) +that has OTLP ingestion enabled — the agent handles the Datadog API key, so the bridge +needs no auth headers: + +```bash +rocketride otel --endpoint http://localhost:4318 +``` + +Datadog natively maps `gen_ai.*` semantic-convention attributes (semconv v1.37+) in its +LLM Observability product, so LLM component spans light up without a Datadog SDK. + +## The span model + +One trace per pipeline run. The hierarchy the bridge builds: + +```text +task task root span (INTERNAL), one per run +│ opened on apaevt_task begin (or the seeded +│ "running" snapshot when attaching mid-run) +│ +└── pipe root span, one per (run, pipe id) segment + │ opened on flow op=begin, closed on op=end; + │ named after the object flowing through + │ + ├── chat component span (llm_openai_1) + │ │ SpanKind CLIENT, gen_ai.operation.name=chat, + │ │ gen_ai.provider.name=openai + │ └─ ● thinking apaevt_sse events attach as span events to the + │ innermost open span of their pipe + │ + └── response_1 plain component span (INTERNAL) + opened on op=enter, closed on op=leave — + paired by component identity, one span per + lane write (including control lanes) +``` + +Runs are correlated primarily on the wire correlation id (`__id`, +`.`), falling back to `(project_id, source)`. Details worth +knowing: + +- **enter/leave pairing is by component identity**, never stack position (the monitor + protocol documents why). Expect one short component span per lane write — including + the `open`/`closing`/`close` control lanes. +- **Component spans still open at run end or bridge shutdown** are closed with status + `UNSET` and `rocketride.span.unclosed=true` — an honest "we never saw the leave", not + an error. +- **Attaching mid-run:** events for pipes the bridge never saw begin get an implicit + root span marked `rocketride.span.implicit=true`. +- **Errors:** a `trace.error` on any flow event sets span status `ERROR`, records an + `exception` span event, and sets `error.type` (`_OTHER` — the wire error is a + free-form string). +- **Task restarts** close the current spans and open a fresh task span with + `rocketride.task.restarted=true`. +- **GenAI conventions:** component ids map to GenAI semantic conventions where the id + makes the role unambiguous — `llm_*` → `chat` (CLIENT), `embedding_*` → `embeddings` + (CLIENT), `agent_*` → `invoke_agent` (INTERNAL), `tool_*` → `execute_tool` (INTERNAL, + with `gen_ai.tool.name`). `gen_ai.provider.name` is set only for providers on the + semconv well-known list (e.g. `llm_openai_*` → `openai`, `llm_anthropic_*` → + `anthropic`); unknown providers omit the attribute rather than inventing a value. + +## Attributes + +| Attribute | On | Meaning | +| -------------------------------- | --------------------- | ------------------------------------------------------------------- | +| `rocketride.project_id` | all spans and metrics | Pipeline project id | +| `rocketride.source` | all spans and metrics | Pipeline source (e.g. `webhook_1`) | +| `rocketride.run_id` | all spans | Wire correlation id of the run (`.`) | +| `rocketride.task.name` | task spans | Task name from the lifecycle event | +| `rocketride.task.restarted` | task spans | `true` when this span was opened by a task restart | +| `rocketride.pipe_id` | pipe/component spans | Pipe index within the pipeline | +| `rocketride.object` | pipe spans | Name of the object flowing through this segment | +| `rocketride.component` | component spans | Component id (e.g. `llm_openai_1`) | +| `rocketride.lane` | component spans | Lane being written (e.g. `text`, `open`, `close`) | +| `rocketride.flow.result` | component spans | Flow result string on leave (e.g. `continue`) | +| `rocketride.span.unclosed` | any span | `true`: closed at run end/shutdown without a matching leave | +| `rocketride.span.implicit` | pipe spans | `true`: created for a pipe whose begin the bridge never saw | +| `rocketride.flow.unmatched_leaves` | pipe spans | Count of leave events that matched no open component span | +| `rocketride.sse.type` | span events | SSE message type (e.g. `thinking`, `tool_call`) | +| `gen_ai.operation.name` | LLM/agent/tool spans | `chat`, `embeddings`, `invoke_agent`, or `execute_tool` | +| `gen_ai.provider.name` | LLM/embedding spans | Well-known provider value derived from the component id | +| `gen_ai.tool.name` | tool spans | Tool name derived from the component id | +| `error.type` | failed spans | Always `_OTHER` (wire errors are free-form strings) | +| `rocketride.trace.data` / `rocketride.result` / `rocketride.sse.data` | content-gated | Payload content — **only with `--include-content`**, 8 KB cap | + +`gen_ai.*` names follow the July 2026 snapshot of the +`open-telemetry/semantic-conventions-genai` registry (Development stability, no tagged +release); deprecated names such as `gen_ai.system` or `gen_ai.usage.prompt_tokens` are +never emitted. + +## Privacy: content is excluded by default + +By default **no pipeline payload content reaches any span** — not lane data +(`trace.data`), not run-segment results, not SSE message bodies. Only structural +metadata (names, ids, lanes, counts, timings) is exported, so the bridge is safe to +point at a shared collector out of the box. + +Opting in with `--include-content` copies payload content into the +`rocketride.trace.data`, `rocketride.result`, and `rocketride.sse.data` attributes, +JSON-serialized and truncated to **8192 characters** per attribute. Treat the flag as +what it is: pipeline inputs and outputs flowing into your telemetry backend. + +## Metrics + +Unless `--no-metrics` is set, every `apaevt_status_update` snapshot (roughly every 500 ms +per running task) is mapped to OTel metrics, exported over the same OTLP endpoint. All +instruments carry `rocketride.project_id` / `rocketride.source` attributes. + +| Instrument | Type | Unit | Meaning | +| -------------------------------- | --------------- | ------------ | ---------------------------------------- | +| `rocketride.objects.total` | up-down counter | `{object}` | Objects seen by the pipeline run | +| `rocketride.objects.completed` | up-down counter | `{object}` | Objects completed | +| `rocketride.objects.failed` | up-down counter | `{object}` | Objects failed | +| `rocketride.rate.count` | gauge | `{object}/s` | Instantaneous object processing rate | +| `rocketride.rate.size` | gauge | `By/s` | Instantaneous byte processing rate | +| `rocketride.cpu.percent` | gauge | `%` | Engine CPU utilization | +| `rocketride.cpu.percent.peak` | gauge | `%` | Peak engine CPU utilization | +| `rocketride.memory.cpu_mb` | gauge | `MBy` | Engine CPU memory usage | +| `rocketride.memory.cpu_mb.peak` | gauge | `MBy` | Peak engine CPU memory usage | +| `rocketride.memory.gpu_mb` | gauge | `MBy` | Engine GPU memory usage | +| `rocketride.memory.gpu_mb.peak` | gauge | `MBy` | Peak engine GPU memory usage | + +Object counters are fed **per-run deltas** between snapshots, so re-sent snapshots don't +double-count; a task restart resets the engine's counts, which legitimately produces +negative deltas. The snapshot's `tokens.*` block is **compute credits (billing), not LLM +tokens**, and is deliberately not exported as `gen_ai.usage.*`. + +## Reconnection and shutdown + +- **Reconnects** use capped exponential backoff (1 s doubling up to 30 s). Monitor + subscriptions are per-connection and not durable, but the SDK replays them on every + reconnect, and the re-seeded "running"/status snapshots are handled idempotently — + already-open spans are not duplicated. The seeded snapshot is also authoritative: + tracked runs it no longer announces (their `end` was missed while disconnected) are + closed with `rocketride.span.unclosed=true` and dropped. +- **Ctrl+C / SIGTERM** closes all open spans (marked `rocketride.span.unclosed=true`), + flushes both exporters, and exits `0`. +- **Startup failure** (engine unreachable, subscribe rejected) prints a clean message to + stderr and exits `2` so supervisors can tell "never started" from "was stopped". + +## Troubleshooting + +| Symptom | Cause / fix | +| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| Exits immediately with code 2 and an install hint | The `otel` extra is missing: `pip install 'rocketride[otel]'`. | +| Bridge runs, tasks appear, but **no component spans** | The run was not started with a trace level. Start runs with `client.use(..., pipelineTraceLevel='summary')` — the bridge cannot enable it for you. | +| Exits with code 2, "connection" in the message | Engine unreachable at startup: check `--uri` / `ROCKETRIDE_URI` and `--apikey` / `ROCKETRIDE_APIKEY`. | +| Bridge runs but nothing arrives in the backend | Exporter can't reach the collector (exports fail in the background; the bridge keeps running). Check host/port — OTLP/HTTP is **4318**, gRPC is **4317** — and that `--protocol` matches the receiver. | +| `--protocol grpc` fails with an install error | The gRPC exporter is not part of the extra: `pip install opentelemetry-exporter-otlp-proto-grpc`. Note Langfuse does not accept gRPC at all. | +| Langfuse/LangSmith receive traces but metric exports error | Their OTLP ingest is traces-only — run with `--no-metrics`. | +| Spans named `chat` carry no model name or token counts | Expected — see [Limitations](#limitations). | + +## Limitations + +Honest edges of a protocol-level bridge: + +- **Timestamps are arrival time.** The monitor wire protocol carries no event + timestamps, so span start/end use the bridge's clock at event arrival. Durations are + bridge-observed (WebSocket latency included), not engine-measured. +- **LLM token usage and model names appear only when nodes surface them in events.** + Flow events at `summary` level carry neither, so `gen_ai.request.model` and + `gen_ai.usage.*` are honestly omitted rather than guessed, and LLM span names degrade + to the bare operation (`chat`, not `chat gpt-4.1`). The status snapshot's `tokens.*` + are compute credits, not LLM tokens, and are never mapped to `gen_ai.usage.*`. +- **`gen_ai.*` conventions are Development stability.** Attribute names follow the July + 2026 snapshot of `open-telemetry/semantic-conventions-genai`; they are centralized in + one constants module and may be revised as the spec evolves. +- **Trace level is the run starter's choice.** `--trace-level` on the bridge is + informational only; there is no protocol surface to change it for running tasks. +- **Restart accounting.** Object up-down counters are per-run deltas, so a task restart + (counts reset) produces a negative step by design. +- **Spans export when they close.** The batch span processor exports a span only at + `end()`, so a run whose `end` event is never observed holds its spans back until the + bridge closes them — at the next reconnect's seeded snapshot (runs no longer + announced are closed), when the tracked-run cap (1024) evicts the + least-recently-eventful run, or at shutdown. Such spans are flagged + `rocketride.span.unclosed=true`. There is deliberately no idle timeout: a quiet but + alive task (e.g. a webhook service) keeps being announced and is never expired by a + clock. Metric delta bookkeeping is likewise capped (4096 runs, least recently + updated evicted first), so a bridge left running for weeks has bounded memory. + +## See also + +- [Monitor protocol reference](../packages/server/docs/observability.md) — the event + stream the bridge consumes +- [Python client](README-python-client.md) — SDK and the rest of the CLI +- [Client libraries overview](README-clients.md) diff --git a/docs/README-python-client.md b/docs/README-python-client.md index 37f79bf7a..563617537 100644 --- a/docs/README-python-client.md +++ b/docs/README-python-client.md @@ -554,11 +554,29 @@ rocketride status --token # Monitor task progress rocketride stop --token # Terminate a running task rocketride list # List all active tasks rocketride events ALL --token # Stream task events +rocketride otel # Export pipeline traces + metrics over OpenTelemetry rocketride rrext_store get_all_projects # List stored projects ``` All commands accept `--uri` and `--apikey` flags, or read from environment variables. +### OpenTelemetry export (`rocketride otel`) + +The `otel` command bridges live pipeline traces and metrics to any OpenTelemetry +collector over OTLP (Jaeger, Grafana, Datadog, Langfuse, LangSmith, ...). It consumes the +engine's documented WebSocket monitor protocol — no engine changes — and requires the +optional extra: + +```bash +pip install 'rocketride[otel]' +rocketride otel --endpoint http://localhost:4318 +``` + +Payload content is excluded from spans by default (`--include-content` opts in, +size-capped). Per-component spans appear for runs started with +`client.use(..., pipelineTraceLevel='summary')`. Full guide: +[README-otel-bridge.md](https://github.com/rocketride-org/rocketride-server/blob/develop/docs/README-otel-bridge.md). + ## Configuration | Variable | Description | diff --git a/packages/docs/content-static/cli.mdx b/packages/docs/content-static/cli.mdx index 2d5d24efc..bee1bd6bd 100644 --- a/packages/docs/content-static/cli.mdx +++ b/packages/docs/content-static/cli.mdx @@ -14,9 +14,9 @@ installing either package puts `rocketride` on your path. > **Python vs TypeScript CLI:** Both packages ship a `rocketride` command. The > core commands (`start`, `upload`, `status`, `stop`, `store`) are available in -> both, but flag names differ in a few places and the Python CLI includes two -> additional commands (`events`, `list`) not present in TypeScript. Differences -> are called out inline below. +> both, but flag names differ in a few places and the Python CLI includes three +> additional commands (`events`, `list`, `otel`) not present in TypeScript. +> Differences are called out inline below. ## Install @@ -140,6 +140,7 @@ connection is encrypted. | `stop` | Stop a running task. | | `events` | Stream all raw events from a running task. Python CLI only. | | `list` | List all active tasks. Python CLI only. | +| `otel` | Export live pipeline traces and metrics to an OpenTelemetry collector. Python CLI only. | | `store` | File-store operations: `dir`, `type`, `write`, `rm`, `mkdir`, `stat`. | ### start @@ -267,6 +268,51 @@ Key flags: > Available in the Python CLI only. +### otel + +Use `otel` to export live pipeline traces and metrics to any OpenTelemetry +collector over OTLP — Jaeger, Grafana, Datadog, Langfuse, LangSmith, or anything +else that speaks OTLP. The command is a pure consumer of the engine's +[monitor event stream](/protocols/websocket/observability): it subscribes to all +tasks your API key owns and maps lifecycle, flow, and status events to OTel +spans and metrics. It runs until interrupted (Ctrl+C), closing open spans and +flushing exporters on shutdown. + +```bash +# Requires the optional OpenTelemetry extra +pip install 'rocketride[otel]' + +# Export to a local collector (default endpoint http://localhost:4318) +rocketride otel + +# Export to a specific backend with auth headers +rocketride otel --endpoint https://cloud.langfuse.com/api/public/otel \ + --headers "Authorization=Basic ${AUTH_VALUE}" --no-metrics + +# Include pipeline payload content in spans (excluded by default, 8 KB cap) +rocketride otel --include-content +``` + +Key flags: + +| Flag | Description | +| --- | --- | +| `--endpoint ` | OTLP base URL; `/v1/traces` and `/v1/metrics` are appended automatically. Defaults to `OTEL_EXPORTER_OTLP_ENDPOINT`, honoring the signal-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` variables when neither is set, else the exporter default (`http://localhost:4318` for http, `localhost:4317` for grpc). | +| `--protocol ` | OTLP transport. Default `http` (http/protobuf); `grpc` needs the separate gRPC exporter package. | +| `--service-name ` | `service.name` resource attribute. Defaults to `OTEL_SERVICE_NAME` or `rocketride-engine`. | +| `--headers ` | Headers sent with every OTLP request (collector auth). Defaults to `OTEL_EXPORTER_OTLP_HEADERS`. | +| `--include-content` | Include pipeline payload content in spans, size-capped. By default no payload content is exported. | +| `--no-metrics` | Export traces only. | + +Per-component flow spans appear only for runs started with a +`pipelineTraceLevel` (e.g. `client.use(..., pipelineTraceLevel='summary')`) — +the bridge cannot change the trace level of runs it did not start. Task +lifecycle spans and metrics are exported regardless. See the +[Observability](/protocols/websocket/observability) page for the underlying +event stream. + +> Available in the Python CLI only. Requires `pip install 'rocketride[otel]'`. + ### store Use `store` to inspect or write files in the engine's built-in file store. This From 704090cb1351d8d5236550ca99c79720202e142f Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:06:39 +0530 Subject: [PATCH 4/6] =?UTF-8?q?fix(sdk):=20handle=20partial=20OpenTelemetr?= =?UTF-8?q?y=20installs=20=E2=80=94=20guard=20the=20HTTP=20exporter=20impo?= =?UTF-8?q?rt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI exposed a real environment our matrix missed: opentelemetry-api/sdk present as another package's transitive dependency (tool_daytona pulls them in) while the OTLP HTTP exporter is absent. Two fixes: - setup.py: the http trace/metric exporter imports are now guarded like the grpc ones — a partial install raises OtelNotInstalledError with the pip install 'rocketride[otel]' hint (CLI exit 2) instead of a raw ModuleNotFoundError. Verified live in a reproduced partial venv. - test_otel_setup.py: importorskip targets the most specific module the tests exercise (opentelemetry.exporter.otlp.proto.http), so the module skips cleanly in CI's partial environment; plus a regression test that simulates the partial install via a None sys.modules entry and asserts the friendly error. Partial venv: 63 passed + 1 module skip; full extra: 79 passed. Co-Authored-By: Claude Fable 5 --- .../src/rocketride/otelbridge/setup.py | 13 +++++++++++-- packages/client-python/tests/test_otel_setup.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/client-python/src/rocketride/otelbridge/setup.py b/packages/client-python/src/rocketride/otelbridge/setup.py index b5340e52e..3801ac689 100644 --- a/packages/client-python/src/rocketride/otelbridge/setup.py +++ b/packages/client-python/src/rocketride/otelbridge/setup.py @@ -111,7 +111,13 @@ def _build_span_exporter(config: Any) -> Any: return OTLPSpanExporter(endpoint=config.endpoint, headers=headers) return OTLPSpanExporter(headers=headers) - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + except ImportError as exc: + # A partial install (opentelemetry-api/sdk present as someone else's + # transitive dependency, exporter absent) must give the same friendly + # hint as a missing extra, not a raw ModuleNotFoundError. + raise OtelNotInstalledError(_INSTALL_HINT) from exc if config.endpoint: return OTLPSpanExporter(endpoint=_resolve_endpoint(config.endpoint, 'v1/traces'), headers=headers) @@ -133,7 +139,10 @@ def _build_metric_exporter(config: Any) -> Any: return OTLPMetricExporter(endpoint=config.endpoint, headers=headers) return OTLPMetricExporter(headers=headers) - from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + try: + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + except ImportError as exc: + raise OtelNotInstalledError(_INSTALL_HINT) from exc if config.endpoint: return OTLPMetricExporter(endpoint=_resolve_endpoint(config.endpoint, 'v1/metrics'), headers=headers) diff --git a/packages/client-python/tests/test_otel_setup.py b/packages/client-python/tests/test_otel_setup.py index 39b3c5062..f2cb481cb 100644 --- a/packages/client-python/tests/test_otel_setup.py +++ b/packages/client-python/tests/test_otel_setup.py @@ -41,6 +41,10 @@ pytest.importorskip('opentelemetry') pytest.importorskip('opentelemetry.sdk') +# CI environments can carry opentelemetry-api/sdk as another package's +# transitive dependency WITHOUT the OTLP exporter; these tests exercise the +# exporter, so skip on the most specific module they need. +pytest.importorskip('opentelemetry.exporter.otlp.proto.http') from rocketride.otelbridge.setup import ( OtelNotInstalledError, @@ -103,6 +107,18 @@ def test_resolve_endpoint_supports_vendor_base_paths(): ) +def test_partial_install_missing_http_exporter_raises_friendly_hint(monkeypatch): + """api/sdk present but exporter absent must raise OtelNotInstalledError, not ModuleNotFoundError.""" + # A None entry in sys.modules makes 'import x' raise ImportError — the + # standard way to simulate the partial install CI exhibits (api/sdk pulled + # in transitively, exporter package absent). + monkeypatch.setitem(sys.modules, 'opentelemetry.exporter.otlp.proto.http.trace_exporter', None) + monkeypatch.setitem(sys.modules, 'opentelemetry.exporter.otlp.proto.http', None) + + with pytest.raises(OtelNotInstalledError, match=r'rocketride\[otel\]'): + _build_span_exporter(BridgeConfigStub(endpoint='http://collector:4318')) + + def test_span_exporter_uses_config_endpoint_and_headers(): config = BridgeConfigStub(endpoint='http://collector:4318', headers={'x-api-key': 'k1'}) exporter = _build_span_exporter(config) From c23b3125742c2c01df19b8702d07de6fc12c11e4 Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:48:28 +0530 Subject: [PATCH 5/6] =?UTF-8?q?fix(sdk):=20OTel=20bridge=20review=20round?= =?UTF-8?q?=20=E2=80=94=20privacy-gate=20errors,=20run-id=20promotion,=20h?= =?UTF-8?q?eader=20precedence,=20shutdown=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit review round on #1612, each finding verified before acting: - mapper: trace.error no longer bypasses the privacy gate — by default only the error signal exports (status ERROR, bare exception event, generic description); --include-content restores verbatim text, now subject to the same 8KB truncation strings previously skipped. The sentinel privacy test plants the sentinel in trace.error. - mapper: a run first seen via flow events (before the task 'running' snapshot) is now promoted from its project|source fallback key to the canonical run id when it arrives — previously a second run state was created (broken parentage) and the orphaned fallback state leaked permanently. Confirmed with a failing repro; guard test ensures two distinct canonical runs sharing (project, source) never merge. - config/setup: only --headers become explicit exporter headers; env headers are left to the SDK so OTEL_EXPORTER_OTLP_TRACES_HEADERS / _METRICS_HEADERS signal precedence works (it was silently defeated). Precedence matrix tests added. - setup: meter provider shutdown wrapped in finally so metrics flush even when tracer shutdown raises. - CLI: --trace-level none message now says flow tracing stays disabled; docs recipes pass collector auth via OTEL_EXPORTER_OTLP_HEADERS env instead of argv (shell-history hygiene); error-mapping doc wording matches the implementation (component leave events). Full extra: 90 passed. Partial install (api/sdk without exporter): 69 passed + clean skip. ruff clean. Co-Authored-By: Claude Fable 5 --- docs/README-otel-bridge.md | 42 ++++++--- .../src/rocketride/cli/commands/otel.py | 17 +++- .../src/rocketride/otelbridge/config.py | 34 +++++-- .../src/rocketride/otelbridge/mapper.py | 60 ++++++++++-- .../src/rocketride/otelbridge/setup.py | 12 ++- .../client-python/tests/test_otel_bridge.py | 18 +++- packages/client-python/tests/test_otel_cli.py | 17 ++++ .../client-python/tests/test_otel_mapper.py | 92 ++++++++++++++++++- .../client-python/tests/test_otel_setup.py | 89 ++++++++++++++++++ packages/docs/content-static/cli.mdx | 9 +- 10 files changed, 347 insertions(+), 43 deletions(-) diff --git a/docs/README-otel-bridge.md b/docs/README-otel-bridge.md index 76aa814f7..a74430b36 100644 --- a/docs/README-otel-bridge.md +++ b/docs/README-otel-bridge.md @@ -94,7 +94,7 @@ rocketride otel [--endpoint URL] [--protocol http|grpc] [--service-name NAME] | `--endpoint ` | `OTEL_EXPORTER_OTLP_ENDPOINT`, else the exporter default (`http://localhost:4318` for http, `localhost:4317` for grpc) | OTLP **base** URL. The signal paths `/v1/traces` / `/v1/metrics` are appended automatically unless already present, so pasting Langfuse's or LangSmith's ingest URL just works. With neither the flag nor `OTEL_EXPORTER_OTLP_ENDPOINT` set, the signal-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` variables are honored too. | | `--protocol

` | `http` | OTLP transport: `http` (http/protobuf) or `grpc`. The gRPC exporter is **not** part of the `otel` extra — see [Troubleshooting](#troubleshooting). | | `--service-name `| `OTEL_SERVICE_NAME` or `rocketride-engine` | The `service.name` resource attribute your backend groups by. | -| `--headers ` | `OTEL_EXPORTER_OTLP_HEADERS` | Comma-separated `key=value` pairs sent with every OTLP request (collector auth). Values are split on the *first* `=` only, so base64 padding survives. | +| `--headers ` | `OTEL_EXPORTER_OTLP_HEADERS` | Comma-separated `key=value` pairs sent with every OTLP request. Values are split on the *first* `=` only, so base64 padding survives. **Prefer the env var for secrets** — command-line arguments are visible in shell history and `ps` output; keep `--headers` for non-secret headers. Without the flag, the OTel SDK resolves `OTEL_EXPORTER_OTLP_HEADERS` and the signal-specific `OTEL_EXPORTER_OTLP_TRACES_HEADERS` / `OTEL_EXPORTER_OTLP_METRICS_HEADERS` itself. | | `--include-content` | off | Include pipeline payload content in spans, truncated to 8 KB per attribute. **By default no payload text reaches any span.** | | `--no-metrics` | off | Export traces only; task status snapshots are not mapped to metrics. | | `--trace-level ` | — | **Informational only.** The bridge cannot change the trace level of runs it did not start; this flag just prints a reminder of how to start traced runs. | @@ -105,7 +105,11 @@ bridge subscribes to every task your API key owns. **Configuration precedence:** explicit CLI flags > standard `OTEL_*` environment variables (`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, -`OTEL_SERVICE_NAME`) > built-in defaults. +`OTEL_SERVICE_NAME`) > built-in defaults. Header environment variables are +resolved by the OTel SDK exporters themselves (never pre-parsed by the bridge), +so the signal-specific `OTEL_EXPORTER_OTLP_TRACES_HEADERS` / +`OTEL_EXPORTER_OTLP_METRICS_HEADERS` keep their spec-defined precedence over the +generic variable whenever `--headers` is not given. **Exit codes:** `0` graceful shutdown (Ctrl+C / SIGTERM), `1` unexpected runtime error, `2` missing dependency (the `otel` extra, or the gRPC exporter with `--protocol grpc`) @@ -136,13 +140,16 @@ Langfuse ingests OTLP **traces over HTTP only** (no gRPC, no metrics) at ```bash # Build the Basic auth value from your Langfuse project keys (pk-lf-... : sk-lf-...) export LANGFUSE_AUTH=$(echo -n 'pk-lf-your-public-key:sk-lf-your-secret-key' | base64) +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic ${LANGFUSE_AUTH}" rocketride otel \ --endpoint https://cloud.langfuse.com/api/public/otel \ - --headers "Authorization=Basic ${LANGFUSE_AUTH}" \ --no-metrics ``` +The auth header travels via `OTEL_EXPORTER_OTLP_HEADERS` rather than a `--headers` +argument so the secret never lands in your shell history or shows up in `ps` output. + Use `https://us.cloud.langfuse.com/api/public/otel` for the US region, or `https:///api/public/otel` for self-hosted (Langfuse v3.22.0+). Pass `--no-metrics`: Langfuse's OTLP ingest is traces-only. Spans from LLM components carry @@ -154,9 +161,11 @@ LangSmith ingests OTLP traces over HTTP at `/otel`, authenticated with an `x-api header (optional `Langsmith-Project` header to pick the project): ```bash +# Env var, not --headers: keeps the API key out of shell history / ps output +export OTEL_EXPORTER_OTLP_HEADERS='x-api-key=your-langsmith-api-key,Langsmith-Project=your-project-name' + rocketride otel \ --endpoint https://api.smith.langchain.com/otel \ - --headers 'x-api-key=your-langsmith-api-key,Langsmith-Project=your-project-name' \ --no-metrics ``` @@ -202,8 +211,9 @@ task task root span (INTERNAL), one per run ``` Runs are correlated primarily on the wire correlation id (`__id`, -`.`), falling back to `(project_id, source)`. Details worth -knowing: +`.`), falling back to `(project_id, source)`. A run first +observed through events lacking `__id` is promoted to its canonical id when that id +later arrives — never tracked twice. Details worth knowing: - **enter/leave pairing is by component identity**, never stack position (the monitor protocol documents why). Expect one short component span per lane write — including @@ -213,9 +223,11 @@ knowing: an error. - **Attaching mid-run:** events for pipes the bridge never saw begin get an implicit root span marked `rocketride.span.implicit=true`. -- **Errors:** a `trace.error` on any flow event sets span status `ERROR`, records an - `exception` span event, and sets `error.type` (`_OTHER` — the wire error is a - free-form string). +- **Errors:** a `trace.error` on a component `leave` event sets span status `ERROR`, + records an `exception` span event, and sets `error.type` (`_OTHER` — the wire error + is a free-form string). The error *text* is treated as payload: by default the + status description is a generic `component error` and the exception event carries + no message; with `--include-content` the wire error text is exported (8 KB cap). - **Task restarts** close the current spans and open a fresh task span with `rocketride.task.restarted=true`. - **GenAI conventions:** component ids map to GenAI semantic conventions where the id @@ -257,12 +269,16 @@ never emitted. ## Privacy: content is excluded by default By default **no pipeline payload content reaches any span** — not lane data -(`trace.data`), not run-segment results, not SSE message bodies. Only structural -metadata (names, ids, lanes, counts, timings) is exported, so the bridge is safe to -point at a shared collector out of the box. +(`trace.data`), not run-segment results, not SSE message bodies, and not the +free-form error text of failed components (node errors routinely quote their input; +the error *signal* — `ERROR` status, `error.type`, the `exception` span event — still +exports, with the generic description `component error`). Only structural metadata +(names, ids, lanes, counts, timings) is exported, so the bridge is safe to point at a +shared collector out of the box. Opting in with `--include-content` copies payload content into the -`rocketride.trace.data`, `rocketride.result`, and `rocketride.sse.data` attributes, +`rocketride.trace.data`, `rocketride.result`, and `rocketride.sse.data` attributes and +exports the verbatim wire error text in span statuses and `exception` events — JSON-serialized and truncated to **8192 characters** per attribute. Treat the flag as what it is: pipeline inputs and outputs flowing into your telemetry backend. diff --git a/packages/client-python/src/rocketride/cli/commands/otel.py b/packages/client-python/src/rocketride/cli/commands/otel.py index fdded5f4b..716b0f5a4 100644 --- a/packages/client-python/src/rocketride/cli/commands/otel.py +++ b/packages/client-python/src/rocketride/cli/commands/otel.py @@ -54,9 +54,13 @@ Usage: rocketride otel --apikey rocketride otel --endpoint http://localhost:4318 --service-name my-engine - rocketride otel --headers 'Authorization=Basic ' --no-metrics + rocketride otel --headers 'Langsmith-Project=my-project' --no-metrics rocketride otel --include-content --apikey +Secrets (collector auth values) are best passed via OTEL_EXPORTER_OTLP_HEADERS +rather than --headers: command-line arguments land in shell history and are +visible in process listings. + Requires the optional OpenTelemetry dependencies: pip install 'rocketride[otel]' @@ -175,7 +179,16 @@ async def execute(self, client: 'RocketRideClient') -> int: # --trace-level is documentation-surface only: the monitor protocol # has no way to change the trace level of runs the bridge didn't start trace_level = getattr(self.args, 'trace_level', None) - if trace_level: + if trace_level == 'none': + # 'none' disables flow tracing, so "emit FLOW events at that + # level" would be nonsense for it. + print( + "Note: --trace-level=none is informational only. 'none' means flow tracing stays " + 'disabled: runs started without a pipelineTraceLevel (or with ' + "pipelineTraceLevel='none') emit no FLOW events, so only task lifecycle spans " + 'and metrics are exported.' + ) + elif trace_level: print( f'Note: --trace-level={trace_level} is informational only. The bridge cannot change the ' f'trace level of runs it did not start; start runs with ' diff --git a/packages/client-python/src/rocketride/otelbridge/config.py b/packages/client-python/src/rocketride/otelbridge/config.py index 1b6d7bba4..6b08938ce 100644 --- a/packages/client-python/src/rocketride/otelbridge/config.py +++ b/packages/client-python/src/rocketride/otelbridge/config.py @@ -33,9 +33,15 @@ - OTEL_EXPORTER_OTLP_ENDPOINT: OTLP base URL (signal paths such as ``/v1/traces`` are appended by the exporter setup, matching the OpenTelemetry SDK's base-endpoint semantics) - - OTEL_EXPORTER_OTLP_HEADERS: comma-separated ``key=value`` pairs sent - with every OTLP export request (e.g. ``Authorization=Basic `` for - Langfuse or ``x-api-key=,Langsmith-Project=`` for LangSmith) + - OTEL_EXPORTER_OTLP_HEADERS (and the signal-specific + OTEL_EXPORTER_OTLP_TRACES_HEADERS / OTEL_EXPORTER_OTLP_METRICS_HEADERS): + comma-separated ``key=value`` pairs sent with every OTLP export request + (e.g. ``Authorization=Basic `` for Langfuse or + ``x-api-key=,Langsmith-Project=`` for LangSmith). These are + deliberately NOT parsed here: without ``--headers`` the exporters are + constructed with no explicit header set and the OTel SDK resolves the + environment itself (signal-specific variables first, then the generic + one). An explicit ``--headers`` overrides all of them. - OTEL_SERVICE_NAME: the ``service.name`` resource attribute When neither ``--endpoint`` nor ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set, the @@ -68,6 +74,8 @@ # Standard OpenTelemetry environment variable names ENV_OTLP_ENDPOINT = 'OTEL_EXPORTER_OTLP_ENDPOINT' +# Header env vars are resolved by the SDK exporters (see from_args_env), never +# parsed here; the name is kept for documentation and tests. ENV_OTLP_HEADERS = 'OTEL_EXPORTER_OTLP_HEADERS' ENV_SERVICE_NAME = 'OTEL_SERVICE_NAME' @@ -127,7 +135,10 @@ class OtelConfig: content reaches any span. no_metrics: When True, only traces are exported; apaevt_status_update snapshots are not mapped to OTel metrics. - headers: Extra headers sent with every OTLP export request. + headers: Explicit extra headers sent with every OTLP export request + (from ``--headers``). When empty, the exporters receive no + explicit headers and the OTel SDK resolves the standard + ``OTEL_EXPORTER_OTLP_*HEADERS`` environment variables itself. """ endpoint: Optional[str] = None @@ -145,7 +156,9 @@ def from_args_env(cls, args: Any, env: Optional[Mapping[str, str]] = None) -> 'O Resolution precedence for each field: CLI argument (when present and non-empty) > standard ``OTEL_*`` environment variable > default. Missing attributes on ``args`` are treated as absent, so partial - namespaces (e.g. in tests) work. + namespaces (e.g. in tests) work. Headers are resolved from + ``--headers`` only; the header environment variables are left to the + OTel SDK exporters so their signal-specific precedence applies. Args: args: Parsed argparse namespace (or any object with optional @@ -165,8 +178,15 @@ def from_args_env(cls, args: Any, env: Optional[Mapping[str, str]] = None) -> 'O protocol = getattr(args, 'protocol', None) or DEFAULT_PROTOCOL service_name = getattr(args, 'service_name', None) or environ.get(ENV_SERVICE_NAME) or DEFAULT_SERVICE_NAME - headers_arg = getattr(args, 'headers', None) - headers = parse_headers(headers_arg if headers_arg else environ.get(ENV_OTLP_HEADERS)) + # Only explicit --headers become constructor headers. When the flag is + # absent, headers stays empty and the exporters are built with NO + # explicit header set, so the OTel SDK resolves the environment itself + # with its documented precedence: signal-specific + # OTEL_EXPORTER_OTLP_TRACES_HEADERS / OTEL_EXPORTER_OTLP_METRICS_HEADERS + # first, then the generic OTEL_EXPORTER_OTLP_HEADERS. Pre-parsing the + # generic variable here would turn it into an explicit header set that + # silently overrides the signal-specific variables. + headers = parse_headers(getattr(args, 'headers', None)) return cls( endpoint=endpoint, diff --git a/packages/client-python/src/rocketride/otelbridge/mapper.py b/packages/client-python/src/rocketride/otelbridge/mapper.py index 3e31c17af..aa03c5f96 100644 --- a/packages/client-python/src/rocketride/otelbridge/mapper.py +++ b/packages/client-python/src/rocketride/otelbridge/mapper.py @@ -33,7 +33,10 @@ - ``apaevt_task`` ``begin`` (or a seeded ``running`` snapshot) opens one task-level root span per run. Runs are keyed primarily by the wire correlation id ``body['__id']`` (``'.'``), falling - back to ``(project_id, source)``. + back to ``(project_id, source)``. A run first observed through + fallback-keyed events is PROMOTED to its canonical id when that id + later arrives (never tracked twice); runs already tracked under a + different canonical id are distinct runs and are never merged. - ``apaevt_flow`` ``op='begin'`` opens a pipe root span for that ``(run, pipe id)`` segment, parented to the run's task span when one is open (else it is a standalone root). @@ -68,9 +71,13 @@ Durations are therefore bridge-observed, not engine-measured. Privacy: payload content (``trace.data``, lane payloads, the run-segment -result delivered in ``end.trace``, and SSE ``data``) never reaches span -attributes or events unless the mapper is built with ``include_content=True``, -and is then truncated to ``MAX_CONTENT_LENGTH`` characters. +result delivered in ``end.trace``, SSE ``data``, and the free-form +``trace.error`` text, which can itself embed payload) never reaches span +attributes, events, or status descriptions unless the mapper is built with +``include_content=True``, and is then truncated to ``MAX_CONTENT_LENGTH`` +characters. The error SIGNAL (span status ERROR, ``error.type``, and the +``exception`` span event) is structural and always exports; by default the +status description is the generic ``GENERIC_ERROR_DESCRIPTION``. GenAI semantic conventions: component spans for ``llm_*`` / ``embedding_*`` / ``agent_*`` / ``tool_*`` nodes carry ``gen_ai.operation.name`` (and @@ -146,6 +153,10 @@ # string, so the semconv fallback value '_OTHER' is used. ERROR_TYPE_FALLBACK = '_OTHER' +# Span status description used in place of the wire's free-form error text +# when include_content is False (error text can embed payload/credentials). +GENERIC_ERROR_DESCRIPTION = 'component error' + # Well-known gen_ai.provider.name values, keyed by RocketRide node provider # (the component id minus its trailing '_' instance suffix). Only values # from the semconv well-known list are emitted; unmapped providers omit the @@ -304,6 +315,22 @@ def _resolve_key(self, run_id: str, project_id: str, source: str) -> str: def _get_run(self, run_id: str, project_id: str, source: str) -> _RunState: key = self._resolve_key(run_id, project_id, source) run = self._runs.pop(key, None) + if run is None and run_id and (project_id or source): + # Promote, never duplicate: a run first observed through events + # without a correlation id lives under the '|' + # fallback key. When its canonical id arrives (task begin or the + # seeded running snapshot), migrate that state to the canonical + # key instead of opening a parallel second run — otherwise its + # open spans are stranded (kept alive by every snapshot that + # announces the same project/source pair) and later flow events + # spawn duplicate implicit roots. Only the fallback-format key is + # migrated; a run tracked under a DIFFERENT canonical id is a + # distinct run and is never merged. + run = self._runs.pop(f'{project_id}|{source}', None) + if run is not None: + logger.debug('promoting fallback-keyed run %s to canonical id %s', run.key, key) + run.key = key + self._restamp_run_id(run, key) if run is None: # Evict before inserting so the new run itself is never a victim. self._evict_runs_above(MAX_TRACKED_RUNS - 1) @@ -317,6 +344,16 @@ def _get_run(self, run_id: str, project_id: str, source: str) -> _RunState: self._aliases[(run.project_id, run.source)] = key return run + @staticmethod + def _restamp_run_id(run: _RunState, key: str) -> None: + """Update ATTR_RUN_ID on a promoted run's still-open spans to the canonical id.""" + if run.task_span is not None: + run.task_span.set_attribute(ATTR_RUN_ID, key) + for pipe in run.pipes.values(): + pipe.root.set_attribute(ATTR_RUN_ID, key) + for entry in pipe.stack: + entry.span.set_attribute(ATTR_RUN_ID, key) + def _evict_runs_above(self, max_size: int) -> None: """Close and drop least-recently-eventful runs until at most max_size remain.""" while len(self._runs) > max_size: @@ -532,12 +569,21 @@ def _flow_leave(self, run: _RunState, pipe_id: int, component: str, trace: dict) span.set_attribute(ATTR_TRACE_DATA, _serialize_content(trace['data'])) error = trace.get('error') if error: - error_text = error if isinstance(error, str) else _serialize_content(error) span.set_attribute(ATTR_ERROR_TYPE, ERROR_TYPE_FALLBACK) # exception.type is omitted: the wire carries no exception class # name, and the '_OTHER' fallback is defined only for error.type. - span.add_event('exception', {'exception.message': error_text}) - span.set_status(self._status(self._status_code.ERROR, error_text)) + if self._include_content: + error_text = _serialize_content(error) + span.add_event('exception', {'exception.message': error_text}) + span.set_status(self._status(self._status_code.ERROR, error_text)) + else: + # The wire error is a free-form string that can embed payload + # (prompts, document text, credentialed URLs), so its text is + # content-gated like every other payload surface. The error + # SIGNAL — ERROR status, error.type, and the exception span + # event — always exports; only the message text is withheld. + span.add_event('exception') + span.set_status(self._status(self._status_code.ERROR, GENERIC_ERROR_DESCRIPTION)) span.end() def _flow_end(self, run: _RunState, pipe_id: int, trace: dict) -> None: diff --git a/packages/client-python/src/rocketride/otelbridge/setup.py b/packages/client-python/src/rocketride/otelbridge/setup.py index 3801ac689..8935721c9 100644 --- a/packages/client-python/src/rocketride/otelbridge/setup.py +++ b/packages/client-python/src/rocketride/otelbridge/setup.py @@ -189,8 +189,14 @@ def build_providers(config: Any) -> Tuple[Any, Any, Callable[[], None]]: meter = meter_provider.get_meter(SCOPE_NAME) def shutdown() -> None: - """Flush pending telemetry and shut down both providers.""" - tracer_provider.shutdown() - meter_provider.shutdown() + """Flush pending telemetry and shut down both providers. + + try/finally: a tracer-provider shutdown failure (e.g. an exporter + raising during the final flush) must not lose the metrics flush. + """ + try: + tracer_provider.shutdown() + finally: + meter_provider.shutdown() return tracer, meter, shutdown diff --git a/packages/client-python/tests/test_otel_bridge.py b/packages/client-python/tests/test_otel_bridge.py index ab9a033b7..441fb40b5 100644 --- a/packages/client-python/tests/test_otel_bridge.py +++ b/packages/client-python/tests/test_otel_bridge.py @@ -29,7 +29,8 @@ Covered here: - OtelConfig precedence: CLI args > OTEL_* env vars > defaults - - OTEL_EXPORTER_OTLP_HEADERS parsing (first-'=' split, whitespace, padding) + - --headers parsing (first-'=' split, whitespace, padding); the header + env vars are deferred to the SDK exporters, never parsed into config - run_bridge event routing (task/flow/sse -> mapper, status -> metrics) - Wildcard monitor subscription (token '*', TASK/SUMMARY/FLOW/SSE) - Startup connection / subscription failure -> exit code 2 @@ -180,13 +181,24 @@ def test_signal_specific_env_left_to_the_sdk_exporters(self): def test_env_used_when_args_absent(self, monkeypatch): monkeypatch.setenv(ENV_OTLP_ENDPOINT, 'https://collector.example:4318') - monkeypatch.setenv(ENV_OTLP_HEADERS, 'Authorization=Basic cGs6c2s=') monkeypatch.setenv(ENV_SERVICE_NAME, 'env-service') config = OtelConfig.from_args_env(SimpleNamespace()) assert config.endpoint == 'https://collector.example:4318' - assert config.headers == {'Authorization': 'Basic cGs6c2s='} assert config.service_name == 'env-service' + def test_header_env_vars_left_to_the_sdk_exporters(self): + # OTEL_EXPORTER_OTLP_HEADERS must NOT be pre-parsed into explicit + # exporter headers: an explicit header set would silently override the + # signal-specific OTEL_EXPORTER_OTLP_TRACES/METRICS_HEADERS variables, + # whose precedence the SDK exporters resolve themselves (see + # test_otel_setup.py for the exporter-level proof). + env = { + ENV_OTLP_HEADERS: 'x-api-key=generic', + 'OTEL_EXPORTER_OTLP_TRACES_HEADERS': 'x-api-key=traces-specific', + } + config = OtelConfig.from_args_env(SimpleNamespace(), env=env) + assert config.headers == {} + def test_args_override_env(self): env = { ENV_OTLP_ENDPOINT: 'https://env.example:4318', diff --git a/packages/client-python/tests/test_otel_cli.py b/packages/client-python/tests/test_otel_cli.py index 5eec3d44c..e99576742 100644 --- a/packages/client-python/tests/test_otel_cli.py +++ b/packages/client-python/tests/test_otel_cli.py @@ -265,6 +265,23 @@ async def fake_run_bridge(client, config): out = capsys.readouterr().out assert 'informational only' in out assert 'pipelineTraceLevel' in out + assert "pipelineTraceLevel='summary'" in out + + async def test_trace_level_none_says_tracing_stays_disabled(self, monkeypatch, capsys): + # 'none' must not be described as a level that emits FLOW events. + monkeypatch.setattr(otel_module, '_otel_available', lambda: True) + + async def fake_run_bridge(client, config): + return 0 + + monkeypatch.setattr(otel_module, 'run_bridge', fake_run_bridge) + + await OtelCommand(FakeCli(), make_args(trace_level='none')).execute(FakeClient()) + + out = capsys.readouterr().out + assert 'informational only' in out + assert 'flow tracing stays disabled' in out + assert 'to emit FLOW events at that level' not in out @pytest.mark.parametrize('exc_type', [OtelNotInstalledError, ImportError]) async def test_missing_dependency_from_bridge_exits_2(self, monkeypatch, capsys, exc_type): diff --git a/packages/client-python/tests/test_otel_mapper.py b/packages/client-python/tests/test_otel_mapper.py index 786a4cd09..9f6af22f5 100644 --- a/packages/client-python/tests/test_otel_mapper.py +++ b/packages/client-python/tests/test_otel_mapper.py @@ -57,6 +57,7 @@ ATTR_SPAN_UNCLOSED, ATTR_TASK_RESTARTED, ATTR_UNMATCHED_LEAVES, + GENERIC_ERROR_DESCRIPTION, MAX_CONTENT_LENGTH, FlowSpanMapper, MetricsMapper, @@ -299,9 +300,7 @@ def test_begin_for_already_open_pipe_recycles_root(): assert len(spans_by_name(exporter, 'second.txt')) == 1 -def test_error_leave_sets_error_status_and_exception_event(): - mapper, exporter = make_mapper() - error_text = 'OpenAI API error: 401 Unauthorized' +def _feed_error_leave(mapper, error_text): mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'])) mapper.handle_event('apaevt_flow', flow('enter', 'llm_openai_1', trace={'lane': 'questions'})) mapper.handle_event( @@ -309,6 +308,12 @@ def test_error_leave_sets_error_status_and_exception_event(): ) mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + +def test_error_leave_with_content_exports_error_text(): + mapper, exporter = make_mapper(include_content=True) + error_text = 'OpenAI API error: 401 Unauthorized' + _feed_error_leave(mapper, error_text) + span = spans_by_name(exporter, 'chat')[0] assert span.status.status_code == StatusCode.ERROR assert error_text in span.status.description @@ -317,6 +322,32 @@ def test_error_leave_sets_error_status_and_exception_event(): assert exception_events and exception_events[0].attributes['exception.message'] == error_text +def test_error_leave_by_default_exports_signal_but_not_error_text(): + """Wire error strings can embed payload; without include_content only the + structural error signal (status, error.type, exception event) exports. + """ + mapper, exporter = make_mapper() + _feed_error_leave(mapper, f'prompt rejected: {SENTINEL}') + + span = spans_by_name(exporter, 'chat')[0] + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == GENERIC_ERROR_DESCRIPTION + assert span.attributes['error.type'] == '_OTHER' + exception_events = [event for event in span.events if event.name == 'exception'] + assert exception_events and not (exception_events[0].attributes or {}) + assert not any_span_contains(exporter, SENTINEL) + + +def test_error_leave_with_content_truncates_string_errors_to_cap(): + mapper, exporter = make_mapper(include_content=True) + _feed_error_leave(mapper, 'e' * (MAX_CONTENT_LENGTH * 2)) + + span = spans_by_name(exporter, 'chat')[0] + exception_events = [event for event in span.events if event.name == 'exception'] + assert len(exception_events[0].attributes['exception.message']) <= MAX_CONTENT_LENGTH + assert len(span.status.description) <= MAX_CONTENT_LENGTH + + # ========================================================================= # SSE EVENTS # ========================================================================= @@ -372,6 +403,11 @@ def _feed_content_events(mapper): 'apaevt_flow', flow('leave', 'llm_openai_1', trace={'lane': 'text', 'data': {'text': SENTINEL}, 'result': 'continue'}), ) + # Error text is a payload surface too: node errors can quote the input. + mapper.handle_event('apaevt_flow', flow('enter', 'llm_openai_2', trace={'lane': 'text'})) + mapper.handle_event( + 'apaevt_flow', flow('leave', 'llm_openai_2', trace={'lane': 'text', 'error': f'bad prompt: {SENTINEL}'}) + ) mapper.handle_event('apaevt_flow', flow('end', 'probe.txt', trace={'name': 'probe.txt', 'text': [SENTINEL]})) @@ -437,6 +473,52 @@ def test_running_snapshot_is_idempotent(): assert len(spans_by_name(exporter, 'task demo.My webhook')) == 1 +def test_late_canonical_id_promotes_fallback_keyed_run(): + """A run first seen through no-__id events must be promoted, not duplicated, + when its canonical id arrives (regression: second _RunState + implicit + duplicate roots + fallback state kept alive forever by snapshots). + """ + mapper, exporter = make_mapper() + # Flow events before any task announcement, without __id -> fallback key. + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'], run_id='')) + mapper.handle_event('apaevt_flow', flow('enter', 'response_1', run_id='')) + # The canonical run id arrives via the seeded running snapshot. + mapper.handle_event( + 'apaevt_task', + { + 'action': 'running', + 'tasks': [{'id': RUN_ID, 'name': 'demo', 'projectId': PROJECT_ID, 'source': SOURCE}], + '__id': '', + }, + ) + # Subsequent flow events carry the canonical id and must hit the SAME run. + mapper.handle_event('apaevt_flow', flow('leave', 'response_1', trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + mapper.handle_event('apaevt_task', task('end')) + + assert mapper.open_span_count() == 0 # nothing stranded under the fallback key + assert len(spans_by_name(exporter, 'task demo')) == 1 + assert len(spans_by_name(exporter, 'probe.txt')) == 1 + assert not spans_by_name(exporter, 'pipe 0') # no implicit duplicate root + leave_span = spans_by_name(exporter, 'response_1')[0] + assert ATTR_SPAN_UNCLOSED not in (leave_span.attributes or {}) + # Open spans of the promoted run are re-stamped with the canonical id. + assert leave_span.attributes['rocketride.run_id'] == RUN_ID + root = spans_by_name(exporter, 'probe.txt')[0] + assert root.attributes['rocketride.run_id'] == RUN_ID + + +def test_promotion_never_merges_distinct_canonical_runs(): + """Two runs with different canonical ids for the same project/source stay separate.""" + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_task', task('begin', run_id='aaaa1111.webhook_1', name='first')) + mapper.handle_event('apaevt_task', task('begin', run_id='bbbb2222.webhook_1', name='second')) + mapper.close_all() + + assert len(spans_by_name(exporter, 'task first')) == 1 + assert len(spans_by_name(exporter, 'task second')) == 1 + + def test_task_restart_closes_and_reopens(): mapper, exporter = make_mapper() mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook')) @@ -696,9 +778,11 @@ def test_full_fixture_replay_produces_coherent_span_forest(): assert chat.status.status_code == StatusCode.ERROR assert chat.parent.span_id == implicit.context.span_id - # Privacy default: real result payload and SSE text never exported. + # Privacy default: real result payload, SSE text, and the wire error text + # (free-form, can embed payload) never exported. assert not any_span_contains(exporter, 'hello otel bridge') assert not any_span_contains(exporter, 'Analyzing your request') + assert not any_span_contains(exporter, '401 Unauthorized') # No deprecated GenAI attribute anywhere. for span in spans: diff --git a/packages/client-python/tests/test_otel_setup.py b/packages/client-python/tests/test_otel_setup.py index f2cb481cb..8bdf8bf44 100644 --- a/packages/client-python/tests/test_otel_setup.py +++ b/packages/client-python/tests/test_otel_setup.py @@ -149,11 +149,67 @@ def test_signal_specific_env_endpoint_honored_when_config_endpoint_absent(monkey def test_env_headers_honored_when_config_headers_absent(monkeypatch): + monkeypatch.delenv('OTEL_EXPORTER_OTLP_TRACES_HEADERS', raising=False) monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=abc123') exporter = _build_span_exporter(BridgeConfigStub(endpoint='http://collector:4318')) assert dict(exporter._headers).get('x-api-key') == 'abc123' +# --------------------------------------------------------------------------- +# Header precedence matrix: --headers > signal-specific env > generic env. +# Only explicit --headers may become constructor headers; env-var resolution +# is the SDK exporters' job, so the signal-specific variables keep their +# spec-defined precedence over the generic one. +# --------------------------------------------------------------------------- + + +def _config_from_cli(headers_arg): + """Resolve an OtelConfig exactly as the CLI does (no env pre-parsing).""" + from types import SimpleNamespace + + from rocketride.otelbridge.config import OtelConfig + + return OtelConfig.from_args_env( + SimpleNamespace(endpoint='http://collector:4318', headers=headers_arg), + env=os.environ, + ) + + +def test_generic_env_headers_reach_exporter_via_sdk_not_config(monkeypatch): + """Generic env only: config carries no headers; the SDK resolves the env var.""" + monkeypatch.delenv('OTEL_EXPORTER_OTLP_TRACES_HEADERS', raising=False) + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=generic') + config = _config_from_cli(None) + assert config.headers == {} + exporter = _build_span_exporter(config) + assert dict(exporter._headers).get('x-api-key') == 'generic' + + +def test_signal_specific_env_headers_beat_generic_env(monkeypatch): + """Signal env present: the SDK's per-signal precedence must win (it would + be defeated if the generic env var were passed as explicit headers). + """ + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=generic') + monkeypatch.setenv('OTEL_EXPORTER_OTLP_TRACES_HEADERS', 'x-api-key=traces-specific') + exporter = _build_span_exporter(_config_from_cli(None)) + assert dict(exporter._headers).get('x-api-key') == 'traces-specific' + + +def test_signal_specific_metrics_env_headers_beat_generic_env(monkeypatch): + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=generic') + monkeypatch.setenv('OTEL_EXPORTER_OTLP_METRICS_HEADERS', 'x-api-key=metrics-specific') + exporter = _build_metric_exporter(_config_from_cli(None)) + assert dict(exporter._headers).get('x-api-key') == 'metrics-specific' + + +def test_cli_headers_flag_beats_all_header_env_vars(monkeypatch): + """Explicit --headers is the one case that becomes constructor headers.""" + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'x-api-key=generic') + monkeypatch.setenv('OTEL_EXPORTER_OTLP_TRACES_HEADERS', 'x-api-key=traces-specific') + exporter = _build_span_exporter(_config_from_cli('x-api-key=explicit')) + assert dict(exporter._headers).get('x-api-key') == 'explicit' + + # ========================================================================= # PROVIDER CONSTRUCTION # ========================================================================= @@ -179,6 +235,39 @@ def test_build_providers_sets_service_name_resource(): shutdown() +def test_shutdown_shuts_down_meter_provider_even_when_tracer_shutdown_raises(monkeypatch): + """A tracer flush failure must not lose the metrics flush (try/finally).""" + import opentelemetry.sdk.metrics as metrics_sdk + import opentelemetry.sdk.trace as trace_sdk + + calls = [] + + class RaisingTracerProvider(trace_sdk.TracerProvider): + def __init__(self, *args, **kwargs): + # Keep the raising shutdown out of the SDK's atexit hook. + kwargs.setdefault('shutdown_on_exit', False) + super().__init__(*args, **kwargs) + + def shutdown(self): + calls.append('tracer') + raise RuntimeError('tracer shutdown boom') + + class RecordingMeterProvider(metrics_sdk.MeterProvider): + def shutdown(self, timeout_millis=30_000): + calls.append('meter') + super().shutdown(timeout_millis=timeout_millis) + + # build_providers imports these names lazily at call time, so patching the + # SDK modules' attributes injects the stubs without faking the SDK itself. + monkeypatch.setattr(trace_sdk, 'TracerProvider', RaisingTracerProvider) + monkeypatch.setattr(metrics_sdk, 'MeterProvider', RecordingMeterProvider) + + _tracer, _meter, shutdown = build_providers(BridgeConfigStub(endpoint='http://localhost:4318', no_metrics=True)) + with pytest.raises(RuntimeError, match='tracer shutdown boom'): + shutdown() + assert calls == ['tracer', 'meter'] + + def test_build_providers_with_metrics_wires_a_periodic_reader(monkeypatch): """With no_metrics=False a periodic reader exports recorded measurements at shutdown.""" import io diff --git a/packages/docs/content-static/cli.mdx b/packages/docs/content-static/cli.mdx index bee1bd6bd..ad702772f 100644 --- a/packages/docs/content-static/cli.mdx +++ b/packages/docs/content-static/cli.mdx @@ -285,9 +285,10 @@ pip install 'rocketride[otel]' # Export to a local collector (default endpoint http://localhost:4318) rocketride otel -# Export to a specific backend with auth headers -rocketride otel --endpoint https://cloud.langfuse.com/api/public/otel \ - --headers "Authorization=Basic ${AUTH_VALUE}" --no-metrics +# Export to a specific backend — pass auth via the env var so the secret +# stays out of shell history and process listings +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic ${AUTH_VALUE}" +rocketride otel --endpoint https://cloud.langfuse.com/api/public/otel --no-metrics # Include pipeline payload content in spans (excluded by default, 8 KB cap) rocketride otel --include-content @@ -300,7 +301,7 @@ Key flags: | `--endpoint ` | OTLP base URL; `/v1/traces` and `/v1/metrics` are appended automatically. Defaults to `OTEL_EXPORTER_OTLP_ENDPOINT`, honoring the signal-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` variables when neither is set, else the exporter default (`http://localhost:4318` for http, `localhost:4317` for grpc). | | `--protocol ` | OTLP transport. Default `http` (http/protobuf); `grpc` needs the separate gRPC exporter package. | | `--service-name ` | `service.name` resource attribute. Defaults to `OTEL_SERVICE_NAME` or `rocketride-engine`. | -| `--headers ` | Headers sent with every OTLP request (collector auth). Defaults to `OTEL_EXPORTER_OTLP_HEADERS`. | +| `--headers ` | Headers sent with every OTLP request. Prefer `OTEL_EXPORTER_OTLP_HEADERS` for secrets — command-line arguments are visible in shell history and `ps` output. | | `--include-content` | Include pipeline payload content in spans, size-capped. By default no payload content is exported. | | `--no-metrics` | Export traces only. | From 7fbf5136e15ceb70f61d72ee8d685c767daa9cde Mon Sep 17 00:00:00 2001 From: nihalnihalani <24360630+nihalnihalani@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:13:05 +0530 Subject: [PATCH 6/6] fix(sdk): absorb stranded fallback run state when canonical state precedes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-on to the run-promotion fix: an SSE event creates the canonical run state WITHOUT registering the (project, source) alias (SSE bodies carry no project/source per the protocol), so an id-less flow for the same run could still open a parallel fallback-keyed state — and because the canonical state already existed, the promotion branch was skipped, stranding the fallback state (kept alive forever by snapshot reconciliation) with duplicate spans. _get_run now reconciles whenever an event carries both the canonical id and project/source: no canonical state → promote (unchanged); canonical state exists → _absorb_fallback_run migrates non-colliding pipes (open spans re-stamped with the canonical run id), closes colliding pipes as rocketride.span.unclosed=true, and drops the fallback key. Only fallback-format keys are touched, so distinct canonical runs never merge (guard test unchanged and green). Both new regression tests failed before the fix. Full extra: 92 passed; partial install: 71 passed + expected skip; ruff clean. Co-Authored-By: Claude Fable 5 --- .../src/rocketride/otelbridge/mapper.py | 77 +++++++++++++++---- .../client-python/tests/test_otel_mapper.py | 50 ++++++++++++ 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/packages/client-python/src/rocketride/otelbridge/mapper.py b/packages/client-python/src/rocketride/otelbridge/mapper.py index aa03c5f96..dc021f8ae 100644 --- a/packages/client-python/src/rocketride/otelbridge/mapper.py +++ b/packages/client-python/src/rocketride/otelbridge/mapper.py @@ -34,8 +34,10 @@ task-level root span per run. Runs are keyed primarily by the wire correlation id ``body['__id']`` (``'.'``), falling back to ``(project_id, source)``. A run first observed through - fallback-keyed events is PROMOTED to its canonical id when that id - later arrives (never tracked twice); runs already tracked under a + fallback-keyed events is reconciled when its canonical id becomes + known: PROMOTED to the canonical key, or ABSORBED into an + already-tracked canonical state (duplicate segments closed as + unclosed) — never tracked twice. Runs already tracked under a different canonical id are distinct runs and are never merged. - ``apaevt_flow`` ``op='begin'`` opens a pipe root span for that ``(run, pipe id)`` segment, parented to the run's task span when one is @@ -315,22 +317,31 @@ def _resolve_key(self, run_id: str, project_id: str, source: str) -> str: def _get_run(self, run_id: str, project_id: str, source: str) -> _RunState: key = self._resolve_key(run_id, project_id, source) run = self._runs.pop(key, None) - if run is None and run_id and (project_id or source): - # Promote, never duplicate: a run first observed through events + if run_id and (project_id or source): + # Reconcile, never duplicate: a run first observed through events # without a correlation id lives under the '|' - # fallback key. When its canonical id arrives (task begin or the - # seeded running snapshot), migrate that state to the canonical - # key instead of opening a parallel second run — otherwise its - # open spans are stranded (kept alive by every snapshot that - # announces the same project/source pair) and later flow events - # spawn duplicate implicit roots. Only the fallback-format key is - # migrated; a run tracked under a DIFFERENT canonical id is a - # distinct run and is never merged. - run = self._runs.pop(f'{project_id}|{source}', None) - if run is not None: - logger.debug('promoting fallback-keyed run %s to canonical id %s', run.key, key) - run.key = key - self._restamp_run_id(run, key) + # fallback key. That state can exist NEXT TO a canonical one when + # no alias was registered yet — e.g. the canonical state was + # seeded by an SSE __id, which carries no project/source. When + # the canonical id and the project/source identity finally meet + # on one event, a fallback-keyed state for the pair is PROMOTED + # to the canonical key (no canonical state yet) or ABSORBED into + # the existing canonical state — otherwise its open spans are + # stranded (kept alive by every snapshot that announces the same + # project/source pair) and later flow events spawn duplicate + # implicit roots. Only the fallback-format key is touched; a run + # tracked under a DIFFERENT canonical id is a distinct run and + # is never merged. + stray = self._runs.pop(f'{project_id}|{source}', None) + if stray is not None: + if run is None: + logger.debug('promoting fallback-keyed run %s to canonical id %s', stray.key, key) + run = stray + run.key = key + self._restamp_run_id(run, key) + else: + logger.debug('absorbing fallback-keyed run %s into canonical run %s', stray.key, key) + self._absorb_fallback_run(run, stray) if run is None: # Evict before inserting so the new run itself is never a victim. self._evict_runs_above(MAX_TRACKED_RUNS - 1) @@ -354,6 +365,38 @@ def _restamp_run_id(run: _RunState, key: str) -> None: for entry in pipe.stack: entry.span.set_attribute(ATTR_RUN_ID, key) + def _absorb_fallback_run(self, run: _RunState, stray: _RunState) -> None: + """ + Merge a stranded fallback-keyed state into the already-tracked canonical run. + + The canonical state is authoritative: stray pipes it does not track + migrate over (their open spans re-stamped with the canonical run id); + a stray pipe whose id the canonical state already tracks is a + duplicate segment and is closed as unclosed, like any other state the + wire abandoned. A stray task span is adopted only when the canonical + run has none; otherwise it too is closed as unclosed. + """ + for pipe_id in list(stray.pipes): + if pipe_id in run.pipes: + self._close_pipe(stray, pipe_id, unclosed=True) + else: + state = stray.pipes.pop(pipe_id) + run.pipes[pipe_id] = state + state.root.set_attribute(ATTR_RUN_ID, run.key) + for entry in state.stack: + entry.span.set_attribute(ATTR_RUN_ID, run.key) + if stray.task_span is not None: + if run.task_span is None: + run.task_span = stray.task_span + run.task_span.set_attribute(ATTR_RUN_ID, run.key) + else: + stray.task_span.set_attribute(ATTR_SPAN_UNCLOSED, True) + stray.task_span.end() + stray.task_span = None + run.name = run.name or stray.name + run.project_id = run.project_id or stray.project_id + run.source = run.source or stray.source + def _evict_runs_above(self, max_size: int) -> None: """Close and drop least-recently-eventful runs until at most max_size remain.""" while len(self._runs) > max_size: diff --git a/packages/client-python/tests/test_otel_mapper.py b/packages/client-python/tests/test_otel_mapper.py index 9f6af22f5..bf006f9c3 100644 --- a/packages/client-python/tests/test_otel_mapper.py +++ b/packages/client-python/tests/test_otel_mapper.py @@ -519,6 +519,56 @@ def test_promotion_never_merges_distinct_canonical_runs(): assert len(spans_by_name(exporter, 'task second')) == 1 +def test_fallback_state_absorbed_when_canonical_state_already_exists(): + """SSE-first ordering: an SSE __id seeds the canonical state WITHOUT + project/source (SSE bodies carry neither), so no alias exists and id-less + flow events open a parallel fallback-keyed state. When the canonical id + and the project/source identity finally meet on one event, the fallback + state must be absorbed — not left stranded next to the canonical one. + """ + mapper, exporter = make_mapper() + # 1. SSE with __id creates the canonical run (implicit root for pipe 5). + mapper.handle_event('apaevt_sse', {'pipe_id': 5, 'type': 'thinking', '__id': RUN_ID}) + # 2. Id-less flow events for the same run: no alias -> fallback state. + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'], run_id='')) + mapper.handle_event('apaevt_flow', flow('enter', 'response_1', run_id='')) + # 3. Canonical id + project/source meet on the task begin event. + mapper.handle_event('apaevt_task', task('begin', name='demo')) + # 4. Later canonical-id flow events must hit the one surviving state. + mapper.handle_event('apaevt_flow', flow('leave', 'response_1', trace={'result': 'continue'})) + mapper.handle_event('apaevt_flow', flow('end', 'probe.txt')) + mapper.handle_event('apaevt_task', task('end')) + + assert mapper.open_span_count() == 0 # nothing stranded under the fallback key + assert len(spans_by_name(exporter, 'task demo')) == 1 + assert len(spans_by_name(exporter, 'probe.txt')) == 1 + assert not spans_by_name(exporter, 'pipe 0') # no duplicate root for pipe 0 + leave_span = spans_by_name(exporter, 'response_1')[0] + assert ATTR_SPAN_UNCLOSED not in (leave_span.attributes or {}) + # Migrated spans are re-stamped with the canonical id. + assert leave_span.attributes['rocketride.run_id'] == RUN_ID + assert spans_by_name(exporter, 'probe.txt')[0].attributes['rocketride.run_id'] == RUN_ID + + +def test_fallback_absorption_closes_colliding_pipes_as_unclosed(): + """When both states opened the same pipe id, the canonical segment wins + and the fallback duplicate is closed as unclosed rather than merged. + """ + mapper, exporter = make_mapper() + mapper.handle_event('apaevt_sse', {'pipe_id': 0, 'type': 'thinking', '__id': RUN_ID}) + mapper.handle_event('apaevt_flow', flow('begin', 'probe.txt', pipes=['probe.txt'], run_id='')) + mapper.handle_event('apaevt_task', task('begin', name='demo')) # reconciles: pipe 0 collides + mapper.handle_event('apaevt_task', task('end')) + + assert mapper.open_span_count() == 0 + # The fallback duplicate for pipe 0 was closed as unclosed at absorption. + stray_root = spans_by_name(exporter, 'probe.txt')[0] + assert stray_root.attributes[ATTR_SPAN_UNCLOSED] is True + # The canonical implicit root (carrying the SSE event) survived until task end. + implicit = spans_by_name(exporter, 'pipe 0')[0] + assert [event for event in implicit.events if event.name == 'thinking'] + + def test_task_restart_closes_and_reopens(): mapper, exporter = make_mapper() mapper.handle_event('apaevt_task', task('begin', name='demo.My webhook'))