From ba5caebf2f7e082b61e56753473d1edcf4de535f Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Wed, 8 Jul 2026 15:57:04 -0700 Subject: [PATCH 01/11] Add Monocle tracing Enable Monocle (OpenTelemetry tracing for LLM apps) with one setup call plus the monocle_apptrace dependency. setup_monocle_telemetry auto-instruments the frameworks already in use and writes traces to .monocle/. Additive; no changes to application logic. --- backend/packages/harness/deerflow/agents/__init__.py | 4 ++++ backend/pyproject.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/backend/packages/harness/deerflow/agents/__init__.py b/backend/packages/harness/deerflow/agents/__init__.py index 4b56cbc067d..450e4e2a559 100644 --- a/backend/packages/harness/deerflow/agents/__init__.py +++ b/backend/packages/harness/deerflow/agents/__init__.py @@ -1,5 +1,9 @@ from .features import Next, Prev, RuntimeFeatures +from monocle_apptrace import setup_monocle_telemetry + +setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list="file") + __all__ = [ "create_deerflow_agent", "RuntimeFeatures", diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 189b4af7967..d84312ce490 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "pyjwt>=2.13.0", "email-validator>=2.0.0", "e2b-code-interpreter>=2.8.1", + "monocle_apptrace", ] [project.optional-dependencies] From 39b4d4943e9572191e810fceb06c716433f24e39 Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Fri, 10 Jul 2026 11:20:47 -0700 Subject: [PATCH 02/11] Config-gate Monocle telemetry in the Gateway lifespan Addresses review on #4024: moves setup_monocle_telemetry out of agents/__init__ import time into the Gateway lifespan, gated by MonocleTracingConfig (MONOCLE_TRACING env, default off). Warns on the Langfuse/global-OTel-provider conflict and relies on monocle_apptrace's own duplicate-setup guard and existing-provider attach. Pins monocle_apptrace>=0.8.8 (+ uv.lock), adds .monocle/ to .gitignore, adds tests (default-off / toggle-on / no import-time setup), and documents exporters, Okahu, and the VS Code viewer in README, config.example.yaml, and backend/AGENTS.md. --- .gitignore | 3 + README.md | 23 +++++- backend/AGENTS.md | 6 ++ backend/app/gateway/app.py | 9 +++ .../harness/deerflow/agents/__init__.py | 4 - .../harness/deerflow/config/__init__.py | 2 + .../harness/deerflow/config/tracing_config.py | 30 ++++++++ .../harness/deerflow/tracing/monocle.py | 36 +++++++++ backend/pyproject.toml | 2 +- backend/tests/test_monocle_tracing.py | 76 +++++++++++++++++++ backend/uv.lock | 45 +++++++++++ config.example.yaml | 25 ++++++ 12 files changed, 252 insertions(+), 9 deletions(-) create mode 100644 backend/packages/harness/deerflow/tracing/monocle.py create mode 100644 backend/tests/test_monocle_tracing.py diff --git a/.gitignore b/.gitignore index 6dba53b46b9..d4649770f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ config.yaml.bak /frontend/playwright-report/ .gstack/ .worktrees + +# Monocle agent-observability trace output (local file exporter) +.monocle/ diff --git a/README.md b/README.md index f7bf683ce08..0ffc2871912 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ DeerFlow has newly integrated the intelligent search and crawling toolset indepe - [IM Channels](#im-channels) - [LangSmith Tracing](#langsmith-tracing) - [Langfuse Tracing](#langfuse-tracing) - - [Using Both Providers](#using-both-providers) + - [Monocle Tracing](#monocle-tracing) + - [Using Multiple Providers](#using-multiple-providers) - [From Deep Research to Super Agent Harness](#from-deep-research-to-super-agent-harness) - [Core Features](#core-features) - [Skills \& Tools](#skills--tools) @@ -592,11 +593,25 @@ If you are using a self-hosted Langfuse instance, set `LANGFUSE_BASE_URL` to you These are injected into `RunnableConfig.metadata` at the graph invocation root for both the gateway path (`runtime/runs/worker.py::run_agent`) and the embedded path (`client.py::DeerFlowClient.stream`), so any LangChain-compatible callback can read them. Set `DEER_FLOW_ENV` (or `ENVIRONMENT`) to tag traces by deployment environment. -#### Using Both Providers +#### Monocle Tracing -If both LangSmith and Langfuse are enabled, DeerFlow attaches both tracing callbacks and reports the same model activity to both systems. +DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle) telemetry for its LangGraph/LangChain runs. -If a provider is explicitly enabled but missing required credentials, or if its callback fails to initialize, DeerFlow fails fast when tracing is initialized during model creation and the error message names the provider that caused the failure. +Add the following to your `.env` file: + +```bash +MONOCLE_TRACING=true +MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: file) +OKAHU_API_KEY=okh_xxxxxxxx # required only for the `okahu` exporter +``` + +Monocle is off by default and is initialized at Gateway startup when enabled. The `file` exporter writes one trace per run to `.monocle/`; open those in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. + +Send your Monocle traces to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze them across many runs. + +#### Using Multiple Providers + +LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback, so it runs alongside LangSmith but not Langfuse. For Docker deployments, tracing is disabled by default. Set `LANGSMITH_TRACING=true` and `LANGSMITH_API_KEY` in your `.env` to enable it. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 8c7d33d51b4..d527cc04eb9 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -698,6 +698,12 @@ LangSmith and Langfuse are both supported. The wiring lives in two layers: Returns `{}` when Langfuse is not in the enabled providers — LangSmith-only deployments are unaffected. Set `DEER_FLOW_ENV` (or `ENVIRONMENT`) to tag traces by deployment environment. Tests live in `tests/test_tracing_factory.py`, `tests/test_tracing_metadata.py`, `tests/test_worker_langfuse_metadata.py`, `tests/test_client_langfuse_metadata.py`, and `tests/test_subagent_executor.py::TestSubagentTracingWiring`. +**Monocle telemetry** is a third provider, structurally unlike LangSmith/Langfuse. It is **not** a LangChain callback: `tracing/monocle.py::setup_monocle_tracing_if_enabled()` calls `monocle_apptrace.setup_monocle_telemetry()` once, which installs a **process-global OTel `TracerProvider`**, patches span serialization, and auto-instruments the openai/langchain/langgraph clients. Because that is a one-time, process-global side effect (not a per-run callback), it is initialized from the **Gateway lifespan** (`app/gateway/app.py`) — never from `build_tracing_callbacks()` — and it is **off by default**. The setup call was deliberately moved out of `agents/__init__.py`, so `import deerflow.agents` must never start tracing (pinned by `tests/test_monocle_tracing.py::test_no_import_time_setup`). + +Unlike the Langfuse metadata above, DeerFlow injects **no** per-run fields into Monocle traces — the only attribute it sets is `workflow_name="deer-flow"`; every span attribute (`span.type`, `entity.*`, token usage, span inputs/outputs, `scope.agentic.session`) is produced by Monocle's own metamodel and auto-instrumentation, so there is no DeerFlow trace-attribute layer to maintain here. + +Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and, when another library owns the global provider, attaches its span processor to it rather than overriding — so the wrapper only gates on config and **warns** when Langfuse (v4) is also enabled, since both use the single global provider and whichever inits first owns it. Tests: `tests/test_monocle_tracing.py`. + ### Config Schema **`config.yaml`** key sections: diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 42ccaca337d..44691c97e2a 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -37,6 +37,7 @@ from app.gateway.trace_middleware import TraceMiddleware, resolve_trace_enabled from deerflow.config import app_config as deerflow_app_config from deerflow.logging_config import DEFAULT_LOG_DATE_FORMAT, DEFAULT_LOG_FORMAT, configure_logging +from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled from deerflow.uploads.manager import cleanup_stale_upload_staging_files AppConfig = deerflow_app_config.AppConfig @@ -189,6 +190,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: config = get_gateway_config() logger.info(f"Starting API Gateway on {config.host}:{config.port}") + # Agent observability (Monocle). Off by default; enabled with + # MONOCLE_TRACING. Initialized here at startup — not at import time — so a + # plain `import deerflow.agents` never installs a process-global tracer. + try: + setup_monocle_tracing_if_enabled() + except Exception: # pragma: no cover - observability must never break startup + logger.exception("Monocle tracing setup failed; continuing without it") + # Pre-warm tiktoken encoding cache so the first memory-injection request # never blocks on the BPE data download (which hits an OpenAI/Azure URL # that may be unreachable in restricted networks — see issue #3402). diff --git a/backend/packages/harness/deerflow/agents/__init__.py b/backend/packages/harness/deerflow/agents/__init__.py index 450e4e2a559..4b56cbc067d 100644 --- a/backend/packages/harness/deerflow/agents/__init__.py +++ b/backend/packages/harness/deerflow/agents/__init__.py @@ -1,9 +1,5 @@ from .features import Next, Prev, RuntimeFeatures -from monocle_apptrace import setup_monocle_telemetry - -setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list="file") - __all__ = [ "create_deerflow_agent", "RuntimeFeatures", diff --git a/backend/packages/harness/deerflow/config/__init__.py b/backend/packages/harness/deerflow/config/__init__.py index bf74dc5e8bc..76751936bed 100644 --- a/backend/packages/harness/deerflow/config/__init__.py +++ b/backend/packages/harness/deerflow/config/__init__.py @@ -9,6 +9,7 @@ get_enabled_tracing_providers, get_explicitly_enabled_tracing_providers, get_tracing_config, + is_monocle_tracing_enabled, is_tracing_enabled, validate_enabled_tracing_providers, ) @@ -27,6 +28,7 @@ "get_tracing_config", "get_explicitly_enabled_tracing_providers", "get_enabled_tracing_providers", + "is_monocle_tracing_enabled", "is_tracing_enabled", "validate_enabled_tracing_providers", ] diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index 399e3742449..75a482a7b3c 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -47,11 +47,27 @@ def validate(self) -> None: raise ValueError(f"Langfuse tracing is enabled but required settings are missing: {', '.join(missing)}") +class MonocleTracingConfig(BaseModel): + """Configuration for Monocle telemetry.""" + + enabled: bool = Field(...) + exporters: str = Field(...) + + @property + def is_configured(self) -> bool: + return self.enabled + + def validate(self) -> None: + # No external credentials are required; the "file" exporter writes locally. + return None + + class TracingConfig(BaseModel): """Tracing configuration for supported providers.""" langsmith: LangSmithTracingConfig = Field(...) langfuse: LangfuseTracingConfig = Field(...) + monocle: MonocleTracingConfig = Field(...) @property def is_configured(self) -> bool: @@ -125,6 +141,10 @@ def get_tracing_config() -> TracingConfig: secret_key=_first_env_value("LANGFUSE_SECRET_KEY"), host=_first_env_value("LANGFUSE_BASE_URL") or "https://cloud.langfuse.com", ), + monocle=MonocleTracingConfig( + enabled=_env_flag_preferred("MONOCLE_TRACING"), + exporters=_first_env_value("MONOCLE_EXPORTERS") or "file", + ), ) return _tracing_config @@ -149,6 +169,16 @@ def is_tracing_enabled() -> bool: return get_tracing_config().is_configured +def is_monocle_tracing_enabled() -> bool: + """Whether Monocle OTel observability is enabled (via ``MONOCLE_TRACING``). + + Kept separate from :func:`get_enabled_tracing_providers` because Monocle is a + process-global instrumentor activated at startup, not a per-run LangChain + callback. + """ + return get_tracing_config().monocle.is_configured + + def reset_tracing_config() -> None: """Discard the cached :class:`TracingConfig` so the next call rebuilds it. diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py new file mode 100644 index 00000000000..c3d8b903f2f --- /dev/null +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -0,0 +1,36 @@ +"""Monocle telemetry: initialized once from the Gateway lifespan when ``MONOCLE_TRACING`` is set.""" + +from __future__ import annotations + +import logging + +from deerflow.config import ( + get_enabled_tracing_providers, + get_tracing_config, + is_monocle_tracing_enabled, +) + +logger = logging.getLogger(__name__) + + +def setup_monocle_tracing_if_enabled() -> bool: + """Initialize Monocle telemetry when ``MONOCLE_TRACING`` is enabled; a no-op otherwise. + + ``monocle_apptrace.setup_monocle_telemetry()`` is itself idempotent and, when another + library already owns the global OpenTelemetry provider, attaches to it instead of + overriding it — so this stays a thin, config-gated wrapper. Returns ``True`` when enabled. + """ + if not is_monocle_tracing_enabled(): + return False + + # Monocle and Langfuse (v4) both use the global OpenTelemetry provider; whichever + # initializes first owns it, so warn when both are on to avoid surprises. + if "langfuse" in get_enabled_tracing_providers(): + logger.warning("MONOCLE_TRACING is enabled alongside Langfuse; both use the global OpenTelemetry provider — verify traces export where you expect.") + + exporters = get_tracing_config().monocle.exporters + from monocle_apptrace import setup_monocle_telemetry + + setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) + logger.info("Monocle telemetry enabled (exporters=%s)", exporters) + return True diff --git a/backend/pyproject.toml b/backend/pyproject.toml index d84312ce490..61738c74977 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "pyjwt>=2.13.0", "email-validator>=2.0.0", "e2b-code-interpreter>=2.8.1", - "monocle_apptrace", + "monocle_apptrace>=0.8.8", ] [project.optional-dependencies] diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py new file mode 100644 index 00000000000..d43df5d9ca5 --- /dev/null +++ b/backend/tests/test_monocle_tracing.py @@ -0,0 +1,76 @@ +"""Tests for Monocle telemetry setup. + +Covers the config gate (``MONOCLE_TRACING`` default off / toggle on), the setup +helper's behavior, and the regression that importing ``deerflow.agents`` no +longer sets up telemetry at import time. +""" + +from __future__ import annotations + +import pytest + +from deerflow.config import is_monocle_tracing_enabled +from deerflow.config.tracing_config import get_tracing_config, reset_tracing_config +from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled + + +@pytest.fixture(autouse=True) +def clear_monocle_env(monkeypatch): + for name in ("MONOCLE_TRACING", "MONOCLE_EXPORTERS"): + monkeypatch.delenv(name, raising=False) + reset_tracing_config() + yield + reset_tracing_config() + + +def test_disabled_by_default(): + assert is_monocle_tracing_enabled() is False + assert get_tracing_config().monocle.enabled is False + + +def test_setup_noop_when_disabled(monkeypatch): + called = False + + def _fail(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", _fail) + assert setup_monocle_tracing_if_enabled() is False + assert called is False + + +def test_toggles_on_and_sets_up(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + + captured: dict = {} + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: captured.update(kw)) + + assert is_monocle_tracing_enabled() is True + assert setup_monocle_tracing_if_enabled() is True + assert captured == {"workflow_name": "deer-flow", "monocle_exporters_list": "file"} + + +def test_custom_exporters(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,console") + reset_tracing_config() + + captured: dict = {} + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: captured.update(kw)) + + assert setup_monocle_tracing_if_enabled() is True + assert captured["monocle_exporters_list"] == "file,console" + + +def test_no_import_time_setup(): + """Regression: importing deerflow.agents must not install telemetry. + + The setup call used to live at module import in ``deerflow/agents/__init__``. + It now happens only via the gateway lifespan, so the package namespace must + not even carry ``setup_monocle_telemetry``. + """ + import deerflow.agents as agents + + assert not hasattr(agents, "setup_monocle_telemetry") diff --git a/backend/uv.lock b/backend/uv.lock index c82e4ec1e2a..a9fb7d36fac 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -786,6 +786,7 @@ dependencies = [ { name = "langgraph-sdk" }, { name = "lark-oapi" }, { name = "markdown-to-mrkdwn" }, + { name = "monocle-apptrace" }, { name = "pyjwt" }, { name = "python-multipart" }, { name = "python-telegram-bot" }, @@ -832,6 +833,7 @@ requires-dist = [ { name = "langgraph-sdk", specifier = ">=0.1.51" }, { name = "lark-oapi", specifier = ">=1.4.0" }, { name = "markdown-to-mrkdwn", specifier = ">=0.3.1" }, + { name = "monocle-apptrace", specifier = ">=0.8.8" }, { name = "pyjwt", specifier = ">=2.13.0" }, { name = "python-multipart", specifier = ">=0.0.31" }, { name = "python-telegram-bot", specifier = ">=21.0" }, @@ -2524,6 +2526,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "monocle-apptrace" +version = "0.8.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rfc3986" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/61/ec2213cc56e2d5d6b67db691a408ca8ba9d8440ed48fc821a422b6a5560e/monocle_apptrace-0.8.8.tar.gz", hash = "sha256:ebebd8b8da8dbf48cdb708e14e1a61293c48e50ec02ba24cf29c01ea776ba02b", size = 241057, upload-time = "2026-07-08T18:01:51.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/8f/3757b6ca53346a32bc271e4d5c0dd64ad920eb1b05e514b00c8ff2e02c02/monocle_apptrace-0.8.8-py3-none-any.whl", hash = "sha256:7e17f0f73e4f446f8e8722251201ffc30429fea0d93d545b023b41bfc4f83503", size = 354697, upload-time = "2026-07-08T18:01:47.407Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -2868,6 +2889,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/4d/ef07ff2fc630849f2080ae0ae73a61f67257905b7ac79066640bfa0c5739/opentelemetry_exporter_otlp_proto_http-1.41.1-py3-none-any.whl", hash = "sha256:1a21e8f49c7a946d935551e90947d6c3eb39236723c6624401da0f33d68edcb4", size = 22673, upload-time = "2026-04-24T13:15:21.313Z" }, ] +[[package]] +name = "opentelemetry-instrumentation" +version = "0.62b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/cb/0523b92c112a6cc70be43724343dc45225d3af134419844d7879a07755d4/opentelemetry_instrumentation-0.62b1.tar.gz", hash = "sha256:90e92a905ba4f84db06ac3aec96701df6c079b2d66e9379f8739f0a1bdcc7f45", size = 34043, upload-time = "2026-04-24T13:22:31.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/0f/45adbaea1f81b847cffdcee4f4b5f89297e42facf7fac78c7aaac4c38e75/opentelemetry_instrumentation-0.62b1-py3-none-any.whl", hash = "sha256:976fc6e640f2006599e97429c949e622c108d0c17c2059347d1e6c93c707f257", size = 34163, upload-time = "2026-04-24T13:21:31.722Z" }, +] + [[package]] name = "opentelemetry-proto" version = "1.41.1" @@ -4012,6 +4048,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] +[[package]] +name = "rfc3986" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026, upload-time = "2022-01-10T00:52:30.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326, upload-time = "2022-01-10T00:52:29.594Z" }, +] + [[package]] name = "rich" version = "15.0.0" diff --git a/config.example.yaml b/config.example.yaml index 9b86608d222..1e35f781f9c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -30,6 +30,31 @@ logging: enabled: false format: text +# ============================================================================ +# Monocle telemetry +# ============================================================================ +# Optional agent tracing via Monocle. Configured through environment variables +# (like LangSmith/Langfuse), not a config.yaml key, and OFF by default. When +# enabled, Monocle is initialized once at Gateway startup and auto-instruments +# the LLM/agent clients; a plain library import does not start tracing. +# +# MONOCLE_TRACING=true # enable (default: off) +# MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: file) +# OKAHU_API_KEY=okh_... # required only for the `okahu` exporter +# +# The `file` exporter writes one trace JSON per run to `.monocle/` under the +# process working directory (git-ignored). Each trace captures the workflow and +# per-agent/LLM/tool spans, including span inputs and outputs (prompts, tool +# arguments, and model responses), token usage, and timings. +# +# Data-exfiltration note: `file` persists prompts and model outputs to local +# disk; non-file exporters (e.g. `okahu`) send that same span data off-box to an +# external collector. Enable only exporters whose data handling you trust, and +# be aware that prompts/outputs may contain sensitive content. +# +# Note: Monocle and Langfuse both install a global OpenTelemetry TracerProvider; +# enable only one OTel-based tracer at a time (see backend/AGENTS.md → Tracing). + # ============================================================================ # Token Usage # ============================================================================ From 745409c438d38915c74a09d04c09c727d9d186f8 Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Fri, 10 Jul 2026 11:26:35 -0700 Subject: [PATCH 03/11] Clarify Monocle/Langfuse single-provider guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the docstring, warning, and AGENTS.md consistent with the README: only one library can own the global OpenTelemetry provider; Monocle initializes at startup before Langfuse's per-run handler, so enabling both drops Langfuse's spans — enable one OTel tracer (LangSmith, a callback, coexists fine). --- backend/AGENTS.md | 2 +- backend/packages/harness/deerflow/tracing/monocle.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 9794b041269..8812484ecec 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -709,7 +709,7 @@ Returns `{}` when Langfuse is not in the enabled providers — LangSmith-only de Unlike the Langfuse metadata above, DeerFlow injects **no** per-run fields into Monocle traces — the only attribute it sets is `workflow_name="deer-flow"`; every span attribute (`span.type`, `entity.*`, token usage, span inputs/outputs, `scope.agentic.session`) is produced by Monocle's own metamodel and auto-instrumentation, so there is no DeerFlow trace-attribute layer to maintain here. -Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and, when another library owns the global provider, attaches its span processor to it rather than overriding — so the wrapper only gates on config and **warns** when Langfuse (v4) is also enabled, since both use the single global provider and whichever inits first owns it. Tests: `tests/test_monocle_tracing.py`. +Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and never force-overrides an existing global provider, so the wrapper only gates on config. It also **warns** when Langfuse (v4) is enabled: only one library can own the single global OpenTelemetry provider, and Monocle initializes here at startup (before Langfuse's per-run handler), so with both on Langfuse loses its spans — enable one OTel tracer. (LangSmith is a plain callback and coexists fine.) Tests: `tests/test_monocle_tracing.py`. ### Config Schema diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index c3d8b903f2f..870fe2e61a9 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -16,17 +16,17 @@ def setup_monocle_tracing_if_enabled() -> bool: """Initialize Monocle telemetry when ``MONOCLE_TRACING`` is enabled; a no-op otherwise. - ``monocle_apptrace.setup_monocle_telemetry()`` is itself idempotent and, when another - library already owns the global OpenTelemetry provider, attaches to it instead of - overriding it — so this stays a thin, config-gated wrapper. Returns ``True`` when enabled. + ``monocle_apptrace.setup_monocle_telemetry()`` is idempotent, so this stays a thin, + config-gated wrapper. Returns ``True`` when enabled. """ if not is_monocle_tracing_enabled(): return False - # Monocle and Langfuse (v4) both use the global OpenTelemetry provider; whichever - # initializes first owns it, so warn when both are on to avoid surprises. + # Only one OpenTelemetry provider can own the process. Monocle initializes here (at + # startup), before Langfuse's per-run handler, so enabling both means Langfuse loses + # its spans — warn so the operator turns one off. if "langfuse" in get_enabled_tracing_providers(): - logger.warning("MONOCLE_TRACING is enabled alongside Langfuse; both use the global OpenTelemetry provider — verify traces export where you expect.") + logger.warning("MONOCLE_TRACING is enabled alongside Langfuse; both need the global OpenTelemetry provider and only one can win. Enable only one of them.") exporters = get_tracing_config().monocle.exporters from monocle_apptrace import setup_monocle_telemetry From 803be488027d504b8aed82ca523f9b8d686a11fb Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Fri, 10 Jul 2026 21:35:14 -0700 Subject: [PATCH 04/11] Address review: optional extra, exporter validation, off-box warning, tests Responds to the second review round. - Make monocle_apptrace an optional extra (deerflow-harness[monocle], re-exposed as deer-flow[monocle]) following the boxlite/tui precedent, so a default install no longer pulls the OpenTelemetry stack. It stays pinned in the dev group for the tracing tests, and enabling MONOCLE_TRACING without the extra raises a clear install error. - Warn loudly at startup whenever any exporter other than `file` is configured, since those move prompts, tool inputs/outputs, and completions beyond the local .monocle/ directory. - Validate MONOCLE_EXPORTERS against the known exporter names and require OKAHU_API_KEY when okahu is selected, mirroring the Langfuse pattern. Validation runs from Monocle's own init (not validate_enabled) so a config typo can never fail agent runs; errors surface at Gateway startup instead. - Grow the tests from 5 to 13: caplog coverage for the Langfuse-conflict and off-box warnings, exporter validation cases, a stronger import-time regression that asserts the global TracerProvider is not replaced, and a subprocess double-invoke test exercising the real check_duplicate_setup. - Docs: config.example.yaml block retitled to a dedicated tracing header; README documents the [monocle] install and scopes tracing to Gateway runs. --- README.md | 10 +- .../harness/deerflow/config/tracing_config.py | 15 +- .../harness/deerflow/tracing/monocle.py | 25 +++- backend/packages/harness/pyproject.toml | 3 + backend/pyproject.toml | 5 +- backend/tests/test_monocle_tracing.py | 140 +++++++++++++++++- backend/uv.lock | 16 +- config.example.yaml | 2 +- 8 files changed, 197 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index afde5398e1d..40a4b02b460 100644 --- a/README.md +++ b/README.md @@ -596,9 +596,13 @@ These are injected into `RunnableConfig.metadata` at the graph invocation root f #### Monocle Tracing -DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle) telemetry for its LangGraph/LangChain runs. +DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle) telemetry for its LangGraph/LangChain runs. Monocle is an optional extra, so install it when you want tracing: -Add the following to your `.env` file: +```bash +pip install "deer-flow[monocle]" +``` + +Then add the following to your `.env` file: ```bash MONOCLE_TRACING=true @@ -606,7 +610,7 @@ MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: OKAHU_API_KEY=okh_xxxxxxxx # required only for the `okahu` exporter ``` -Monocle is off by default and is initialized at Gateway startup when enabled. The `file` exporter writes one trace per run to `.monocle/`; open those in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. +Monocle is off by default and is initialized at Gateway startup when enabled, so it covers runs served by the Gateway (the embedded `DeerFlowClient` and the TUI do not start it). The `file` exporter writes one trace per run to `.monocle/`; open those in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. Send your Monocle traces to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze them across many runs. diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index 75a482a7b3c..e6c350be2e6 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -47,19 +47,29 @@ def validate(self) -> None: raise ValueError(f"Langfuse tracing is enabled but required settings are missing: {', '.join(missing)}") +_MONOCLE_EXPORTERS = ("file", "console", "okahu", "s3", "blob", "gcs") + + class MonocleTracingConfig(BaseModel): """Configuration for Monocle telemetry.""" enabled: bool = Field(...) exporters: str = Field(...) + okahu_api_key: str | None = Field(...) @property def is_configured(self) -> bool: return self.enabled def validate(self) -> None: - # No external credentials are required; the "file" exporter writes locally. - return None + if not self.enabled: + return + selected = [e.strip() for e in self.exporters.split(",") if e.strip()] + unknown = [e for e in selected if e not in _MONOCLE_EXPORTERS] + if unknown: + raise ValueError(f"MONOCLE_EXPORTERS has unknown exporter(s): {', '.join(unknown)}. Allowed: {', '.join(_MONOCLE_EXPORTERS)}.") + if "okahu" in selected and not self.okahu_api_key: + raise ValueError("Monocle 'okahu' exporter is selected but OKAHU_API_KEY is not set.") class TracingConfig(BaseModel): @@ -144,6 +154,7 @@ def get_tracing_config() -> TracingConfig: monocle=MonocleTracingConfig( enabled=_env_flag_preferred("MONOCLE_TRACING"), exporters=_first_env_value("MONOCLE_EXPORTERS") or "file", + okahu_api_key=_first_env_value("OKAHU_API_KEY"), ), ) return _tracing_config diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index 870fe2e61a9..efa65265739 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -22,14 +22,35 @@ def setup_monocle_tracing_if_enabled() -> bool: if not is_monocle_tracing_enabled(): return False + monocle = get_tracing_config().monocle + # Fail fast on an unknown MONOCLE_EXPORTERS value or a missing OKAHU_API_KEY, + # with a clear message, before instrumenting. Validated here (not in the + # per-run callback path) so a config typo never breaks agent runs. + monocle.validate() + # Only one OpenTelemetry provider can own the process. Monocle initializes here (at # startup), before Langfuse's per-run handler, so enabling both means Langfuse loses # its spans — warn so the operator turns one off. if "langfuse" in get_enabled_tracing_providers(): logger.warning("MONOCLE_TRACING is enabled alongside Langfuse; both need the global OpenTelemetry provider and only one can win. Enable only one of them.") - exporters = get_tracing_config().monocle.exporters - from monocle_apptrace import setup_monocle_telemetry + exporters = monocle.exporters + + # Any exporter other than `file` moves trace data (prompts, tool inputs and + # outputs, completions) beyond the local .monocle/ directory, e.g. to an + # external collector (okahu, s3, ...). Warn loudly so it can't happen + # unnoticed. + non_file = [e.strip() for e in exporters.split(",") if e.strip() and e.strip() != "file"] + if non_file: + logger.warning( + "Monocle is exporting trace data (prompts, tool inputs/outputs, completions) beyond the local .monocle/ file via: %s. Make sure that destination is trusted.", + ", ".join(non_file), + ) + + try: + from monocle_apptrace import setup_monocle_telemetry + except ImportError as exc: + raise RuntimeError("MONOCLE_TRACING is enabled but monocle_apptrace is not installed. Install the optional extra: pip install 'deer-flow[monocle]'.") from exc setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) logger.info("Monocle telemetry enabled (exporters=%s)", exporters) diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 746d5badfdd..d78a19e270a 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -65,6 +65,9 @@ postgres = [ redis = ["redis>=5.0.0"] pymupdf = ["pymupdf4llm>=0.0.17"] boxlite = ["boxlite>=0.9.7"] +# Agent observability (Monocle). Optional so a default install stays free of the +# OpenTelemetry stack; only pulled in when MONOCLE_TRACING is used. +monocle = ["monocle_apptrace>=0.8.8"] [build-system] requires = ["hatchling"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 61738c74977..ce6d12c8e78 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,13 +22,13 @@ dependencies = [ "pyjwt>=2.13.0", "email-validator>=2.0.0", "e2b-code-interpreter>=2.8.1", - "monocle_apptrace>=0.8.8", ] [project.optional-dependencies] postgres = ["deerflow-harness[postgres]"] redis = ["deerflow-harness[redis]"] discord = ["discord.py>=2.7.0"] +monocle = ["deerflow-harness[monocle]"] [dependency-groups] dev = [ @@ -37,6 +37,9 @@ dev = [ "pytest>=9.0.3", "pytest-asyncio>=1.3.0", "ruff>=0.14.11", + # Monocle tracer (also the deerflow-harness[monocle] extra); kept in the dev + # group so the tracing tests can import it without forcing it onto installs. + "monocle_apptrace>=0.8.8", # redis is an optional runtime extra (deerflow-harness[redis]); pin it in the # dev group so the stream-bridge tests can always import/exercise the redis # bridge without forcing it onto production installs. diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index d43df5d9ca5..00b551304bf 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -1,22 +1,37 @@ """Tests for Monocle telemetry setup. Covers the config gate (``MONOCLE_TRACING`` default off / toggle on), the setup -helper's behavior, and the regression that importing ``deerflow.agents`` no -longer sets up telemetry at import time. +helper's behavior (off-box exporter warning, Langfuse-conflict warning, exporter +validation, idempotency), and the regression that importing ``deerflow.agents`` +no longer sets up telemetry at import time. """ from __future__ import annotations +import logging +import subprocess +import sys +import textwrap + import pytest from deerflow.config import is_monocle_tracing_enabled from deerflow.config.tracing_config import get_tracing_config, reset_tracing_config from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled +_TRACING_ENV = ( + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", + "LANGFUSE_TRACING", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", +) + @pytest.fixture(autouse=True) def clear_monocle_env(monkeypatch): - for name in ("MONOCLE_TRACING", "MONOCLE_EXPORTERS"): + for name in _TRACING_ENV: monkeypatch.delenv(name, raising=False) reset_tracing_config() yield @@ -64,13 +79,126 @@ def test_custom_exporters(monkeypatch): assert captured["monocle_exporters_list"] == "file,console" -def test_no_import_time_setup(): +def test_warns_on_non_file_exporter(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,s3") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + warnings = [r.message for r in caplog.records if "beyond the local" in r.message] + assert warnings, "expected an off-box exporter warning" + assert "s3" in warnings[0] + + +def test_no_off_box_warning_for_file_exporter(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") # default exporter is file + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert not any("beyond the local" in r.message for r in caplog.records) + + +def test_warns_when_langfuse_also_enabled(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert any("Langfuse" in r.message and "OpenTelemetry" in r.message for r in caplog.records) + + +def test_no_langfuse_warning_when_only_monocle(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert not any("Langfuse" in r.message for r in caplog.records) + + +def test_rejects_unknown_exporter(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "fle") + reset_tracing_config() + with pytest.raises(ValueError, match="unknown exporter"): + setup_monocle_tracing_if_enabled() + + +def test_okahu_exporter_requires_api_key(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + reset_tracing_config() + with pytest.raises(ValueError, match="OKAHU_API_KEY"): + setup_monocle_tracing_if_enabled() + + +def test_okahu_exporter_with_api_key_ok(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + monkeypatch.setenv("OKAHU_API_KEY", "okh_test") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + assert setup_monocle_tracing_if_enabled() is True + + +def test_no_import_time_setup(monkeypatch): """Regression: importing deerflow.agents must not install telemetry. The setup call used to live at module import in ``deerflow/agents/__init__``. - It now happens only via the gateway lifespan, so the package namespace must - not even carry ``setup_monocle_telemetry``. + It now happens only via the gateway lifespan, so re-importing the package + must neither expose ``setup_monocle_telemetry`` nor replace the global OTel + ``TracerProvider`` (which is what ``setup_monocle_telemetry`` does). """ + from opentelemetry import trace + + provider_before = trace.get_tracer_provider() + + # Force a fresh import so ``__init__`` actually re-executes. + for name in [m for m in list(sys.modules) if m == "deerflow.agents" or m.startswith("deerflow.agents.")]: + monkeypatch.delitem(sys.modules, name, raising=False) import deerflow.agents as agents assert not hasattr(agents, "setup_monocle_telemetry") + assert trace.get_tracer_provider() is provider_before + + +def test_double_invoke_is_idempotent(): + """Calling setup twice must not double-instrument. + + Exercises upstream ``check_duplicate_setup`` with the real tracer (no mock). + Run in a subprocess so the process-global OTel provider it installs never + leaks into the rest of the suite. + """ + script = textwrap.dedent( + """ + import os + os.environ["MONOCLE_TRACING"] = "true" + os.environ["MONOCLE_EXPORTERS"] = "console" # avoid writing .monocle/ files + from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled + from monocle_apptrace.instrumentation.common.instrumentor import get_monocle_instrumentor + + assert setup_monocle_tracing_if_enabled() is True + first = get_monocle_instrumentor() + assert first is not None + assert setup_monocle_tracing_if_enabled() is True + assert get_monocle_instrumentor() is first # no second provider installed + print("IDEMPOTENT_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "IDEMPOTENT_OK" in result.stdout diff --git a/backend/uv.lock b/backend/uv.lock index a9fb7d36fac..eb3d35036b7 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -786,7 +786,6 @@ dependencies = [ { name = "langgraph-sdk" }, { name = "lark-oapi" }, { name = "markdown-to-mrkdwn" }, - { name = "monocle-apptrace" }, { name = "pyjwt" }, { name = "python-multipart" }, { name = "python-telegram-bot" }, @@ -800,6 +799,9 @@ dependencies = [ discord = [ { name = "discord-py" }, ] +monocle = [ + { name = "deerflow-harness", extra = ["monocle"] }, +] postgres = [ { name = "deerflow-harness", extra = ["postgres"] }, ] @@ -810,6 +812,7 @@ redis = [ [package.dev-dependencies] dev = [ { name = "blockbuster" }, + { name = "monocle-apptrace" }, { name = "prompt-toolkit" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -822,6 +825,7 @@ dev = [ requires-dist = [ { name = "bcrypt", specifier = ">=4.0.0" }, { name = "deerflow-harness", editable = "packages/harness" }, + { name = "deerflow-harness", extras = ["monocle"], marker = "extra == 'monocle'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["redis"], marker = "extra == 'redis'", editable = "packages/harness" }, { name = "dingtalk-stream", specifier = ">=0.24.3" }, @@ -833,7 +837,6 @@ requires-dist = [ { name = "langgraph-sdk", specifier = ">=0.1.51" }, { name = "lark-oapi", specifier = ">=1.4.0" }, { name = "markdown-to-mrkdwn", specifier = ">=0.3.1" }, - { name = "monocle-apptrace", specifier = ">=0.8.8" }, { name = "pyjwt", specifier = ">=2.13.0" }, { name = "python-multipart", specifier = ">=0.0.31" }, { name = "python-telegram-bot", specifier = ">=21.0" }, @@ -842,11 +845,12 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, { name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" }, ] -provides-extras = ["postgres", "redis", "discord"] +provides-extras = ["postgres", "redis", "discord", "monocle"] [package.metadata.requires-dev] dev = [ { name = "blockbuster", specifier = ">=1.5.26,<1.6" }, + { name = "monocle-apptrace", specifier = ">=0.8.8" }, { name = "prompt-toolkit", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, @@ -901,6 +905,9 @@ dependencies = [ boxlite = [ { name = "boxlite" }, ] +monocle = [ + { name = "monocle-apptrace" }, +] ollama = [ { name = "langchain-ollama" }, ] @@ -955,6 +962,7 @@ requires-dist = [ { name = "langgraph-sdk", specifier = ">=0.1.51" }, { name = "markdownify", specifier = ">=1.2.2" }, { name = "markitdown", extras = ["all", "xlsx"], specifier = ">=0.0.1a2" }, + { name = "monocle-apptrace", marker = "extra == 'monocle'", specifier = ">=0.8.8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.3.3" }, { name = "psycopg-pool", marker = "extra == 'postgres'", specifier = ">=3.3.0" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -967,7 +975,7 @@ requires-dist = [ { name = "textual", marker = "extra == 'tui'", specifier = ">=0.80" }, { name = "tiktoken", specifier = ">=0.8.0" }, ] -provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite"] +provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite", "monocle"] [[package]] name = "defusedxml" diff --git a/config.example.yaml b/config.example.yaml index e8681db60ef..7736f3f7ef7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -31,7 +31,7 @@ logging: format: text # ============================================================================ -# Monocle telemetry +# Tracing / Observability (Monocle) # ============================================================================ # Optional agent tracing via Monocle. Configured through environment variables # (like LangSmith/Langfuse), not a config.yaml key, and OFF by default. When From c2910f2a95573b42bf22e0b9868b5efaeb61f8fa Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Fri, 10 Jul 2026 22:01:16 -0700 Subject: [PATCH 05/11] docs: align Monocle README section with the other tracing providers Lead with what Monocle is and captures, drop the install step (the dev group already ships monocle_apptrace via uv sync; unusual installs get the RuntimeError), and point the missing-package error at the repo-native command (uv sync --extra monocle / deerflow-harness[monocle]). Co-Authored-By: Claude Fable 5 --- README.md | 12 +++--------- backend/packages/harness/deerflow/tracing/monocle.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 40a4b02b460..126f24d601f 100644 --- a/README.md +++ b/README.md @@ -596,13 +596,9 @@ These are injected into `RunnableConfig.metadata` at the graph invocation root f #### Monocle Tracing -DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle) telemetry for its LangGraph/LangChain runs. Monocle is an optional extra, so install it when you want tracing: +DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle), an OpenTelemetry-based tracer for agentic applications. It records each run end-to-end — LLM calls, agent steps, tool and MCP invocations — with their inputs, outputs, timings, and token counts. -```bash -pip install "deer-flow[monocle]" -``` - -Then add the following to your `.env` file: +Add the following to your `.env` file: ```bash MONOCLE_TRACING=true @@ -610,9 +606,7 @@ MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: OKAHU_API_KEY=okh_xxxxxxxx # required only for the `okahu` exporter ``` -Monocle is off by default and is initialized at Gateway startup when enabled, so it covers runs served by the Gateway (the embedded `DeerFlowClient` and the TUI do not start it). The `file` exporter writes one trace per run to `.monocle/`; open those in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. - -Send your Monocle traces to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze them across many runs. +Each run writes one trace file to `.monocle/`; open it in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. Connect to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze traces across runs and run trace-based and agentic evaluations (via the `okahu` exporter). #### Using Multiple Providers diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index efa65265739..8dca2f68aba 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -50,7 +50,7 @@ def setup_monocle_tracing_if_enabled() -> bool: try: from monocle_apptrace import setup_monocle_telemetry except ImportError as exc: - raise RuntimeError("MONOCLE_TRACING is enabled but monocle_apptrace is not installed. Install the optional extra: pip install 'deer-flow[monocle]'.") from exc + raise RuntimeError("MONOCLE_TRACING is enabled but monocle_apptrace is not installed. Install the 'monocle' extra: `uv sync --extra monocle` in backend/, or `pip install 'deerflow-harness[monocle]'`.") from exc setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) logger.info("Monocle telemetry enabled (exporters=%s)", exporters) From c1f8bff35437371a1b063f66f3360438a893eeec Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Sat, 11 Jul 2026 02:05:30 -0700 Subject: [PATCH 06/11] Address review: verified Langfuse coexistence, lifespan test, scope docs Responds to the third review round. The Langfuse conflict claim was wrong, verified empirically against langfuse 4.5.1 in both init orders: whichever library initializes second reuses the existing global TracerProvider and attaches its own span processor, so neither side loses spans. Dropped the warning and its tests, corrected the README, AGENTS.md, and config.example.yaml statements, and pinned the verified behavior with test_coexists_with_langfuse (real monocle + real langfuse in a subprocess, no mocks). One honest caveat documented: both processors see all spans, so Monocle's exporters also capture Langfuse's spans when both are enabled. Also from the review: - Document the Gateway-only scope in AGENTS.md: the lifespan is the sole call site, so the embedded DeerFlowClient and TUI are not instrumented; embedded users call setup_monocle_tracing_if_enabled() themselves. - Add test_gateway_lifespan_initializes_monocle pinning the lifespan wiring. - Comment why MonocleTracingConfig.is_configured is intentionally coarser than LangSmith/Langfuse (composite validation lives in validate() at startup). - Note that monocle_exporters_list takes the comma-separated string as-is. - Module-level importorskip("monocle_apptrace") so minimal installs collect the test module cleanly. --- README.md | 2 +- backend/AGENTS.md | 4 +- .../harness/deerflow/config/tracing_config.py | 4 + .../harness/deerflow/tracing/monocle.py | 13 +-- backend/tests/test_monocle_tracing.py | 105 ++++++++++++++---- config.example.yaml | 5 +- 6 files changed, 101 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 126f24d601f..07c7d04bea8 100644 --- a/README.md +++ b/README.md @@ -610,7 +610,7 @@ Each run writes one trace file to `.monocle/`; open it in the [Monocle VS Code e #### Using Multiple Providers -LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback, so it runs alongside LangSmith but not Langfuse. +LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback; Langfuse shares that provider, so all three can run together. With both Monocle and Langfuse enabled, Monocle's exporters also capture Langfuse's spans. For Docker deployments, tracing is disabled by default. Set `LANGSMITH_TRACING=true` and `LANGSMITH_API_KEY` in your `.env` to enable it. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 8812484ecec..91c5e2f1a47 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -705,11 +705,11 @@ LangSmith and Langfuse are both supported. The wiring lives in two layers: Returns `{}` when Langfuse is not in the enabled providers — LangSmith-only deployments are unaffected. Set `DEER_FLOW_ENV` (or `ENVIRONMENT`) to tag traces by deployment environment. Tests live in `tests/test_tracing_factory.py`, `tests/test_tracing_metadata.py`, `tests/test_worker_langfuse_metadata.py`, `tests/test_client_langfuse_metadata.py`, and `tests/test_subagent_executor.py::TestSubagentTracingWiring`. -**Monocle telemetry** is a third provider, structurally unlike LangSmith/Langfuse. It is **not** a LangChain callback: `tracing/monocle.py::setup_monocle_tracing_if_enabled()` calls `monocle_apptrace.setup_monocle_telemetry()` once, which installs a **process-global OTel `TracerProvider`**, patches span serialization, and auto-instruments the openai/langchain/langgraph clients. Because that is a one-time, process-global side effect (not a per-run callback), it is initialized from the **Gateway lifespan** (`app/gateway/app.py`) — never from `build_tracing_callbacks()` — and it is **off by default**. The setup call was deliberately moved out of `agents/__init__.py`, so `import deerflow.agents` must never start tracing (pinned by `tests/test_monocle_tracing.py::test_no_import_time_setup`). +**Monocle telemetry** is a third provider, structurally unlike LangSmith/Langfuse. It is **not** a LangChain callback: `tracing/monocle.py::setup_monocle_tracing_if_enabled()` calls `monocle_apptrace.setup_monocle_telemetry()` once, which installs a **process-global OTel `TracerProvider`**, patches span serialization, and auto-instruments the openai/langchain/langgraph clients. Because that is a one-time, process-global side effect (not a per-run callback), it is initialized from the **Gateway lifespan** (`app/gateway/app.py`) — never from `build_tracing_callbacks()` — and it is **off by default**. The setup call was deliberately moved out of `agents/__init__.py`, so `import deerflow.agents` must never start tracing (pinned by `tests/test_monocle_tracing.py::test_no_import_time_setup`). The Gateway lifespan is the **sole call site** (pinned by `test_gateway_lifespan_initializes_monocle`), so unlike LangSmith/Langfuse — which attach at the graph roots and cover every path — the embedded `DeerFlowClient` and the TUI are not instrumented; embedded users who want Monocle traces call `setup_monocle_tracing_if_enabled()` themselves before running the agent. Unlike the Langfuse metadata above, DeerFlow injects **no** per-run fields into Monocle traces — the only attribute it sets is `workflow_name="deer-flow"`; every span attribute (`span.type`, `entity.*`, token usage, span inputs/outputs, `scope.agentic.session`) is produced by Monocle's own metamodel and auto-instrumentation, so there is no DeerFlow trace-attribute layer to maintain here. -Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and never force-overrides an existing global provider, so the wrapper only gates on config. It also **warns** when Langfuse (v4) is enabled: only one library can own the single global OpenTelemetry provider, and Monocle initializes here at startup (before Langfuse's per-run handler), so with both on Langfuse loses its spans — enable one OTel tracer. (LangSmith is a plain callback and coexists fine.) Tests: `tests/test_monocle_tracing.py`. +Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and never force-overrides an existing global provider, so the wrapper only gates on config. Coexistence with Langfuse (v4, also OTel-based) is **verified**: whichever library initializes second reuses the existing global `TracerProvider` and attaches its own span processor, so neither side loses spans (pinned by `test_coexists_with_langfuse`). Both processors see all spans, so Monocle's exporters also capture Langfuse's spans when both are enabled. (LangSmith is a plain callback and coexists trivially.) Tests: `tests/test_monocle_tracing.py`. ### Config Schema diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index e6c350be2e6..db2c61b7c5d 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -59,6 +59,10 @@ class MonocleTracingConfig(BaseModel): @property def is_configured(self) -> bool: + # Intentionally coarser than LangSmith/Langfuse (which fold credential + # presence in): whether Monocle needs credentials depends on the exporter + # mix, so that composite check lives in validate(), called from setup at + # Gateway startup — where a bad config fails loudly without touching runs. return self.enabled def validate(self) -> None: diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index 8dca2f68aba..64010af04d5 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -5,7 +5,6 @@ import logging from deerflow.config import ( - get_enabled_tracing_providers, get_tracing_config, is_monocle_tracing_enabled, ) @@ -28,12 +27,11 @@ def setup_monocle_tracing_if_enabled() -> bool: # per-run callback path) so a config typo never breaks agent runs. monocle.validate() - # Only one OpenTelemetry provider can own the process. Monocle initializes here (at - # startup), before Langfuse's per-run handler, so enabling both means Langfuse loses - # its spans — warn so the operator turns one off. - if "langfuse" in get_enabled_tracing_providers(): - logger.warning("MONOCLE_TRACING is enabled alongside Langfuse; both need the global OpenTelemetry provider and only one can win. Enable only one of them.") - + # Coexistence with Langfuse (v4, also OTel-based) is verified: whichever + # library initializes second reuses the existing global TracerProvider and + # attaches its own span processor, so neither side loses spans (see + # test_coexists_with_langfuse). Both processors see all spans, so Monocle's + # exporters also capture Langfuse's spans when both are enabled. exporters = monocle.exporters # Any exporter other than `file` moves trace data (prompts, tool inputs and @@ -52,6 +50,7 @@ def setup_monocle_tracing_if_enabled() -> bool: except ImportError as exc: raise RuntimeError("MONOCLE_TRACING is enabled but monocle_apptrace is not installed. Install the 'monocle' extra: `uv sync --extra monocle` in backend/, or `pip install 'deerflow-harness[monocle]'`.") from exc + # monocle_exporters_list takes the comma-separated string as-is (monocle_apptrace's API). setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) logger.info("Monocle telemetry enabled (exporters=%s)", exporters) return True diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index 00b551304bf..e7c29785227 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -1,20 +1,28 @@ """Tests for Monocle telemetry setup. Covers the config gate (``MONOCLE_TRACING`` default off / toggle on), the setup -helper's behavior (off-box exporter warning, Langfuse-conflict warning, exporter -validation, idempotency), and the regression that importing ``deerflow.agents`` -no longer sets up telemetry at import time. +helper's behavior (off-box exporter warning, exporter validation, idempotency, +Langfuse coexistence), the Gateway-lifespan wiring, and the regression that +importing ``deerflow.agents`` no longer sets up telemetry at import time. """ from __future__ import annotations +import asyncio import logging import subprocess import sys import textwrap +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch import pytest +# monocle_apptrace is an optional extra (pinned in the dev group); skip the whole +# module in minimal installs instead of erroring at collection. +pytest.importorskip("monocle_apptrace") + from deerflow.config import is_monocle_tracing_enabled from deerflow.config.tracing_config import get_tracing_config, reset_tracing_config from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled @@ -104,29 +112,44 @@ def test_no_off_box_warning_for_file_exporter(monkeypatch, caplog): assert not any("beyond the local" in r.message for r in caplog.records) -def test_warns_when_langfuse_also_enabled(monkeypatch, caplog): - monkeypatch.setenv("MONOCLE_TRACING", "true") - monkeypatch.setenv("LANGFUSE_TRACING", "true") - monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") - monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") - reset_tracing_config() - monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) +def test_coexists_with_langfuse(): + """Monocle and Langfuse (v4, OTel-based) share the global provider without span loss. - with caplog.at_level(logging.WARNING): - assert setup_monocle_tracing_if_enabled() is True - - assert any("Langfuse" in r.message and "OpenTelemetry" in r.message for r in caplog.records) + Verified against the installed langfuse: whichever library initializes second + reuses the existing global ``TracerProvider`` and attaches its own span + processor, so both sides keep exporting. Runs the real setup (no mocks) in a + subprocess so the process-global provider never leaks into the suite. + """ + script = textwrap.dedent( + """ + import os + os.environ["MONOCLE_TRACING"] = "true" + os.environ["MONOCLE_EXPORTERS"] = "console" + os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-test" + os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-test" + os.environ["LANGFUSE_HOST"] = "http://127.0.0.1:9" # unreachable; offline test + from opentelemetry import trace -def test_no_langfuse_warning_when_only_monocle(monkeypatch, caplog): - monkeypatch.setenv("MONOCLE_TRACING", "true") - reset_tracing_config() - monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled - with caplog.at_level(logging.WARNING): + # Gateway order: Monocle at startup, Langfuse per-run afterwards. assert setup_monocle_tracing_if_enabled() is True + provider = trace.get_tracer_provider() + + from langfuse import Langfuse - assert not any("Langfuse" in r.message for r in caplog.records) + Langfuse(tracing_enabled=True) + + assert trace.get_tracer_provider() is provider # provider not replaced + names = [type(p).__name__ for p in provider._active_span_processor._span_processors] + assert any("Langfuse" in n for n in names), names # Langfuse attached alongside Monocle + print("COEXIST_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "COEXIST_OK" in result.stdout def test_rejects_unknown_exporter(monkeypatch): @@ -202,3 +225,45 @@ def test_double_invoke_is_idempotent(): result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) assert result.returncode == 0, result.stderr assert "IDEMPOTENT_OK" in result.stdout + + +def test_gateway_lifespan_initializes_monocle(): + """The Gateway lifespan is the sole Monocle call site; pin that wiring. + + Mirrors the patching in ``test_gateway_lifespan_shutdown.py`` so the lifespan + can be driven directly, and asserts the setup helper runs during startup. + """ + from fastapi import FastAPI + + from app.gateway.app import lifespan + + @asynccontextmanager + async def _noop_langgraph_runtime(_app, _startup_config): + yield + + startup_config = SimpleNamespace(log_level="INFO", memory=SimpleNamespace(token_counting="char")) + fake_service = MagicMock() + fake_service.get_status = MagicMock(return_value={}) + + async def fake_start(_startup_config): + return fake_service + + setup_spy = MagicMock(return_value=False) + + with ( + patch("app.gateway.app.get_app_config", return_value=startup_config), + patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), + patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), + patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), + patch("app.channels.service.start_channel_service", side_effect=fake_start), + patch("app.channels.service.stop_channel_service", AsyncMock()), + ): + + async def drive() -> None: + async with lifespan(FastAPI()): + pass + + asyncio.run(drive()) + + setup_spy.assert_called_once_with() diff --git a/config.example.yaml b/config.example.yaml index 7736f3f7ef7..34df0f5123e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -52,8 +52,9 @@ logging: # external collector. Enable only exporters whose data handling you trust, and # be aware that prompts/outputs may contain sensitive content. # -# Note: Monocle and Langfuse both install a global OpenTelemetry TracerProvider; -# enable only one OTel-based tracer at a time (see backend/AGENTS.md → Tracing). +# Note: Monocle shares the global OpenTelemetry TracerProvider with Langfuse, so +# both can be enabled together; Monocle's exporters then also capture Langfuse's +# spans (see backend/AGENTS.md → Tracing). # ============================================================================ # Token Usage From 93f968d266f95d21b38103c49eadb775a974065a Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Sat, 11 Jul 2026 03:06:15 -0700 Subject: [PATCH 07/11] docs: reword Monocle intro sentence --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 768f78f49a8..fb3da8821d5 100644 --- a/README.md +++ b/README.md @@ -596,7 +596,7 @@ These are injected into `RunnableConfig.metadata` at the graph invocation root f #### Monocle Tracing -DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle), an OpenTelemetry-based tracer for agentic applications. It records each run end-to-end — LLM calls, agent steps, tool and MCP invocations — with their inputs, outputs, timings, and token counts. +DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle), an OpenTelemetry-based tracer for agentic applications. It records each run end-to-end: LLM calls, agent steps, and tool and MCP invocations, with their inputs, outputs, timings, and token counts. Add the following to your `.env` file: From e0122c997e312b55d19e4c909678c8127c559efa Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Sat, 11 Jul 2026 08:45:22 -0700 Subject: [PATCH 08/11] fix(tests): run the import-time regression in a subprocess test_no_import_time_setup deleted deerflow.agents* from sys.modules and re-imported to force __init__ to re-execute. The re-import creates new module objects, and restoring the old sys.modules entries afterwards leaves the parent package's attribute bindings pointing at the new ones, so any later test that resolves a deerflow.agents.* dotted path (monkeypatch.setattr in test_summarization_middleware, test_thread_data_middleware, and others) failed with "module 'deerflow.agents' has no attribute ...". Run the check in a subprocess instead: the import is genuinely fresh, the assertion is stronger (the provider must still be the SDK-less proxy, proving nothing was installed at any point), and no module identity leaks into the rest of the suite. --- backend/tests/test_monocle_tracing.py | 34 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index e7c29785227..1d27b868f05 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -178,25 +178,33 @@ def test_okahu_exporter_with_api_key_ok(monkeypatch): assert setup_monocle_tracing_if_enabled() is True -def test_no_import_time_setup(monkeypatch): +def test_no_import_time_setup(): """Regression: importing deerflow.agents must not install telemetry. The setup call used to live at module import in ``deerflow/agents/__init__``. - It now happens only via the gateway lifespan, so re-importing the package - must neither expose ``setup_monocle_telemetry`` nor replace the global OTel - ``TracerProvider`` (which is what ``setup_monocle_telemetry`` does). + It now happens only via the gateway lifespan, so a plain import must neither + expose ``setup_monocle_telemetry`` nor install a global OTel + ``TracerProvider`` (which is what ``setup_monocle_telemetry`` does). Runs in + a subprocess so the import is genuinely fresh and, unlike deleting + ``sys.modules`` entries in-process, cannot corrupt module identity for the + rest of the suite. """ - from opentelemetry import trace - - provider_before = trace.get_tracer_provider() + script = textwrap.dedent( + """ + from opentelemetry import trace + from opentelemetry.trace import ProxyTracerProvider - # Force a fresh import so ``__init__`` actually re-executes. - for name in [m for m in list(sys.modules) if m == "deerflow.agents" or m.startswith("deerflow.agents.")]: - monkeypatch.delitem(sys.modules, name, raising=False) - import deerflow.agents as agents + import deerflow.agents as agents - assert not hasattr(agents, "setup_monocle_telemetry") - assert trace.get_tracer_provider() is provider_before + assert not hasattr(agents, "setup_monocle_telemetry") + # Still the SDK-less default proxy: no provider was installed on import. + assert isinstance(trace.get_tracer_provider(), ProxyTracerProvider) + print("IMPORT_CLEAN_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "IMPORT_CLEAN_OK" in result.stdout def test_double_invoke_is_idempotent(): From 6c3654857ac6118d40fa27ab4c862d99a1271c89 Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Mon, 13 Jul 2026 16:19:45 -0700 Subject: [PATCH 09/11] Address review: console warning scope, embedded hint, honest naming, doc alignment Responds to the post-approval review round: - Scope the off-box exporter warning to the remote exporters (okahu, s3, blob, gcs): console writes to local stdout and no longer trips it. config.example.yaml's data-handling note now distinguishes file / console / remote likewise. - Rename MonocleTracingConfig.is_configured to is_enabled so the boolean reads as what it checks; the exporter-dependent credential check stays in validate(), run at Gateway startup. - Hint on the embedded path: build_tracing_callbacks() logs a debug line when MONOCLE_TRACING is set but setup never ran in this process, so embedded DeerFlowClient/TUI users are not left with silent no-op tracing. Backed by a process-global setup flag. - Re-export setup_monocle_tracing_if_enabled from deerflow.tracing, matching the package convention. - Note the deliberate fail-open-at-startup contrast with LangSmith/Langfuse in the lifespan, and the OTel SDK-internals dependency in the coexistence test. - Test hygiene: clear MONOCLE_* env in the tracing config/factory fixtures; reset the setup flag in the monocle test fixture; reword the README Langfuse-spans claim as the shared-provider inference it is. - Document that .monocle/ trace files are never rotated or cleaned up. --- README.md | 2 +- backend/app/gateway/app.py | 2 + .../harness/deerflow/config/tracing_config.py | 10 ++-- .../harness/deerflow/tracing/__init__.py | 2 + .../harness/deerflow/tracing/factory.py | 11 ++++ .../harness/deerflow/tracing/monocle.py | 22 +++++--- backend/tests/test_monocle_tracing.py | 54 +++++++++++++++++++ backend/tests/test_tracing_config.py | 3 ++ backend/tests/test_tracing_factory.py | 3 ++ config.example.yaml | 11 ++-- 10 files changed, 102 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fb3da8821d5..4f3e24843db 100644 --- a/README.md +++ b/README.md @@ -610,7 +610,7 @@ Each run writes one trace file to `.monocle/`; open it in the [Monocle VS Code e #### Using Multiple Providers -LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback; Langfuse shares that provider, so all three can run together. With both Monocle and Langfuse enabled, Monocle's exporters also capture Langfuse's spans. +LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback; Langfuse shares that provider, so all three can run together. Because both span processors sit on the same shared provider, Monocle's exporters also see Langfuse's spans when both are enabled. For Docker deployments, tracing is disabled by default. Set `LANGSMITH_TRACING=true` and `LANGSMITH_API_KEY` in your `.env` to enable it. diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 44691c97e2a..ba29762de0a 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -193,6 +193,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # Agent observability (Monocle). Off by default; enabled with # MONOCLE_TRACING. Initialized here at startup — not at import time — so a # plain `import deerflow.agents` never installs a process-global tracer. + # Unlike LangSmith/Langfuse, whose validation failures abort the agent run, + # a bad Monocle config only logs: the Gateway keeps serving without tracing. try: setup_monocle_tracing_if_enabled() except Exception: # pragma: no cover - observability must never break startup diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index db2c61b7c5d..89768dcba60 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -58,11 +58,9 @@ class MonocleTracingConfig(BaseModel): okahu_api_key: str | None = Field(...) @property - def is_configured(self) -> bool: - # Intentionally coarser than LangSmith/Langfuse (which fold credential - # presence in): whether Monocle needs credentials depends on the exporter - # mix, so that composite check lives in validate(), called from setup at - # Gateway startup — where a bad config fails loudly without touching runs. + def is_enabled(self) -> bool: + # Unlike the siblings' is_configured, no credential check here: that is + # exporter-dependent and lives in validate(), run at Gateway startup. return self.enabled def validate(self) -> None: @@ -191,7 +189,7 @@ def is_monocle_tracing_enabled() -> bool: process-global instrumentor activated at startup, not a per-run LangChain callback. """ - return get_tracing_config().monocle.is_configured + return get_tracing_config().monocle.is_enabled def reset_tracing_config() -> None: diff --git a/backend/packages/harness/deerflow/tracing/__init__.py b/backend/packages/harness/deerflow/tracing/__init__.py index 6d00e9c69b1..cfec1f256c2 100644 --- a/backend/packages/harness/deerflow/tracing/__init__.py +++ b/backend/packages/harness/deerflow/tracing/__init__.py @@ -1,8 +1,10 @@ from .factory import build_tracing_callbacks from .metadata import build_langfuse_trace_metadata, inject_langfuse_metadata +from .monocle import setup_monocle_tracing_if_enabled __all__ = [ "build_langfuse_trace_metadata", "build_tracing_callbacks", "inject_langfuse_metadata", + "setup_monocle_tracing_if_enabled", ] diff --git a/backend/packages/harness/deerflow/tracing/factory.py b/backend/packages/harness/deerflow/tracing/factory.py index a8ef8572125..896f8f316ef 100644 --- a/backend/packages/harness/deerflow/tracing/factory.py +++ b/backend/packages/harness/deerflow/tracing/factory.py @@ -1,12 +1,17 @@ from __future__ import annotations +import logging from typing import Any from deerflow.config import ( get_enabled_tracing_providers, get_tracing_config, + is_monocle_tracing_enabled, validate_enabled_tracing_providers, ) +from deerflow.tracing.monocle import is_monocle_setup_completed + +logger = logging.getLogger(__name__) def _create_langsmith_tracer(config) -> Any: @@ -32,6 +37,12 @@ def _create_langfuse_handler(config) -> Any: def build_tracing_callbacks() -> list[Any]: """Build callbacks for all explicitly enabled tracing providers.""" validate_enabled_tracing_providers() + # Monocle is not a callback provider; this per-run path is just where an + # embedded process that skipped Gateway-lifespan setup can be told about it. + if is_monocle_tracing_enabled() and not is_monocle_setup_completed(): + logger.debug( + "MONOCLE_TRACING is set but Monocle is not initialized in this process — only the Gateway lifespan runs setup automatically; embedded/TUI callers must call deerflow.tracing.setup_monocle_tracing_if_enabled() themselves." + ) enabled_providers = get_enabled_tracing_providers() if not enabled_providers: return [] diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index 64010af04d5..728f36a5be1 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -11,6 +11,15 @@ logger = logging.getLogger(__name__) +# Read by build_tracing_callbacks() to hint embedded/TUI processes that +# enabled MONOCLE_TRACING but never ran the Gateway-lifespan setup. +_setup_completed = False + + +def is_monocle_setup_completed() -> bool: + """Whether :func:`setup_monocle_tracing_if_enabled` ran in this process.""" + return _setup_completed + def setup_monocle_tracing_if_enabled() -> bool: """Initialize Monocle telemetry when ``MONOCLE_TRACING`` is enabled; a no-op otherwise. @@ -34,15 +43,12 @@ def setup_monocle_tracing_if_enabled() -> bool: # exporters also capture Langfuse's spans when both are enabled. exporters = monocle.exporters - # Any exporter other than `file` moves trace data (prompts, tool inputs and - # outputs, completions) beyond the local .monocle/ directory, e.g. to an - # external collector (okahu, s3, ...). Warn loudly so it can't happen - # unnoticed. - non_file = [e.strip() for e in exporters.split(",") if e.strip() and e.strip() != "file"] - if non_file: + # `console` stays on local stdout, so only the remote exporters are flagged. + off_box = [e.strip() for e in exporters.split(",") if e.strip() and e.strip() not in ("file", "console")] + if off_box: logger.warning( "Monocle is exporting trace data (prompts, tool inputs/outputs, completions) beyond the local .monocle/ file via: %s. Make sure that destination is trusted.", - ", ".join(non_file), + ", ".join(off_box), ) try: @@ -52,5 +58,7 @@ def setup_monocle_tracing_if_enabled() -> bool: # monocle_exporters_list takes the comma-separated string as-is (monocle_apptrace's API). setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) + global _setup_completed + _setup_completed = True logger.info("Monocle telemetry enabled (exporters=%s)", exporters) return True diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index 1d27b868f05..0b897594b22 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -41,6 +41,9 @@ def clear_monocle_env(monkeypatch): for name in _TRACING_ENV: monkeypatch.delenv(name, raising=False) + # The setup-completed flag is process-global; reset it so a test that runs + # (mocked) setup cannot change how later tests observe the embedded hint. + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) reset_tracing_config() yield reset_tracing_config() @@ -112,6 +115,19 @@ def test_no_off_box_warning_for_file_exporter(monkeypatch, caplog): assert not any("beyond the local" in r.message for r in caplog.records) +def test_no_off_box_warning_for_console_exporter(monkeypatch, caplog): + """``console`` writes to local stdout, so it must not trip the off-box warning.""" + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,console") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert not any("beyond the local" in r.message for r in caplog.records) + + def test_coexists_with_langfuse(): """Monocle and Langfuse (v4, OTel-based) share the global provider without span loss. @@ -142,6 +158,10 @@ def test_coexists_with_langfuse(): Langfuse(tracing_enabled=True) assert trace.get_tracer_provider() is provider # provider not replaced + # NOTE: reaches into OTel SDK internals (no public API lists a + # provider's span processors). The SDK is pinned by uv.lock; if a bump + # renames these attributes, update this introspection — the coexistence + # behavior itself is unaffected. names = [type(p).__name__ for p in provider._active_span_processor._span_processors] assert any("Langfuse" in n for n in names), names # Langfuse attached alongside Monocle print("COEXIST_OK") @@ -178,6 +198,40 @@ def test_okahu_exporter_with_api_key_ok(monkeypatch): assert setup_monocle_tracing_if_enabled() is True +def test_embedded_hint_when_enabled_but_uninitialized(monkeypatch, caplog): + """``build_tracing_callbacks()`` hints when Monocle is enabled but setup never ran. + + The embedded ``DeerFlowClient`` and the TUI never hit the Gateway lifespan, + so this debug line is the only in-process signal explaining why no Monocle + traces appear. + """ + from deerflow.tracing import build_tracing_callbacks + + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) + + with caplog.at_level(logging.DEBUG): + assert build_tracing_callbacks() == [] + + assert any("not initialized in this process" in r.message for r in caplog.records) + + +def test_no_embedded_hint_after_setup(monkeypatch, caplog): + from deerflow.tracing import build_tracing_callbacks + + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + assert setup_monocle_tracing_if_enabled() is True + + with caplog.at_level(logging.DEBUG): + build_tracing_callbacks() + + assert not any("not initialized in this process" in r.message for r in caplog.records) + + def test_no_import_time_setup(): """Regression: importing deerflow.agents must not install telemetry. diff --git a/backend/tests/test_tracing_config.py b/backend/tests/test_tracing_config.py index 943401c974c..164d70d3a14 100644 --- a/backend/tests/test_tracing_config.py +++ b/backend/tests/test_tracing_config.py @@ -28,6 +28,9 @@ def clear_tracing_env(monkeypatch): "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL", + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", ): monkeypatch.delenv(name, raising=False) _reset_tracing_cache() diff --git a/backend/tests/test_tracing_factory.py b/backend/tests/test_tracing_factory.py index 723e42e803d..eeac84224a2 100644 --- a/backend/tests/test_tracing_factory.py +++ b/backend/tests/test_tracing_factory.py @@ -28,6 +28,9 @@ def clear_tracing_env(monkeypatch): "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL", + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", ): monkeypatch.delenv(name, raising=False) reset_tracing_config() diff --git a/config.example.yaml b/config.example.yaml index be5a3df3fcd..867b5097c3b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -45,12 +45,15 @@ logging: # The `file` exporter writes one trace JSON per run to `.monocle/` under the # process working directory (git-ignored). Each trace captures the workflow and # per-agent/LLM/tool spans, including span inputs and outputs (prompts, tool -# arguments, and model responses), token usage, and timings. +# arguments, and model responses), token usage, and timings. Trace files are +# never rotated or cleaned up automatically; prune `.monocle/` periodically if +# you run with tracing enabled long-term. # # Data-exfiltration note: `file` persists prompts and model outputs to local -# disk; non-file exporters (e.g. `okahu`) send that same span data off-box to an -# external collector. Enable only exporters whose data handling you trust, and -# be aware that prompts/outputs may contain sensitive content. +# disk, and `console` prints them to stdout; the remote exporters (`okahu`, +# `s3`, `blob`, `gcs`) send that same span data off-box to an external +# collector. Enable only exporters whose data handling you trust, and be aware +# that prompts/outputs may contain sensitive content. # # Note: Monocle shares the global OpenTelemetry TracerProvider with Langfuse, so # both can be enabled together; Monocle's exporters then also capture Langfuse's From 3ac4310900888f69e917ed1d4a0555cd1e1a536d Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Mon, 13 Jul 2026 17:06:57 -0700 Subject: [PATCH 10/11] fix(tests): pin the factory logger level in the embedded-hint tests configure_logging() from earlier tests in the full suite pins an explicit INFO level on the logger hierarchy, so a root-level caplog.at_level(DEBUG) never sees the factory's debug hint. Scope caplog to deerflow.tracing.factory so the test is independent of suite ordering. --- backend/tests/test_monocle_tracing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index 0b897594b22..69efe202d9e 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -211,7 +211,10 @@ def test_embedded_hint_when_enabled_but_uninitialized(monkeypatch, caplog): reset_tracing_config() monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) - with caplog.at_level(logging.DEBUG): + # Scoped to the factory's logger: earlier tests in the suite may have run + # configure_logging(), which pins an explicit INFO level on the hierarchy + # that a root-level caplog.at_level(DEBUG) would not override. + with caplog.at_level(logging.DEBUG, logger="deerflow.tracing.factory"): assert build_tracing_callbacks() == [] assert any("not initialized in this process" in r.message for r in caplog.records) @@ -226,7 +229,7 @@ def test_no_embedded_hint_after_setup(monkeypatch, caplog): monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) assert setup_monocle_tracing_if_enabled() is True - with caplog.at_level(logging.DEBUG): + with caplog.at_level(logging.DEBUG, logger="deerflow.tracing.factory"): build_tracing_callbacks() assert not any("not initialized in this process" in r.message for r in caplog.records) From 2920b60e20033a2d44385c353ea42908d906c72c Mon Sep 17 00:00:00 2001 From: Mohammed Ansari Date: Mon, 13 Jul 2026 17:18:37 -0700 Subject: [PATCH 11/11] Address review: co-export disclosure, lifespan failure test, exporter parse dedup - Off-box warning now notes that Langfuse's spans are exported too when both providers are enabled and share the global OTel provider; pinned both ways by tests. - Pin the lifespan fail-open contract: a raising Monocle setup is logged and the Gateway keeps serving (pragma dropped now that the path is exercised). README notes a config error is reported at startup and tracing stays off until restart. - Hoist exporter parsing into MonocleTracingConfig.exporter_list so validate() and the off-box warning cannot diverge, and note the upstream coupling on the exporter allow-list. - Reduce config.example.yaml's Monocle block to a pointer; the capture, retention, and data-handling detail lives in README's Monocle section. --- README.md | 2 + backend/app/gateway/app.py | 2 +- .../harness/deerflow/config/tracing_config.py | 10 ++- .../harness/deerflow/tracing/monocle.py | 9 ++- backend/tests/test_monocle_tracing.py | 64 +++++++++++++++++++ config.example.yaml | 27 +------- 6 files changed, 86 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e1f6a4b30cb..c98e23f1d87 100644 --- a/README.md +++ b/README.md @@ -608,6 +608,8 @@ OKAHU_API_KEY=okh_xxxxxxxx # required only for the `okahu` exporter Each run writes one trace file to `.monocle/`; open it in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. Connect to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze traces across runs and run trace-based and agentic evaluations (via the `okahu` exporter). +Traces capture span inputs and outputs verbatim — prompts, tool arguments, and model responses — plus token usage and timings. The `file` exporter keeps them on local disk and never rotates or cleans them up, so prune `.monocle/` periodically; the remote exporters (`okahu`, `s3`, `blob`, `gcs`) send that same data off-box, so enable only destinations you trust. Monocle is initialized once at Gateway startup: a configuration error (unknown exporter, missing `OKAHU_API_KEY`) is logged there and tracing stays off until the Gateway restarts. + #### Using Multiple Providers LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback; Langfuse shares that provider, so all three can run together. Because both span processors sit on the same shared provider, Monocle's exporters also see Langfuse's spans when both are enabled. diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index ba29762de0a..21ba2ac8dc3 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -197,7 +197,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # a bad Monocle config only logs: the Gateway keeps serving without tracing. try: setup_monocle_tracing_if_enabled() - except Exception: # pragma: no cover - observability must never break startup + except Exception: # observability must never break startup logger.exception("Monocle tracing setup failed; continuing without it") # Pre-warm tiktoken encoding cache so the first memory-injection request diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index 89768dcba60..79fb158e62f 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -47,6 +47,9 @@ def validate(self) -> None: raise ValueError(f"Langfuse tracing is enabled but required settings are missing: {', '.join(missing)}") +# Manual mirror of monocle_apptrace's supported exporters, kept local so a typo +# fails at startup with a clear message instead of an opaque upstream error. +# Update this tuple when a monocle_apptrace bump adds or renames an exporter. _MONOCLE_EXPORTERS = ("file", "console", "okahu", "s3", "blob", "gcs") @@ -63,10 +66,15 @@ def is_enabled(self) -> bool: # exporter-dependent and lives in validate(), run at Gateway startup. return self.enabled + @property + def exporter_list(self) -> list[str]: + """The configured exporters, parsed once so validation and setup agree.""" + return [e.strip() for e in self.exporters.split(",") if e.strip()] + def validate(self) -> None: if not self.enabled: return - selected = [e.strip() for e in self.exporters.split(",") if e.strip()] + selected = self.exporter_list unknown = [e for e in selected if e not in _MONOCLE_EXPORTERS] if unknown: raise ValueError(f"MONOCLE_EXPORTERS has unknown exporter(s): {', '.join(unknown)}. Allowed: {', '.join(_MONOCLE_EXPORTERS)}.") diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py index 728f36a5be1..4da611263c5 100644 --- a/backend/packages/harness/deerflow/tracing/monocle.py +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -5,6 +5,7 @@ import logging from deerflow.config import ( + get_enabled_tracing_providers, get_tracing_config, is_monocle_tracing_enabled, ) @@ -44,11 +45,15 @@ def setup_monocle_tracing_if_enabled() -> bool: exporters = monocle.exporters # `console` stays on local stdout, so only the remote exporters are flagged. - off_box = [e.strip() for e in exporters.split(",") if e.strip() and e.strip() not in ("file", "console")] + off_box = [e for e in monocle.exporter_list if e not in ("file", "console")] if off_box: + # Monocle's exporters see every span on the shared global provider, so a + # co-enabled OTel provider's spans leave the box too. + langfuse_note = " Langfuse is also enabled and shares the global provider, so its spans are exported there as well." if "langfuse" in get_enabled_tracing_providers() else "" logger.warning( - "Monocle is exporting trace data (prompts, tool inputs/outputs, completions) beyond the local .monocle/ file via: %s. Make sure that destination is trusted.", + "Monocle is exporting trace data (prompts, tool inputs/outputs, completions) beyond the local .monocle/ file via: %s. Make sure that destination is trusted.%s", ", ".join(off_box), + langfuse_note, ) try: diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index 69efe202d9e..075897ab86f 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -102,6 +102,26 @@ def test_warns_on_non_file_exporter(monkeypatch, caplog): warnings = [r.message for r in caplog.records if "beyond the local" in r.message] assert warnings, "expected an off-box exporter warning" assert "s3" in warnings[0] + assert "Langfuse" not in warnings[0] # only mentioned when Langfuse is co-enabled + + +def test_off_box_warning_mentions_langfuse_when_co_enabled(monkeypatch, caplog): + """With Langfuse sharing the global provider, its spans leave the box too — say so.""" + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + monkeypatch.setenv("OKAHU_API_KEY", "okh_test") + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + warnings = [r.message for r in caplog.records if "beyond the local" in r.message] + assert warnings, "expected an off-box exporter warning" + assert "Langfuse" in warnings[0] def test_no_off_box_warning_for_file_exporter(monkeypatch, caplog): @@ -332,3 +352,47 @@ async def drive() -> None: asyncio.run(drive()) setup_spy.assert_called_once_with() + + +def test_gateway_lifespan_survives_monocle_setup_failure(caplog): + """A raising Monocle setup (e.g. bad MONOCLE_EXPORTERS) must not break startup. + + Pins the lifespan's fail-open contract: the error is logged and the Gateway + keeps serving without tracing. + """ + from fastapi import FastAPI + + from app.gateway.app import lifespan + + @asynccontextmanager + async def _noop_langgraph_runtime(_app, _startup_config): + yield + + startup_config = SimpleNamespace(log_level="INFO", memory=SimpleNamespace(token_counting="char")) + fake_service = MagicMock() + fake_service.get_status = MagicMock(return_value={}) + + async def fake_start(_startup_config): + return fake_service + + setup_spy = MagicMock(side_effect=ValueError("MONOCLE_EXPORTERS has unknown exporter(s): fle.")) + + with ( + patch("app.gateway.app.get_app_config", return_value=startup_config), + patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), + patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), + patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), + patch("app.channels.service.start_channel_service", side_effect=fake_start), + patch("app.channels.service.stop_channel_service", AsyncMock()), + ): + + async def drive() -> None: + async with lifespan(FastAPI()): + pass + + with caplog.at_level(logging.ERROR, logger="app.gateway.app"): + asyncio.run(drive()) # completes despite the raising setup + + setup_spy.assert_called_once_with() + assert any("Monocle tracing setup failed" in r.message for r in caplog.records) diff --git a/config.example.yaml b/config.example.yaml index 75dcf8462d9..385dc8af650 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -34,30 +34,9 @@ logging: # Tracing / Observability (Monocle) # ============================================================================ # Optional agent tracing via Monocle. Configured through environment variables -# (like LangSmith/Langfuse), not a config.yaml key, and OFF by default. When -# enabled, Monocle is initialized once at Gateway startup and auto-instruments -# the LLM/agent clients; a plain library import does not start tracing. -# -# MONOCLE_TRACING=true # enable (default: off) -# MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: file) -# OKAHU_API_KEY=okh_... # required only for the `okahu` exporter -# -# The `file` exporter writes one trace JSON per run to `.monocle/` under the -# process working directory (git-ignored). Each trace captures the workflow and -# per-agent/LLM/tool spans, including span inputs and outputs (prompts, tool -# arguments, and model responses), token usage, and timings. Trace files are -# never rotated or cleaned up automatically; prune `.monocle/` periodically if -# you run with tracing enabled long-term. -# -# Data-exfiltration note: `file` persists prompts and model outputs to local -# disk, and `console` prints them to stdout; the remote exporters (`okahu`, -# `s3`, `blob`, `gcs`) send that same span data off-box to an external -# collector. Enable only exporters whose data handling you trust, and be aware -# that prompts/outputs may contain sensitive content. -# -# Note: Monocle shares the global OpenTelemetry TracerProvider with Langfuse, so -# both can be enabled together; Monocle's exporters then also capture Langfuse's -# spans (see backend/AGENTS.md → Tracing). +# (MONOCLE_TRACING, MONOCLE_EXPORTERS, OKAHU_API_KEY — like LangSmith/Langfuse), +# not config.yaml keys, and OFF by default. See README.md → "Monocle Tracing" +# for setup, what each exporter captures, and where the trace data goes. # ============================================================================ # Token Usage