Add trace-based behavioral tests with Monocle Test Tools

.gitignore

@@ -2,6 +2,9 @@
 *.egg-info
 *.pyc
 
+# Monocle runtime trace output (committed fixtures live in monocle-test/traces/)
+.monocle/
+
 # Python
 __pycache__/
 *.py[cod]

tests/monocle/README.md

@@ -0,0 +1,62 @@
+# Open Deep Research behavioural tests (Monocle Test Tools)
+
+Trace-based tests that lock in Open Deep Research's behaviour. Monocle records
+each run as a structured trace -- the agent invocation, LLM token usage, and
+timings -- and each test asserts against that trace: which agent ran, what it was
+asked, what it produced, and its token/duration cost. A later prompt, model, or
+config change that regresses the behaviour fails here.
+
+## Layout
+
+- `test_opendeepresearch.py` — the suite: four offline tests (one per curated question) + one live test
+- `conftest.py` — Monocle setup, `.env` loading, and `run_opendeepresearch()`
+- `traces/` — recorded good-trace fixtures the offline tests replay
+- `requirements.txt` — dependencies
+
+## Tests
+
+| Test | Scenario | What it shows |
+|---|---|---|
+| `test_earth_seasons` | What causes Earth's seasons (explainer) | agent, verbatim output, token + duration budget |
+| `test_renewable_vs_nonrenewable` | Renewable vs. nonrenewable energy (comparison) | agent, output, `contains_any_output`, budget |
+| `test_ocean_tides` | What causes ocean tides (explainer) | agent, output, `contains_any_output`, budget |
+| `test_tcp_vs_udp` | TCP vs. UDP (comparison) | agent, output, `contains_any_output`, budget |
+| `test_tcp_vs_udp_live` | TCP vs. UDP, run live | live run, structure + budget only |
+
+The offline tests replay recorded traces with budgets measured from those runs
+(rounded up with headroom). The live test drives the agent end-to-end and asserts
+structure and budget only, since the output legitimately varies run to run.
+
+Open Deep Research runs its search inside the model call (OpenAI-native web
+search in the `openai.resources.responses` model-api spans), so its traces
+contain no `agentic.tool.invocation` spans. The tests assert the agent
+invocation, output, and budgets that exist in the trace, and do not assert tool
+calls.
+
+## Run
+
+```bash
+pip install -r requirements.txt
+pytest tests/monocle/ -k "not live"   # offline, no network, no keys
+pytest tests/monocle/                 # includes the live runs (needs OPENAI_API_KEY)
+```
+
+The live tests skip unless `OPENAI_API_KEY` is set. They use OpenAI-native
+search (so no Tavily/other search key is needed) and are cost-capped to a single
+researcher iteration on `gpt-4o-mini`.
+
+## Add your own test
+
+1. Run Open Deep Research under Monocle and capture a trace of a run you're happy
+   with (Monocle writes trace JSON to `.monocle/` by default).
+2. Move it into `traces/` and load it with
+   `monocle_trace_asserter.validator.add_remote_spans(JSONSpanLoader.from_json(path))`.
+3. Assert with the fluent API — `called_agent(...)`, `contains_output(...)`,
+   `contains_any_output(...)`, `under_token_limit(...)`,
+   `under_duration(..., span_type="workflow")` — then add it alongside the others.
+
+## Evaluations (optional)
+
+Each test carries a commented-out `check_eval("hallucination", ...)` chain.
+Monocle can run evaluation checks against a trace; set `OKAHU_API_KEY` and
+uncomment to enable.

tests/monocle/conftest.py

@@ -0,0 +1,58 @@
+"""Pytest scaffold for the Open Deep Research Monocle test suite.
+
+Enables Monocle tracing, loads the repo `.env`, and exposes
+``run_opendeepresearch`` -- the single entry the live tests use to drive the
+agent under instrumentation.
+"""
+import os
+import uuid
+from pathlib import Path
+
+try:
+    from dotenv import load_dotenv
+except ImportError:  # python-dotenv is optional -- only used to auto-load .env for the live tests
+    load_dotenv = None
+from monocle_apptrace import setup_monocle_telemetry
+
+HERE = Path(__file__).resolve().parent
+TRACES = HERE / "traces"
+REPO_ROOT = HERE.parent.parent
+
+# Only export captured spans to the configured exporters (okahu/file) for FAILING
+# tests -- a failing trace is the one worth inspecting. This also sidesteps a
+# monocle_test_tools export-path detail: on a passing test it re-stamps a status
+# attribute onto every captured span, which raises on the *live* tests because
+# real (finished) OpenTelemetry spans have immutable attributes. Offline tests
+# are unaffected (their spans are loaded dicts). Overridable from the environment.
+os.environ.setdefault("MONOCLE_EXPORT_FAILED_TESTS_ONLY", "true")
+
+setup_monocle_telemetry(workflow_name="open-deep-research")
+
+if load_dotenv and (REPO_ROOT / ".env").exists():
+    load_dotenv(REPO_ROOT / ".env")
+
+
+async def run_opendeepresearch(message: str) -> str:
+    """Run Open Deep Research once and return its final report text.
+
+    Uses OpenAI-native web search and is cost-capped to a single researcher
+    iteration on gpt-4o-mini (override via the ODR_MAX_* env vars).
+    """
+    from open_deep_research.deep_researcher import deep_researcher
+
+    config = {"configurable": {
+        "search_api": "openai",
+        "allow_clarification": False,
+        "max_researcher_iterations": int(os.environ.get("ODR_MAX_ITERATIONS", 1)),
+        "max_concurrent_research_units": int(os.environ.get("ODR_MAX_CONCURRENT_UNITS", 1)),
+        "max_react_tool_calls": int(os.environ.get("ODR_MAX_TOOL_CALLS", 1)),
+        "research_model": "openai:gpt-4o-mini", "research_model_max_tokens": 4000,
+        "summarization_model": "openai:gpt-4o-mini", "summarization_model_max_tokens": 4000,
+        "compression_model": "openai:gpt-4o-mini", "compression_model_max_tokens": 4000,
+        "final_report_model": "openai:gpt-4o-mini", "final_report_model_max_tokens": 4000,
+        "thread_id": f"odr-{uuid.uuid4().hex[:8]}",
+    }}
+    result = await deep_researcher.ainvoke(
+        {"messages": [{"role": "user", "content": message}]}, config=config,
+    )
+    return result.get("final_report", "")

tests/monocle/requirements.txt

@@ -0,0 +1,5 @@
+# Installing monocle_test_tools pulls in everything this suite needs
+# (pytest, pytest-asyncio, and monocle_apptrace come transitively).
+monocle_test_tools
+# Auto-loads the repo .env for the live tests (optional).
+python-dotenv

tests/monocle/test_opendeepresearch.py

@@ -0,0 +1,123 @@
+"""Trace-based behavioural tests for Open Deep Research, using Monocle Test Tools.
+
+Each test asserts against the Monocle trace a run emits -- which agent ran, what
+it was asked, what it produced, and its token/duration cost. Four offline tests
+replay recorded good traces (fast, no keys), one per curated question; a single
+live test runs the agent end-to-end.
+
+    pytest tests/monocle/ -k "not live"   # offline, no keys
+    pytest tests/monocle/                 # includes the live run (needs OPENAI_API_KEY)
+
+Open Deep Research runs its search inside the model call (OpenAI-native web
+search in the `openai.resources.responses` model-api spans), so its traces carry
+NO `agentic.tool.invocation` spans. The tests therefore assert the agent
+invocation, output, and budgets that actually exist in the trace, and do not
+assert tool calls.
+"""
+import asyncio
+import os
+
+import pytest
+from monocle_test_tools import TraceAssertion
+
+from conftest import TRACES, run_opendeepresearch
+
+# Recorded good traces (captured from this repo under monocle_apptrace 0.8.8),
+# one per curated question.
+TRACE_SEASONS = str(TRACES / "monocle_trace_open-deep-research_44ab7d2b1a08a7a1de1413be4b08dc46_2026-07-09_12.16.43.json")
+TRACE_ENERGY = str(TRACES / "monocle_trace_open-deep-research_cebe8a23280881e45b22640af87f6e00_2026-07-09_12.16.58.json")
+TRACE_TIDES = str(TRACES / "monocle_trace_open-deep-research_2198c840ff4f156e64ea911eef0a5c71_2026-07-09_12.17.19.json")
+TRACE_TCP_UDP = str(TRACES / "monocle_trace_open-deep-research_5d853757970d9be9f527ce95d1b1e4dc_2026-07-09_12.17.44.json")
+
+
+# --- Offline: replay recorded good traces, one per curated question -------
+
+def test_earth_seasons(monocle_trace_asserter: TraceAssertion):
+    """What causes Earth's seasons (explainer). Real trace: 3,427 total tokens,
+    ~14.6s workflow duration; agent = LangGraph (CompiledStateGraph)."""
+    monocle_trace_asserter.with_trace_source("file", trace_path=TRACE_SEASONS)
+
+    monocle_trace_asserter.called_agent("LangGraph").contains_output("Earth's Seasons")
+    monocle_trace_asserter.contains_any_output("season", "seasons", "tilt", "axial", "Earth")
+    monocle_trace_asserter.under_token_limit(20_000)
+    monocle_trace_asserter.under_duration(60, span_type="workflow")
+
+    # Eval layer (deferred -- set OKAHU_API_KEY and uncomment to enable):
+    # monocle_trace_asserter.with_evaluation("okahu").check_eval("hallucination", "no_hallucination") \
+    #     .check_eval("contextual_precision", "high_precision") \
+    #     .check_eval("sentiment", "positive") \
+    #     .check_eval("bias", "unbiased")
+
+
+def test_renewable_vs_nonrenewable(monocle_trace_asserter: TraceAssertion):
+    """Renewable vs. nonrenewable energy sources (comparison). Real trace: 5,054
+    total tokens, ~16.9s workflow duration; agent = LangGraph."""
+    monocle_trace_asserter.with_trace_source("file", trace_path=TRACE_ENERGY)
+
+    monocle_trace_asserter.called_agent("LangGraph").contains_output("Renewable and Nonrenewable Energy Sources")
+    monocle_trace_asserter.contains_any_output("renewable", "nonrenewable", "energy")
+    monocle_trace_asserter.under_token_limit(20_000)
+    monocle_trace_asserter.under_duration(60, span_type="workflow")
+
+    # monocle_trace_asserter.with_evaluation("okahu").check_eval("hallucination", "no_hallucination") \
+    #     .check_eval("contextual_precision", "high_precision") \
+    #     .check_eval("sentiment", "positive") \
+    #     .check_eval("bias", "unbiased")
+
+
+def test_ocean_tides(monocle_trace_asserter: TraceAssertion):
+    """What causes ocean tides (explainer). Real trace: 5,445 total tokens,
+    ~21.4s workflow duration; agent = LangGraph."""
+    monocle_trace_asserter.with_trace_source("file", trace_path=TRACE_TIDES)
+
+    monocle_trace_asserter.called_agent("LangGraph").contains_output("Ocean Tides")
+    monocle_trace_asserter.contains_any_output("tide", "tides", "moon", "gravitational")
+    monocle_trace_asserter.under_token_limit(20_000)
+    monocle_trace_asserter.under_duration(60, span_type="workflow")
+
+    # monocle_trace_asserter.with_evaluation("okahu").check_eval("hallucination", "no_hallucination") \
+    #     .check_eval("contextual_precision", "high_precision") \
+    #     .check_eval("sentiment", "positive") \
+    #     .check_eval("bias", "unbiased")
+
+
+def test_tcp_vs_udp(monocle_trace_asserter: TraceAssertion):
+    """TCP vs. UDP (comparison). Real trace: 5,041 total tokens, ~21.3s workflow
+    duration; agent = LangGraph."""
+    monocle_trace_asserter.with_trace_source("file", trace_path=TRACE_TCP_UDP)
+
+    monocle_trace_asserter.called_agent("LangGraph").contains_output("TCP and UDP")
+    monocle_trace_asserter.contains_any_output("TCP", "UDP", "protocol", "packet")
+    monocle_trace_asserter.under_token_limit(20_000)
+    monocle_trace_asserter.under_duration(60, span_type="workflow")
+
+    # monocle_trace_asserter.with_evaluation("okahu").check_eval("hallucination", "no_hallucination") \
+    #     .check_eval("contextual_precision", "high_precision") \
+    #     .check_eval("sentiment", "positive") \
+    #     .check_eval("bias", "unbiased")
+
+
+# --- Live: run the agent end-to-end ---------------------------------------
+# Output text varies run to run, so this asserts structure + budget, with
+# contains_any_output kept phrasing-robust. Uses OpenAI-native search (only
+# OPENAI_API_KEY needed); the runner is cost-capped to one researcher iteration.
+
+def test_tcp_vs_udp_live(monocle_trace_asserter: TraceAssertion):
+    """Comparison path, run live: the main differences between TCP and UDP."""
+    if not os.environ.get("OPENAI_API_KEY"):
+        pytest.skip("OPENAI_API_KEY not set -- cannot run the live open-deep-research graph")
+
+    asyncio.run(monocle_trace_asserter.validator.test_workflow_async(
+        run_opendeepresearch,
+        {"test_input": ("What are the main differences between the TCP and UDP protocols?",)},
+    ))
+
+    monocle_trace_asserter.called_agent("LangGraph")
+    monocle_trace_asserter.contains_any_output("TCP", "UDP", "protocol", "packet")
+    monocle_trace_asserter.under_token_limit(500_000)
+    monocle_trace_asserter.under_duration(300, units="seconds", span_type="workflow")
+
+    # monocle_trace_asserter.with_evaluation("okahu").check_eval("hallucination", "no_hallucination") \
+    #     .check_eval("contextual_precision", "high_precision") \
+    #     .check_eval("sentiment", "positive") \
+    #     .check_eval("bias", "unbiased")