From 0e76fa98b206b9de824ace33bbc604db87fa8c6f Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 6 Oct 2026 10:58:30 -0700 Subject: [PATCH 1/3] [AIC-3495] feat(AIC-3495): record which public helpers an application calls Send one $ld:ai:sdk:usage event per helper per client, and only for calls that come from outside the SDK. Co-authored-by: Cursor --- .../launchdarkly_ai_claude_agents/graph.py | 6 +- .../launchdarkly_ai_claude_agents/handler.py | 22 +- .../native_graph.py | 2 + .../claude-agents/tests/test_sdk_usage.py | 77 ++++++ .../handler.py | 24 +- .../claude-messages/tests/test_sdk_usage.py | 73 ++++++ .../src/launchdarkly_ai_server/__init__.py | 4 + .../src/launchdarkly_ai_server/client.py | 16 ++ .../evaluations/module.py | 3 + .../src/launchdarkly_ai_server/graph.py | 9 + .../src/launchdarkly_ai_server/judges.py | 5 +- .../src/launchdarkly_ai_server/lifecycle.py | 6 + .../src/launchdarkly_ai_server/sdk_usage.py | 125 +++++++++ .../src/launchdarkly_ai_server/utils.py | 2 + packages/client/tests/test_client.py | 15 +- packages/client/tests/test_sdk_usage.py | 239 ++++++++++++++++++ .../launchdarkly_ai_langchain_agents/graph.py | 6 +- .../handler.py | 24 +- .../native_graph.py | 2 + .../langchain-agents/tests/test_sdk_usage.py | 83 ++++++ .../handler.py | 28 +- .../tests/test_sdk_usage.py | 79 ++++++ .../launchdarkly_ai_openai_agents/graph.py | 6 +- .../launchdarkly_ai_openai_agents/handler.py | 22 +- .../native_graph.py | 2 + .../openai-agents/tests/test_sdk_usage.py | 77 ++++++ .../handler.py | 24 +- .../openai-messages/tests/test_sdk_usage.py | 74 ++++++ 28 files changed, 1013 insertions(+), 42 deletions(-) create mode 100644 packages/claude-agents/tests/test_sdk_usage.py create mode 100644 packages/claude-messages/tests/test_sdk_usage.py create mode 100644 packages/client/src/launchdarkly_ai_server/sdk_usage.py create mode 100644 packages/client/tests/test_sdk_usage.py create mode 100644 packages/langchain-agents/tests/test_sdk_usage.py create mode 100644 packages/langchain-messages/tests/test_sdk_usage.py create mode 100644 packages/openai-agents/tests/test_sdk_usage.py create mode 100644 packages/openai-messages/tests/test_sdk_usage.py diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py index 83d6e311..fc588fbb 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py @@ -5,7 +5,7 @@ from typing import Any from launchdarkly_ai_claude_agents.handler import create_claude_agents_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_server import graph, report_usage, within_sdk def claude_graph(key: str, **options: Any) -> Any: @@ -15,4 +15,6 @@ def claude_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_claude_agents_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_claude_agents_handler()], **options) + report_usage("claude-agents.claudeGraph") + with within_sdk(): + return graph(key, handlers=[create_claude_agents_handler()], **options) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index 32ee9fd6..ac3ab298 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -35,6 +35,7 @@ ProviderHandler, SpanMessage, SpanMessagePart, + call_within_sdk, compose_history, config, content_to_text, @@ -42,10 +43,12 @@ end_span_once, end_unfinished_spans, parse_template, + report_usage, set_conversation_id_if_absent, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, + within_sdk, ) from .spans import ( @@ -503,6 +506,12 @@ def _build_query_options( def create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: + report_usage("claude-agents.createClaudeAgentsHandler") + with within_sdk(): + return _create_claude_agents_handler(capture_content=capture_content) + + +def _create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: """Creates a ``ProviderHandler`` for Anthropic's Claude via the claude-agent-sdk. Set *capture_content* to put prompts, model output, tool arguments and tool results on the @@ -886,10 +895,13 @@ def claude_agents( **kwargs: Any, ) -> Any: """Convenience wrapper: creates a handler and calls config(...).invoke().""" + report_usage("claude-agents.claudeAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_claude_agents_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_claude_agents_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py index b0ffadb9..2ad60c37 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py @@ -18,6 +18,7 @@ NativeTool, get_client, make_track_data, + report_usage, to_ld_context, ) @@ -207,6 +208,7 @@ def to_claude_agents( Returns an object with a ``.invoke(input, variables)`` coroutine. """ + report_usage("claude-agents.toClaudeAgents") _opts = opts or {} async def invoke( diff --git a/packages/claude-agents/tests/test_sdk_usage.py b/packages/claude-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..24645624 --- /dev/null +++ b/packages/claude-agents/tests/test_sdk_usage.py @@ -0,0 +1,77 @@ +"""Tests for TESTING.md §3.27 — claude-agents.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_claude_agents import ( + claude_agents, + claude_graph, + create_claude_agents_handler, + to_claude_agents, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("claude-agents.claudeAgents", lambda: claude_agents("k", "q", CONTEXT)), + ("claude-agents.createClaudeAgentsHandler", create_claude_agents_handler), + ("claude-agents.claudeGraph", lambda: claude_graph("k")), + ("claude-agents.toClaudeAgents", lambda: to_claude_agents(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 0de014a3..6cc5c0d2 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -11,6 +11,7 @@ ProviderHandler, SpanMessage, SpanMessagePart, + call_within_sdk, compose_history, config, content_to_text, @@ -19,10 +20,12 @@ end_unfinished_spans, is_content_blocks, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, to_semconv_finish_reason, + within_sdk, ) from .spans import ( @@ -350,6 +353,14 @@ def _is_coroutine(fn: Any) -> bool: def create_claude_messages_handler(*, capture_content: bool = False) -> ProviderHandler: + report_usage("claude-messages.createClaudeMessagesHandler") + with within_sdk(): + return _create_claude_messages_handler(capture_content=capture_content) + + +def _create_claude_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: """Creates a ``ProviderHandler`` for Anthropic Claude (messages API). Requires ``anthropic`` to be installed as a peer dependency. @@ -708,10 +719,13 @@ def claude_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("claude-messages.claudeMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_claude_messages_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_claude_messages_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/claude-messages/tests/test_sdk_usage.py b/packages/claude-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..5af13e9b --- /dev/null +++ b/packages/claude-messages/tests/test_sdk_usage.py @@ -0,0 +1,73 @@ +"""Tests for TESTING.md §3.27 — claude-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_claude_messages import ( + claude_messages, + create_claude_messages_handler, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("claude-messages.claudeMessages", lambda: claude_messages("k", "q", CONTEXT)), + ("claude-messages.createClaudeMessagesHandler", create_claude_messages_handler), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 4001239e..8fa4812c 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -63,6 +63,7 @@ resolve_tools, ) from .sdk_info import SDK_INFO_CONTEXT, SDK_INFO_EVENT, register_ai_sdk_package +from .sdk_usage import call_within_sdk, report_usage, within_sdk from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -239,6 +240,9 @@ "SDK_INFO_CONTEXT", "SDK_INFO_EVENT", "register_ai_sdk_package", + "call_within_sdk", + "report_usage", + "within_sdk", # judges "build_judge_tasks", "run_judge", diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index f9f598f2..eba61461 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -72,6 +72,19 @@ async def invoke( context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None, + ) -> ProviderResponse[Any]: + from .sdk_usage import report_usage, within_sdk + + report_usage("client.config.invoke") + with within_sdk(): + return await self._invoke_body(user_input, context, variables, history) + + async def _invoke_body( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> ProviderResponse[Any]: resolved_handler_list = resolve_handlers( self._registry, self._normalize_handlers() @@ -165,6 +178,9 @@ def stream( call has already exited. Binding here — at call time — is what lets a caller hand the generator off and iterate it later. """ + from .sdk_usage import report_usage + + report_usage("client.config.stream") return bind_conversation_id( self._stream_events(user_input, context, variables, history) ) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 4cc684aa..d0315d0b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -508,6 +508,9 @@ def init_evaluations( transport: Transport = urllib_transport, ) -> EvaluationsModule: """Resolve credentials and construct the evaluations module.""" + from ..sdk_usage import report_usage + + report_usage("client.init_evaluations") token = api_token or _env("LD_API_TOKEN") if not token: raise EvaluationsError( diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 02e81f55..048cdec8 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -837,6 +837,9 @@ async def resolve_graph( ``context`` is a keyword-only argument, mirroring the TypeScript ``resolveGraph(key, { context, handlers, toolHandlers, registry })`` shape. """ + from .sdk_usage import report_usage + + report_usage("client.resolveGraph") resolved_handlers = resolve_handlers(registry, handlers) resolved_tools = resolve_tools(registry, tool_handlers) options = { @@ -869,6 +872,9 @@ async def invoke( variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None, ) -> ProviderGraphResponse: + from .sdk_usage import report_usage + + report_usage("client.graph.invoke") from opentelemetry import trace from .judges import run_judges @@ -1068,6 +1074,9 @@ def stream( this call may have already exited. Binding the conversation id and capturing the OTel parent here — at call time — matches ``config().stream()`` and the TypeScript graph stream. """ + from .sdk_usage import report_usage + + report_usage("client.graph.stream") from opentelemetry import context as otel_context caller_context = otel_context.get_current() diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 56d7e032..bd34e135 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -11,6 +11,7 @@ numeric_score, parse_judge_response, ) +from .sdk_usage import report_usage from .types import ( AiConfigRep, JudgeResult, @@ -232,6 +233,7 @@ async def build_judge_tasks( ``samplingRate`` causes them to be skipped are excluded from the list. Returns an empty list when no active judges are configured. """ + report_usage("client.buildJudgeTasks") from .lifecycle import extract_variation judge_config_block = ( @@ -349,8 +351,9 @@ async def run_judge( ``get_client().track()`` on the main thread. Returns ``None`` when no compatible handler is found or the response cannot - be parsed as ``{"score": ..., "reasoning": ...}``. + be parsed as ``{"score": ..., "reasoning": ...}``. """ + report_usage("client.runJudge") from .tracking import execute_and_track def _matches(h: ProviderHandler) -> bool: diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 18e311c6..f9bc8497 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -7,6 +7,7 @@ from typing import Any from .sdk_info import flush_ai_sdk_info, reset_ai_sdk_info +from .sdk_usage import flush_sdk_usage, report_usage, reset_sdk_usage from .types import InitClientOptions from .utils import model_stamps_from_meta @@ -142,6 +143,7 @@ async def init_client( # Idempotent — if already initialized, return the existing client if _client is not None: flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client # BYOC path — pre-initialized client @@ -149,6 +151,7 @@ async def init_client( _client = client _setup_telemetry(opts.get("sdkKey", "byoc"), opts) flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client # Resolve SDK key @@ -196,6 +199,7 @@ async def init_client( _client = ld_client _setup_telemetry(sdk_key, opts) flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client @@ -213,6 +217,7 @@ async def shutdown() -> None: _client = None _tracer_provider = None reset_ai_sdk_info() + reset_sdk_usage() if local_provider is not None: try: @@ -270,6 +275,7 @@ async def inspect_config( - ``config`` (dict | None): the parsed AI config, or None when disabled/invalid - ``meta`` (dict | None): the variation metadata, or None when unreachable """ + report_usage("client.inspectConfig") from .types_validation import parse_ai_config from .utils import to_ld_context # late import avoids circular dependency diff --git a/packages/client/src/launchdarkly_ai_server/sdk_usage.py b/packages/client/src/launchdarkly_ai_server/sdk_usage.py new file mode 100644 index 00000000..9f0173ae --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/sdk_usage.py @@ -0,0 +1,125 @@ +"""Records which public AI SDK helpers an application calls. + +One ``$ld:ai:sdk:usage`` event per helper per client. A call made before a +client exists is held and sent on the next init, on the same path as sdk-info. +A call made from inside another helper does not report. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any + +from .sdk_info import SDK_INFO_CONTEXT + +SDK_USAGE_EVENT = "$ld:ai:sdk:usage" +_SDK_USAGE_LANGUAGE = "python" +_SDK_NAME = "launchdarkly-ai-server" + +_pending: set[str] = set() +_reported: set[str] = set() +_depth: ContextVar[int] = ContextVar("ld_ai_sdk_usage_depth", default=0) + + +def _peek_client() -> Any: + from . import lifecycle + + return lifecycle._client + + +def _version() -> str: + from . import __version__ + + return __version__ + + +def _deliver(client: Any, helper: str) -> None: + from .utils import to_ld_context + + try: + client.track( + SDK_USAGE_EVENT, + to_ld_context(client, SDK_INFO_CONTEXT), + { + "aiSdkName": _SDK_NAME, + "aiSdkVersion": _version(), + "aiSdkLanguage": _SDK_USAGE_LANGUAGE, + "helper": helper, + }, + 1, + ) + except Exception: + # A throw from track, or from building the payload, must not fail the helper. + return + + +def report_usage(helper: str) -> None: + """Record a public helper. + + Sends immediately when a client exists, otherwise holds the helper until + :func:`flush_sdk_usage`. No-ops when already reported or when called from + inside :func:`within_sdk`. + """ + try: + if _depth.get() > 0: + return + if helper in _reported or helper in _pending: + return + client = _peek_client() + if client is None: + _pending.add(helper) + return + _deliver(client, helper) + _reported.add(helper) + except Exception: + _pending.discard(helper) + _reported.add(helper) + + +@contextmanager +def within_sdk() -> Iterator[None]: + """Mark the body as an internal SDK call so nested helpers do not report.""" + token = _depth.set(_depth.get() + 1) + try: + yield + finally: + _depth.reset(token) + + +def call_within_sdk(fn: Callable[[], Any]) -> Any: + """Call ``fn`` inside the SDK scope. + + A wrapper reports itself, then uses this so the helpers it calls do not + report. When ``fn`` returns an awaitable, the scope stays active until that + awaitable finishes: an async function does not run until it is awaited, and + the scope has to cover that run. + """ + with within_sdk(): + result = fn() + if isinstance(result, Awaitable): + + async def _drive() -> Any: + with within_sdk(): + return await result + + return _drive() + return result + + +def flush_sdk_usage(client: Any) -> None: + """Send every helper that was recorded before a client existed.""" + if not _pending: + return + held = list(_pending) + _pending.clear() + for helper in held: + _deliver(client, helper) + _reported.add(helper) + + +def reset_sdk_usage() -> None: + """Drop the reported set so the next client hears each helper again.""" + _reported.clear() + _pending.clear() diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 2958de6e..24493711 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from typing import Any, Literal +from .sdk_usage import report_usage from .types import ( AiConfigRep, GraphNode, @@ -28,6 +29,7 @@ def create_handler( Wraps a plain async callable in a :class:`ProviderHandler` with the given ``provides_for`` metadata and optional streaming implementation. """ + report_usage("client.createHandler") return ProviderHandler( fn=fn, provides_for=provides_for, diff --git a/packages/client/tests/test_client.py b/packages/client/tests/test_client.py index d7357308..072069cb 100644 --- a/packages/client/tests/test_client.py +++ b/packages/client/tests/test_client.py @@ -22,6 +22,15 @@ # --------------------------------------------------------------------------- CONTEXT = {"kind": "user", "key": "u1"} +_SDK_BOOKKEEPING = {"$ld:ai:sdk:usage", "$ld:ai:sdk:info"} + + +def _generation_calls(client: MagicMock) -> list[Any]: + return [ + call + for call in client.track.call_args_list + if call[0][0] not in _SDK_BOOKKEEPING + ] def _make_client(response: Any = "Hello", usage: dict | None = None) -> MagicMock: @@ -128,7 +137,7 @@ async def test_track_data_copies_model_key_and_version_from_ld_meta( assert result.track_data["modelVersion"] == 3 assert isinstance(result.track_data["modelVersion"], int) assert mock_ld_client.track.call_args_list - for call in mock_ld_client.track.call_args_list: + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" assert payload["modelVersion"] == 3 @@ -141,7 +150,7 @@ async def test_track_data_omits_model_key_and_version_when_absent( assert result.track_data is not None assert "modelKey" not in result.track_data assert "modelVersion" not in result.track_data - for call in mock_ld_client.track.call_args_list: + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert "modelKey" not in payload assert "modelVersion" not in payload @@ -175,7 +184,7 @@ async def test_stream_track_data_copies_model_key_and_version( async for _ in m.stream("q", CONTEXT): pass assert mock_ld_client.track.call_args_list - for call in mock_ld_client.track.call_args_list: + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" assert payload["modelVersion"] == 3 diff --git a/packages/client/tests/test_sdk_usage.py b/packages/client/tests/test_sdk_usage.py new file mode 100644 index 00000000..4f4cc6dd --- /dev/null +++ b/packages/client/tests/test_sdk_usage.py @@ -0,0 +1,239 @@ +"""Tests for TESTING.md §3.27 helper usage events.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_server import ( + SDK_INFO_CONTEXT, + ProviderHandler, + __version__, + build_judge_tasks, + config, + create_handler, + graph, + init_client, + init_evaluations, + inspect_config, + resolve_graph, + run_judge, + shutdown, +) + +CONTEXT = {"kind": "user", "key": "user-1"} +HELPER = "client.buildJudgeTasks" + + +def _client(variation: Any = None) -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=variation) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _install(variation: Any = None) -> MagicMock: + await shutdown() + client = _client(variation) + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + client.track.reset_mock() + return client + + +def _usage(client: MagicMock, helper: str | None = None) -> list[Any]: + return [ + call + for call in client.track.call_args_list + if call.args[0] == "$ld:ai:sdk:usage" + and (helper is None or call.args[2].get("helper") == helper) + ] + + +def _expect_usage(client: MagicMock, helper: str) -> None: + calls = _usage(client, helper) + assert len(calls) == 1 + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +def _judge_kwargs() -> dict[str, Any]: + return { + "config": { + "model": {"name": "m"}, + "provider": {"name": "OpenAI"}, + "instructions": "i", + }, + "user_context": CONTEXT, + "handler": MagicMock(), + "llm_response": "", + "base_track_data": { + "runId": "r", + "configKey": "k", + "variationKey": "v", + "version": 1, + "modelName": "m", + "providerName": "OpenAI", + }, + } + + +def _call(helper: str) -> Any: + if helper == "client.config.invoke": + return config(key="k").invoke("q", CONTEXT) + if helper == "client.config.stream": + return config(key="k").stream("q", CONTEXT) + if helper == "client.graph.invoke": + return graph("k").invoke("q", CONTEXT) + if helper == "client.graph.stream": + return graph("k").stream("q", CONTEXT) + if helper == "client.resolveGraph": + return resolve_graph("k", context=CONTEXT) + if helper == "client.inspectConfig": + return inspect_config("k", CONTEXT) + if helper == "client.runJudge": + return run_judge(MagicMock(), []) + if helper == "client.buildJudgeTasks": + return build_judge_tasks(**_judge_kwargs()) + if helper == "client.createHandler": + + async def _fn(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": ""} + + create_handler(("OpenAI", "messages"), _fn) + return None + try: + init_evaluations() + except Exception: + return None + return None + + +@pytest.fixture +async def client() -> Any: + installed = await _install() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + "helper", + [ + "client.config.invoke", + "client.config.stream", + "client.graph.invoke", + "client.graph.stream", + "client.resolveGraph", + "client.inspectConfig", + "client.runJudge", + "client.buildJudgeTasks", + "client.createHandler", + "client.init_evaluations", + ], +) +async def test_helper_sends_one_usage_event(client: MagicMock, helper: str) -> None: + await _settle(_call(helper)) + _expect_usage(client, helper) + if helper == "client.graph.invoke": + assert _usage(client, "client.graph.stream") == [] + if helper == "client.graph.stream": + assert _usage(client, "client.graph.invoke") == [] + + +async def test_repeat_call_sends_nothing_until_shutdown(client: MagicMock) -> None: + await build_judge_tasks(**_judge_kwargs()) + _expect_usage(client, HELPER) + await build_judge_tasks(**_judge_kwargs()) + assert len(_usage(client, HELPER)) == 1 + + await shutdown() + fresh = await _install() + await build_judge_tasks(**_judge_kwargs()) + _expect_usage(fresh, HELPER) + + +async def test_helper_before_client_is_sent_on_init() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + assert _usage(client, HELPER) == [] + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + _expect_usage(client, HELPER) + await shutdown() + + +async def test_track_failure_does_not_fail_the_helper_or_retry( + client: MagicMock, +) -> None: + client.track.side_effect = RuntimeError("client closed") + result = await build_judge_tasks(**_judge_kwargs()) + assert result == [] + assert len(_usage(client, HELPER)) == 1 + + client.track.reset_mock(side_effect=True) + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client, HELPER) == [] + + +async def test_generation_events_gain_no_usage_fields() -> None: + await shutdown() + try: + installed = await _install( + { + "model": {"name": "gpt-4o"}, + "provider": {"name": "OpenAI"}, + "instructions": "Be helpful.", + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "messages", + }, + } + ) + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok", "usage": {"input_tokens": 2, "output_tokens": 3}} + + await config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + ).invoke("q", CONTEXT) + + _expect_usage(installed, "client.config.invoke") + others = [ + call + for call in installed.track.call_args_list + if call.args[0] not in {"$ld:ai:sdk:usage", "$ld:ai:sdk:info"} + ] + names = [call.args[0] for call in others] + assert "$ld:ai:duration:total" in names + assert "$ld:ai:generation:success" in names + assert "$ld:ai:tokens:total" in names + for call in others: + payload = call.args[2] + assert "helper" not in payload + assert "aiSdkName" not in payload + assert "aiSdkVersion" not in payload + assert "aiSdkLanguage" not in payload + finally: + await shutdown() diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py index 95dc0c11..1a0f4817 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py @@ -5,7 +5,7 @@ from typing import Any from launchdarkly_ai_langchain_agents.handler import create_langchain_agents_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_server import graph, report_usage, within_sdk def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: @@ -15,4 +15,6 @@ def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_langchain_agents_handler(llm)], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_langchain_agents_handler(llm)], **options) + report_usage("langchain-agents.langchainGraph") + with within_sdk(): + return graph(key, handlers=[create_langchain_agents_handler(llm)], **options) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 41c69ed6..c56637e4 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -17,6 +17,7 @@ ProviderHandler, SpanMessage, SpanMessagePart, + call_within_sdk, compose_history, config, create_handler, @@ -27,8 +28,10 @@ lang_chain_span_messages, lang_chain_span_usage, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, + within_sdk, ) from .messages import to_lang_chain_messages @@ -255,6 +258,14 @@ def _run_usage_from_messages(messages: list[Any]) -> Any: def create_langchain_agents_handler( llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + report_usage("langchain-agents.createLangChainAgentsHandler") + with within_sdk(): + return _create_langchain_agents_handler(llm, capture_content=capture_content) + + +def _create_langchain_agents_handler( + llm: Any = None, *, capture_content: bool = False ) -> ProviderHandler: """Creates a ``ProviderHandler`` for LangChain via ``create_react_agent``. @@ -570,10 +581,13 @@ def langchain_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("langchain-agents.langchainAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_langchain_agents_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_langchain_agents_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py index 190e5f87..135dd95d 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py @@ -20,6 +20,7 @@ get_client, make_track_data, parse_template, + report_usage, to_ld_context, ) @@ -132,6 +133,7 @@ def to_lang_graph( {"context": ctx}, ).invoke("I was double charged") """ + report_usage("langchain-agents.toLangGraph") _opts = opts or {} async def invoke( diff --git a/packages/langchain-agents/tests/test_sdk_usage.py b/packages/langchain-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..c5f187dd --- /dev/null +++ b/packages/langchain-agents/tests/test_sdk_usage.py @@ -0,0 +1,83 @@ +"""Tests for TESTING.md §3.27 — langchain-agents.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_langchain_agents import ( + create_langchain_agents_handler, + langchain_agents, + langchain_graph, + to_lang_graph, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ( + "langchain-agents.langchainAgents", + lambda: langchain_agents("k", "q", CONTEXT), + ), + ( + "langchain-agents.createLangChainAgentsHandler", + create_langchain_agents_handler, + ), + ("langchain-agents.langchainGraph", lambda: langchain_graph("k")), + ("langchain-agents.toLangGraph", lambda: to_lang_graph(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index b30e8b80..4c7b45f0 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -13,6 +13,7 @@ SpanMessage, SpanMessagePart, SpanUsage, + call_within_sdk, compose_history, config, create_handler, @@ -27,9 +28,11 @@ lang_chain_span_usage, number_or_zero, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, + within_sdk, ) from .spans import ( @@ -371,6 +374,14 @@ async def _run_structured_turn( def create_langchain_messages_handler( llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + report_usage("langchain-messages.createLangChainHandler") + with within_sdk(): + return _create_langchain_messages_handler(llm, capture_content=capture_content) + + +def _create_langchain_messages_handler( + llm: Any = None, *, capture_content: bool = False ) -> ProviderHandler: """ Creates a ``ProviderHandler`` for LangChain (chat models). @@ -1013,12 +1024,15 @@ def langchain_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("langchain-messages.langchainMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_langchain_messages_handler( - llm=llm, capture_content=capture_content - ), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_langchain_messages_handler( + llm, capture_content=capture_content + ), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/langchain-messages/tests/test_sdk_usage.py b/packages/langchain-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..2d7f9b59 --- /dev/null +++ b/packages/langchain-messages/tests/test_sdk_usage.py @@ -0,0 +1,79 @@ +"""Tests for TESTING.md §3.27 — langchain-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_langchain_messages import ( + create_langchain_messages_handler, + langchain_messages, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ( + "langchain-messages.langchainMessages", + lambda: langchain_messages("k", "q", CONTEXT), + ), + ( + "langchain-messages.createLangChainHandler", + create_langchain_messages_handler, + ), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py index bac10979..83a44349 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py @@ -5,7 +5,7 @@ from typing import Any from launchdarkly_ai_openai_agents.handler import create_openai_agent_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_server import graph, report_usage, within_sdk def openai_graph(key: str, **options: Any) -> Any: @@ -15,4 +15,6 @@ def openai_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_openai_agent_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_openai_agent_handler()], **options) + report_usage("openai-agents.openaiGraph") + with within_sdk(): + return graph(key, handlers=[create_openai_agent_handler()], **options) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index ac97a9aa..2729127e 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -30,6 +30,7 @@ ProviderHandler, RunUsage, SpanUsage, + call_within_sdk, compose_history, config, content_to_text, @@ -39,10 +40,12 @@ end_unfinished_spans, image_block_to_url, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, text_message, + within_sdk, ) from .spans import ( @@ -458,6 +461,12 @@ def _usage_from_error(error: BaseException) -> SpanUsage | None: def create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: + report_usage("openai-agents.createOpenAIAgentHandler") + with within_sdk(): + return _create_openai_agent_handler(capture_content=capture_content) + + +def _create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: """Creates a ``ProviderHandler`` for OpenAI via the openai-agents SDK. Set *capture_content* to put prompts, model output, tool arguments and tool results on the @@ -712,10 +721,13 @@ def openai_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("openai-agents.openaiAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_openai_agent_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_openai_agent_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py index ac8b0a90..96847ddf 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py @@ -20,6 +20,7 @@ get_client, make_track_data, parse_template, + report_usage, to_ld_context, ) @@ -102,6 +103,7 @@ def to_openai_agents( {"context": ctx}, ).invoke("I was double charged") """ + report_usage("openai-agents.toOpenAIAgents") _opts = opts or {} async def invoke( diff --git a/packages/openai-agents/tests/test_sdk_usage.py b/packages/openai-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..5c424a35 --- /dev/null +++ b/packages/openai-agents/tests/test_sdk_usage.py @@ -0,0 +1,77 @@ +"""Tests for TESTING.md §3.27 — openai-agents.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_openai_agents import ( + create_openai_agent_handler, + openai_agents, + openai_graph, + to_openai_agents, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("openai-agents.openaiAgents", lambda: openai_agents("k", "q", CONTEXT)), + ("openai-agents.createOpenAIAgentHandler", create_openai_agent_handler), + ("openai-agents.openaiGraph", lambda: openai_graph("k")), + ("openai-agents.toOpenAIAgents", lambda: to_openai_agents(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index a404fdcb..bcf692c8 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -12,6 +12,7 @@ RunUsage, SpanMessage, SpanMessagePart, + call_within_sdk, compose_history, config, content_to_text, @@ -22,9 +23,11 @@ image_block_to_url, is_content_blocks, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, + within_sdk, ) from .spans import ( @@ -211,6 +214,14 @@ async def _run_model_turn( def create_openai_messages_handler(*, capture_content: bool = False) -> ProviderHandler: + report_usage("openai-messages.createOpenAIHandler") + with within_sdk(): + return _create_openai_messages_handler(capture_content=capture_content) + + +def _create_openai_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: """ Creates a ``ProviderHandler`` for OpenAI (responses API). Requires ``openai`` to be installed as a peer dependency. @@ -641,10 +652,13 @@ def openai_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("openai-messages.openaiMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( - key=config_key, - handler=create_openai_messages_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) + return call_within_sdk( + lambda: config( + key=config_key, + handler=create_openai_messages_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) + ) diff --git a/packages/openai-messages/tests/test_sdk_usage.py b/packages/openai-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..06697886 --- /dev/null +++ b/packages/openai-messages/tests/test_sdk_usage.py @@ -0,0 +1,74 @@ +"""Tests for TESTING.md §3.27 — openai-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_openai_messages import ( + create_openai_messages_handler, + openai_messages, +) +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} +ANONYMOUS = SDK_INFO_CONTEXT + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("openai-messages.openaiMessages", lambda: openai_messages("k", "q", CONTEXT)), + ("openai-messages.createOpenAIHandler", create_openai_messages_handler), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == ANONYMOUS + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + } + assert calls[0].args[3] == 1 From 682af4d98aa3779e370bdda0f2b8066e3d54cd30 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 6 Oct 2026 11:28:24 -0700 Subject: [PATCH 2/3] [AIC-3495] fix(AIC-3495): report usage through internal versions, not a call-scope flag The ContextVar scope (within_sdk / call_within_sdk) decided who the caller was at run time, and got it wrong in both directions. Graph wrappers returned a public GraphInstance, so a later .invoke()/.stream() reported client.graph.* outside the scope. config().invoke wrapped the user's handler and tools in the scope, so helpers the application called from its own code were dropped, while config().stream had no scope and reported them: the same handler gave different results per path. Follow the spec (TESTING.md 3.27): each reporting helper has a non-reporting internal version, and SDK code calls that version. - Remove within_sdk, call_within_sdk and the ContextVar, and their exports. - Add _create_handler, _config, _graph (importable, not in __all__) and _build_judge_tasks. ConfigInstance/GraphInstance invoke and stream report, then delegate to non-reporting _invoke/_stream; _config/_graph return subclasses whose invoke/stream do not report. stream still reports at call time. - Package factories report, then call their own _ factory, which calls _create_handler. Wrappers and graph wrappers use _config/_graph and the _ factories. Public factories get their docstrings back. - Guard the registry with a lock. A helper is marked reported before delivery and the client is read under the lock, so a helper held while init runs is not lost and concurrent first calls send once. - Tests: graph wrapper invoke/stream and native adapter invoke emit only their own helper; helpers called from a user handler and tool report under both invoke and stream; core rows emit only their own helper; pending flush on SDK-key and already-initialized init; the test setter does not flush; payload-build failures are swallowed and not retried; tool-call and graph payloads carry no usage fields. test_client.py again requires generation events to exist. Co-Authored-By: Claude Opus 5.5 --- .../launchdarkly_ai_claude_agents/graph.py | 7 +- .../launchdarkly_ai_claude_agents/handler.py | 32 ++- packages/claude-agents/tests/test_graph.py | 8 +- packages/claude-agents/tests/test_handler.py | 6 +- .../claude-agents/tests/test_sdk_usage.py | 39 ++++ .../handler.py | 36 ++-- .../claude-messages/tests/test_handler.py | 8 +- .../src/launchdarkly_ai_server/__init__.py | 7 +- .../src/launchdarkly_ai_server/client.py | 71 +++++- .../src/launchdarkly_ai_server/graph.py | 67 +++++- .../src/launchdarkly_ai_server/judges.py | 26 ++- .../src/launchdarkly_ai_server/sdk_usage.py | 87 +++----- .../src/launchdarkly_ai_server/utils.py | 10 + packages/client/tests/test_client.py | 4 +- packages/client/tests/test_sdk_usage.py | 203 ++++++++++++++++++ .../launchdarkly_ai_langchain_agents/graph.py | 7 +- .../handler.py | 36 ++-- packages/langchain-agents/tests/test_graph.py | 10 +- .../langchain-agents/tests/test_handler.py | 8 +- .../langchain-agents/tests/test_sdk_usage.py | 39 ++++ .../handler.py | 40 ++-- .../langchain-messages/tests/test_handler.py | 8 +- .../launchdarkly_ai_openai_agents/graph.py | 7 +- .../launchdarkly_ai_openai_agents/handler.py | 32 ++- packages/openai-agents/tests/test_graph.py | 8 +- packages/openai-agents/tests/test_handler.py | 12 +- .../openai-agents/tests/test_sdk_usage.py | 39 ++++ .../handler.py | 36 ++-- .../openai-messages/tests/test_handler.py | 8 +- 29 files changed, 661 insertions(+), 240 deletions(-) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py index fc588fbb..994ffe04 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py @@ -4,8 +4,8 @@ from typing import Any -from launchdarkly_ai_claude_agents.handler import create_claude_agents_handler -from launchdarkly_ai_server import graph, report_usage, within_sdk +from launchdarkly_ai_claude_agents.handler import _create_claude_agents_handler +from launchdarkly_ai_server import _graph, report_usage def claude_graph(key: str, **options: Any) -> Any: @@ -16,5 +16,4 @@ def claude_graph(key: str, **options: Any) -> Any: Use the base ``graph()`` directly for multi-provider graphs. """ report_usage("claude-agents.claudeGraph") - with within_sdk(): - return graph(key, handlers=[create_claude_agents_handler()], **options) + return _graph(key, handlers=[_create_claude_agents_handler()], **options) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index ac3ab298..dbf9468f 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -35,11 +35,10 @@ ProviderHandler, SpanMessage, SpanMessagePart, - call_within_sdk, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, end_span_once, end_unfinished_spans, parse_template, @@ -48,7 +47,6 @@ set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, - within_sdk, ) from .spans import ( @@ -506,17 +504,17 @@ def _build_query_options( def create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: - report_usage("claude-agents.createClaudeAgentsHandler") - with within_sdk(): - return _create_claude_agents_handler(capture_content=capture_content) - - -def _create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: """Creates a ``ProviderHandler`` for Anthropic's Claude via the claude-agent-sdk. Set *capture_content* to put prompts, model output, tool arguments and tool results on the emitted spans. It defaults to off. See TELEMETRY-CONTRACT.md section 7. """ + report_usage("claude-agents.createClaudeAgentsHandler") + return _create_claude_agents_handler(capture_content=capture_content) + + +def _create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: + """Non-reporting :func:`create_claude_agents_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -703,7 +701,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("Anthropic", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -898,10 +896,8 @@ def claude_agents( report_usage("claude-agents.claudeAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_claude_agents_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_claude_agents_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/claude-agents/tests/test_graph.py b/packages/claude-agents/tests/test_graph.py index be7ff327..0e6f05d1 100644 --- a/packages/claude-agents/tests/test_graph.py +++ b/packages/claude-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestClaudeGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key", extra_kwarg=42) kw = mock_graph.call_args[1] diff --git a/packages/claude-agents/tests/test_handler.py b/packages/claude-agents/tests/test_handler.py index 5651391e..83aef4de 100644 --- a/packages/claude-agents/tests/test_handler.py +++ b/packages/claude-agents/tests/test_handler.py @@ -1604,7 +1604,7 @@ def _fake_config(**_kwargs: Any) -> Any: return instance monkeypatch.setattr( - "launchdarkly_ai_claude_agents.handler.config", _fake_config + "launchdarkly_ai_claude_agents.handler._config", _fake_config ) result = await claude_agents("cfg-key", "hi", {"key": "u1"}) assert result == {"output": "ok"} @@ -1633,7 +1633,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(_handler_mod, "config", mock_config_fn): + with patch.object(_handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_agents.handler import claude_agents ctx = {"kind": "user", "key": "u1"} @@ -1656,7 +1656,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(_handler_mod, "config", mock_config_fn): + with patch.object(_handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_agents.handler import claude_agents ctx = {"kind": "user", "key": "u1"} diff --git a/packages/claude-agents/tests/test_sdk_usage.py b/packages/claude-agents/tests/test_sdk_usage.py index 24645624..06586199 100644 --- a/packages/claude-agents/tests/test_sdk_usage.py +++ b/packages/claude-agents/tests/test_sdk_usage.py @@ -1,5 +1,6 @@ """Tests for TESTING.md §3.27 — claude-agents.""" +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -75,3 +76,41 @@ async def test_helper_sends_one_usage_event( "helper": helper, } assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = claude_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["claude-agents.claudeGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_claude_agents(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["claude-agents.toClaudeAgents"] diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 6cc5c0d2..9bd00c08 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -11,11 +11,10 @@ ProviderHandler, SpanMessage, SpanMessagePart, - call_within_sdk, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, end_span_once, end_unfinished_spans, is_content_blocks, @@ -25,7 +24,6 @@ set_output_content_attributes, set_tool_call_content_attributes, to_semconv_finish_reason, - within_sdk, ) from .spans import ( @@ -353,14 +351,6 @@ def _is_coroutine(fn: Any) -> bool: def create_claude_messages_handler(*, capture_content: bool = False) -> ProviderHandler: - report_usage("claude-messages.createClaudeMessagesHandler") - with within_sdk(): - return _create_claude_messages_handler(capture_content=capture_content) - - -def _create_claude_messages_handler( - *, capture_content: bool = False -) -> ProviderHandler: """Creates a ``ProviderHandler`` for Anthropic Claude (messages API). Requires ``anthropic`` to be installed as a peer dependency. @@ -370,6 +360,14 @@ def _create_claude_messages_handler( meaning models, token counts, timings and tool names, until a caller asks for more. Turning this on sends the text of every request and response to whatever collector the SDK is pointed at. """ + report_usage("claude-messages.createClaudeMessagesHandler") + return _create_claude_messages_handler(capture_content=capture_content) + + +def _create_claude_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_claude_messages_handler`, used by this package's wrappers.""" import importlib anthropic_mod = importlib.import_module("anthropic") @@ -468,7 +466,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("Anthropic", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -722,10 +720,8 @@ def claude_messages( report_usage("claude-messages.claudeMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_claude_messages_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_claude_messages_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/claude-messages/tests/test_handler.py b/packages/claude-messages/tests/test_handler.py index f07336fe..0a0353d9 100644 --- a/packages/claude-messages/tests/test_handler.py +++ b/packages/claude-messages/tests/test_handler.py @@ -1191,7 +1191,7 @@ def test_calls_through_to_model_call(self, mock_anthropic: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_messages.handler import claude_messages ctx = {"kind": "user", "key": "u1"} @@ -1214,7 +1214,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_messages.handler import claude_messages ctx = {"kind": "user", "key": "u1"} @@ -1811,8 +1811,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_claude_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_claude_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.claude_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 8fa4812c..cc8ae5cf 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -6,6 +6,7 @@ __version__ = "0.2.4" # x-release-please-version from .client import ConfigInstance, config +from .client import _config as _config from .content import ( SpanMessage, SpanMessagePart, @@ -39,6 +40,7 @@ init_evaluations, ) from .graph import GraphInstance, graph, resolve_graph +from .graph import _graph as _graph from .history import ( any_multimodal, compose_history, @@ -63,7 +65,7 @@ resolve_tools, ) from .sdk_info import SDK_INFO_CONTEXT, SDK_INFO_EVENT, register_ai_sdk_package -from .sdk_usage import call_within_sdk, report_usage, within_sdk +from .sdk_usage import report_usage from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -127,6 +129,7 @@ to_ld_context, to_usage_dict, ) +from .utils import _create_handler as _create_handler __all__ = [ # noqa: RUF022 # types @@ -240,9 +243,7 @@ "SDK_INFO_CONTEXT", "SDK_INFO_EVENT", "register_ai_sdk_package", - "call_within_sdk", "report_usage", - "within_sdk", # judges "build_judge_tasks", "run_judge", diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index eba61461..0f77383f 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -5,9 +5,10 @@ from typing import Any from .conversation import bind_conversation_id -from .judges import build_judge_tasks, run_judges +from .judges import _build_judge_tasks, run_judges from .lifecycle import extract_variation from .registry import resolve_handlers, resolve_tools +from .sdk_usage import report_usage from .tracking import execute_and_stream, execute_and_track from .types import ( AiConfigRep, @@ -73,13 +74,10 @@ async def invoke( variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None, ) -> ProviderResponse[Any]: - from .sdk_usage import report_usage, within_sdk - report_usage("client.config.invoke") - with within_sdk(): - return await self._invoke_body(user_input, context, variables, history) + return await self._invoke(user_input, context, variables, history) - async def _invoke_body( + async def _invoke( self, user_input: str | None, context: LDContext, @@ -129,7 +127,7 @@ async def _invoke_body( usage_obj = to_usage_dict(usage) if self._skip_judges: - judge_tasks = await build_judge_tasks( + judge_tasks = await _build_judge_tasks( config=config, user_context=context, handler=handler, @@ -176,11 +174,20 @@ def stream( Deliberately not an ``async def`` with ``yield``: a generator body does not run until the first ``__anext__``, by which point a :func:`conversation_id` block wrapped around this call has already exited. Binding here — at call time — is what lets a caller hand the - generator off and iterate it later. + generator off and iterate it later. The usage report runs here for the same reason: it + fires on the call, before any iteration. """ - from .sdk_usage import report_usage - report_usage("client.config.stream") + return self._stream(user_input, context, variables, history) + + def _stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[StreamEvent, None]: + """Non-reporting :meth:`stream`: binds the conversation id at call time.""" return bind_conversation_id( self._stream_events(user_input, context, variables, history) ) @@ -248,6 +255,32 @@ async def _stream_events( } +class _InternalConfigInstance(ConfigInstance): + """A ``ConfigInstance`` whose ``invoke`` and ``stream`` do not report usage. + + Returned by :func:`_config` for SDK wrappers: the wrapper reports itself, and the + config call it makes is not a call from the application. + """ + + async def invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderResponse[Any]: + return await self._invoke(user_input, context, variables, history) + + def stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[StreamEvent, None]: + return self._stream(user_input, context, variables, history) + + def config( *, key: str, @@ -275,3 +308,21 @@ def config( registry=registry, skip_judges=skip_judges, ) + + +def _config( + *, + key: str, + handler: ProviderHandler | list[ProviderHandler] | None = None, + tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, + registry: Any = None, + skip_judges: bool = False, +) -> ConfigInstance: + """Non-reporting :func:`config` for SDK wrappers.""" + return _InternalConfigInstance( + key=key, + handler=handler, + tool_handlers=tool_handlers, + registry=registry, + skip_judges=skip_judges, + ) diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 048cdec8..beb39d97 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -875,6 +875,16 @@ async def invoke( from .sdk_usage import report_usage report_usage("client.graph.invoke") + return await self._invoke(user_input, context, variables, history) + + async def _invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderGraphResponse: + """Non-reporting :meth:`invoke`.""" from opentelemetry import trace from .judges import run_judges @@ -1072,11 +1082,22 @@ def stream( Deliberately not an ``async def`` with ``yield``: a generator body does not run until the first ``__anext__``, by which point a ``conversation_id`` / caller span scope wrapped around this call may have already exited. Binding the conversation id and capturing the OTel parent - here — at call time — matches ``config().stream()`` and the TypeScript graph stream. + here — at call time — matches ``config().stream()`` and the TypeScript graph stream. The + usage report also runs here, on the call, before any iteration. """ from .sdk_usage import report_usage report_usage("client.graph.stream") + return self._stream(user_input, context, variables, history) + + def _stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[GraphStreamEvent, None]: + """Non-reporting :meth:`stream`: binds the conversation id and OTel parent at call time.""" from opentelemetry import context as otel_context caller_context = otel_context.get_current() @@ -1298,6 +1319,32 @@ async def _stream_events( end_span_once(span, ended, abandoned=True, cancelled=cancelled) +class _InternalGraphInstance(GraphInstance): + """A ``GraphInstance`` whose ``invoke`` and ``stream`` do not report usage. + + Returned by :func:`_graph` for SDK graph wrappers: the wrapper reports itself, and the + graph calls made through the object it returns are not separate helper calls. + """ + + async def invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderGraphResponse: + return await self._invoke(user_input, context, variables, history) + + def stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[GraphStreamEvent, None]: + return self._stream(user_input, context, variables, history) + + def graph( key: str, *, @@ -1318,3 +1365,21 @@ def graph( "graph_judge": graph_judge, } return GraphInstance(key=key, options=options) + + +def _graph( + key: str, + *, + handlers: list[ProviderHandler] | None = None, + tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, + registry: Any = None, + graph_judge: str | None = None, +) -> GraphInstance: + """Non-reporting :func:`graph` for SDK graph wrappers.""" + options = { + "handlers": handlers, + "tool_handlers": tool_handlers, + "registry": registry, + "graph_judge": graph_judge, + } + return _InternalGraphInstance(key=key, options=options) diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index bd34e135..d3e8377b 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -234,6 +234,30 @@ async def build_judge_tasks( Returns an empty list when no active judges are configured. """ report_usage("client.buildJudgeTasks") + return await _build_judge_tasks( + config=config, + user_context=user_context, + handler=handler, + handlers=handlers, + llm_response=llm_response, + base_track_data=base_track_data, + user_input=user_input, + trajectory=trajectory, + ) + + +async def _build_judge_tasks( + *, + config: AiConfigRep, + user_context: LDContext, + handler: ProviderHandler, + handlers: list[ProviderHandler] | None = None, + llm_response: str, + base_track_data: TrackData, + user_input: str | None = None, + trajectory: str = "", +) -> list[JudgeTask]: + """Non-reporting :func:`build_judge_tasks`, used by ``config().invoke``.""" from .lifecycle import extract_variation judge_config_block = ( @@ -351,7 +375,7 @@ async def run_judge( ``get_client().track()`` on the main thread. Returns ``None`` when no compatible handler is found or the response cannot - be parsed as ``{"score": ..., "reasoning": ...}``. + be parsed as ``{"score": ..., "reasoning": ...}``. """ report_usage("client.runJudge") from .tracking import execute_and_track diff --git a/packages/client/src/launchdarkly_ai_server/sdk_usage.py b/packages/client/src/launchdarkly_ai_server/sdk_usage.py index 9f0173ae..41a1717e 100644 --- a/packages/client/src/launchdarkly_ai_server/sdk_usage.py +++ b/packages/client/src/launchdarkly_ai_server/sdk_usage.py @@ -2,14 +2,15 @@ One ``$ld:ai:sdk:usage`` event per helper per client. A call made before a client exists is held and sent on the next init, on the same path as sdk-info. -A call made from inside another helper does not report. + +Only a call from outside the SDK reports. Each reporting helper has a +non-reporting internal version, and SDK code calls that version instead, so +this module never needs to know who the caller is. """ from __future__ import annotations -from collections.abc import Awaitable, Callable, Iterator -from contextlib import contextmanager -from contextvars import ContextVar +import threading from typing import Any from .sdk_info import SDK_INFO_CONTEXT @@ -20,7 +21,12 @@ _pending: set[str] = set() _reported: set[str] = set() -_depth: ContextVar[int] = ContextVar("ld_ai_sdk_usage_depth", default=0) +# Guards _pending and _reported. A helper is marked reported under the lock +# before it is delivered, so concurrent first calls send it once. The client is +# read under the lock too: init sets the client and then flushes, and the flush +# waits here, so a helper held just as init runs is still sent by that flush. +# Delivery runs outside the lock so a slow track() does not block other callers. +_lock = threading.Lock() def _peek_client() -> Any: @@ -59,67 +65,38 @@ def report_usage(helper: str) -> None: """Record a public helper. Sends immediately when a client exists, otherwise holds the helper until - :func:`flush_sdk_usage`. No-ops when already reported or when called from - inside :func:`within_sdk`. + :func:`flush_sdk_usage`. No-ops when already reported or held. Never raises. """ try: - if _depth.get() > 0: - return - if helper in _reported or helper in _pending: - return - client = _peek_client() - if client is None: - _pending.add(helper) - return + with _lock: + client = _peek_client() + if helper in _reported or helper in _pending: + return + if client is None: + _pending.add(helper) + return + _reported.add(helper) _deliver(client, helper) - _reported.add(helper) except Exception: - _pending.discard(helper) - _reported.add(helper) - - -@contextmanager -def within_sdk() -> Iterator[None]: - """Mark the body as an internal SDK call so nested helpers do not report.""" - token = _depth.set(_depth.get() + 1) - try: - yield - finally: - _depth.reset(token) - - -def call_within_sdk(fn: Callable[[], Any]) -> Any: - """Call ``fn`` inside the SDK scope. - - A wrapper reports itself, then uses this so the helpers it calls do not - report. When ``fn`` returns an awaitable, the scope stays active until that - awaitable finishes: an async function does not run until it is awaited, and - the scope has to cover that run. - """ - with within_sdk(): - result = fn() - if isinstance(result, Awaitable): - - async def _drive() -> Any: - with within_sdk(): - return await result - - return _drive() - return result + with _lock: + _pending.discard(helper) + _reported.add(helper) def flush_sdk_usage(client: Any) -> None: """Send every helper that was recorded before a client existed.""" - if not _pending: - return - held = list(_pending) - _pending.clear() + with _lock: + if not _pending: + return + held = list(_pending) + _pending.clear() + _reported.update(held) for helper in held: _deliver(client, helper) - _reported.add(helper) def reset_sdk_usage() -> None: """Drop the reported set so the next client hears each helper again.""" - _reported.clear() - _pending.clear() + with _lock: + _reported.clear() + _pending.clear() diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 24493711..7c656abc 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -30,6 +30,16 @@ def create_handler( ``provides_for`` metadata and optional streaming implementation. """ report_usage("client.createHandler") + return _create_handler(provides_for, fn, stream_fn, capture_content) + + +def _create_handler( + provides_for: tuple[str, Literal["agent", "messages"]], + fn: _HandlerFn, + stream_fn: _StreamFn | None = None, + capture_content: bool = False, +) -> ProviderHandler: + """Non-reporting :func:`create_handler`, used by the package factories.""" return ProviderHandler( fn=fn, provides_for=provides_for, diff --git a/packages/client/tests/test_client.py b/packages/client/tests/test_client.py index 072069cb..1ea9f66a 100644 --- a/packages/client/tests/test_client.py +++ b/packages/client/tests/test_client.py @@ -136,7 +136,7 @@ async def test_track_data_copies_model_key_and_version_from_ld_meta( assert result.track_data["modelKey"] == "my-model" assert result.track_data["modelVersion"] == 3 assert isinstance(result.track_data["modelVersion"], int) - assert mock_ld_client.track.call_args_list + assert _generation_calls(mock_ld_client) for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" @@ -183,7 +183,7 @@ async def test_stream_track_data_copies_model_key_and_version( m = config(key="flag", handler=_make_handler(stream_chunks=["a", "b"])) async for _ in m.stream("q", CONTEXT): pass - assert mock_ld_client.track.call_args_list + assert _generation_calls(mock_ld_client) for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" diff --git a/packages/client/tests/test_sdk_usage.py b/packages/client/tests/test_sdk_usage.py index 4f4cc6dd..962d9da3 100644 --- a/packages/client/tests/test_sdk_usage.py +++ b/packages/client/tests/test_sdk_usage.py @@ -6,6 +6,7 @@ import pytest import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.sdk_usage as sdk_usage_module from launchdarkly_ai_server import ( SDK_INFO_CONTEXT, ProviderHandler, @@ -53,6 +54,25 @@ def _usage(client: MagicMock, helper: str | None = None) -> list[Any]: ] +def _helpers(client: MagicMock) -> list[str]: + return [call.args[2].get("helper") for call in _usage(client)] + + +_USAGE_FIELDS = ("helper", "aiSdkName", "aiSdkVersion", "aiSdkLanguage") + +_ENABLED_CONFIG: dict[str, Any] = { + "model": {"name": "gpt-4o"}, + "provider": {"name": "OpenAI"}, + "instructions": "Be helpful.", + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "messages", + }, +} + + def _expect_usage(client: MagicMock, helper: str) -> None: calls = _usage(client, helper) assert len(calls) == 1 @@ -151,6 +171,9 @@ async def client() -> Any: async def test_helper_sends_one_usage_event(client: MagicMock, helper: str) -> None: await _settle(_call(helper)) _expect_usage(client, helper) + # One call emits only its own helper: config.invoke emits no client.createHandler, + # client.buildJudgeTasks, and so on. + assert _helpers(client) == [helper] if helper == "client.graph.invoke": assert _usage(client, "client.graph.stream") == [] if helper == "client.graph.stream": @@ -181,6 +204,38 @@ async def test_helper_before_client_is_sent_on_init() -> None: await shutdown() +async def test_held_helper_is_sent_on_sdk_key_init() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + stub = _client() + ld_module = MagicMock() + ld_module.Config = MagicMock(return_value=MagicMock()) + ld_module.LDClient = MagicMock(return_value=stub) + with ( + patch.object(lifecycle_module, "_setup_telemetry", return_value=None), + patch("importlib.import_module", return_value=ld_module), + ): + await init_client({"sdkKey": "sdk-key"}) + ld_module.LDClient.assert_called_once() + _expect_usage(stub, HELPER) + await shutdown() + + +async def test_test_setter_does_not_flush_but_already_initialized_init_does() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + lifecycle_module._set_client_for_testing(client) + assert _usage(client) == [] + + # init_client returns the existing client on this path and flushes what was held. + assert await init_client() is client + _expect_usage(client, HELPER) + await shutdown() + + async def test_track_failure_does_not_fail_the_helper_or_retry( client: MagicMock, ) -> None: @@ -194,6 +249,154 @@ async def test_track_failure_does_not_fail_the_helper_or_retry( assert _usage(client, HELPER) == [] +async def test_payload_failure_does_not_fail_the_helper_or_retry( + client: MagicMock, +) -> None: + with patch.object( + sdk_usage_module, "_version", side_effect=RuntimeError("no version") + ) as version: + assert await build_judge_tasks(**_judge_kwargs()) == [] + assert await build_judge_tasks(**_judge_kwargs()) == [] + # Building the payload threw before track, once. The helper counts as reported. + assert version.call_count == 1 + assert _usage(client) == [] + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client) == [] + + +async def test_held_payload_failure_on_flush_is_swallowed() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + with ( + patch.object(lifecycle_module, "_setup_telemetry", return_value=None), + patch.object( + sdk_usage_module, "_version", side_effect=RuntimeError("no version") + ), + ): + assert await init_client(client=client) is client + assert _usage(client) == [] + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client) == [] + await shutdown() + + +async def test_config_invoke_does_not_report_internal_judge_tasks() -> None: + """``skip_judges`` builds judge tasks through the internal path.""" + judged = { + **_ENABLED_CONFIG, + "judgeConfiguration": {"judges": [{"key": "judge", "samplingRate": 1}]}, + } + installed = await _install(judged) + try: + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok"} + + result = await config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + skip_judges=True, + ).invoke("q", CONTEXT) + assert result.judge_tasks + assert _helpers(installed) == ["client.config.invoke"] + finally: + await shutdown() + + +@pytest.mark.parametrize("method", ["invoke", "stream"]) +async def test_helpers_called_from_user_handler_and_tool_still_report( + method: str, +) -> None: + """Code the application passes in is outside the SDK, under invoke and stream alike.""" + installed = await _install(_ENABLED_CONFIG) + try: + + async def _lookup() -> str: + await build_judge_tasks(**_judge_kwargs()) + return "found" + + async def _handler( + _config: Any, + _input: Any, + tool_handlers: dict[str, Any], + *_a: Any, + **_k: Any, + ) -> dict[str, Any]: + create_handler(("OpenAI", "messages"), _handler) + return {"output": await tool_handlers["lookup"]()} + + instance = config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + tool_handlers={"lookup": _lookup}, + ) + if method == "invoke": + await instance.invoke("q", CONTEXT) + else: + async for _ in instance.stream("q", CONTEXT): + pass + + assert _helpers(installed) == [ + f"client.config.{method}", + "client.createHandler", + "client.buildJudgeTasks", + ] + tool_calls = [ + call + for call in installed.track.call_args_list + if call.args[0] == "$ld:ai:tool_call" + ] + assert len(tool_calls) == 1 + assert tool_calls[0].args[2]["toolKey"] == "lookup" + for field in _USAGE_FIELDS: + assert field not in tool_calls[0].args[2] + finally: + await shutdown() + + +def _graph_variation(key: str, _ctx: Any, _default: Any) -> Any: + if key == "graph-key": + return {"root": "root-node", "edges": {"root-node": [{"key": "leaf-node"}]}} + return {**_ENABLED_CONFIG, "provider": {"name": "TestProvider"}} + + +@pytest.mark.parametrize("method", ["invoke", "stream"]) +async def test_graph_events_gain_no_usage_fields(method: str) -> None: + installed = await _install() + installed.variation = AsyncMock(side_effect=_graph_variation) + try: + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok", "usage": {"input_tokens": 1, "output_tokens": 1}} + + instance = graph( + "graph-key", + handlers=[ + ProviderHandler(fn=_handler, provides_for=("TestProvider", "messages")) + ], + ) + if method == "invoke": + await instance.invoke("q", CONTEXT) + else: + async for _ in instance.stream("q", CONTEXT): + pass + + assert _helpers(installed) == [f"client.graph.{method}"] + graph_calls = [ + call + for call in installed.track.call_args_list + if str(call.args[0]).startswith("$ld:ai:graph:") + ] + assert "$ld:ai:graph:invocation_success" in [c.args[0] for c in graph_calls] + for call in graph_calls: + for field in _USAGE_FIELDS: + assert field not in call.args[2] + finally: + await shutdown() + + async def test_generation_events_gain_no_usage_fields() -> None: await shutdown() try: diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py index 1a0f4817..f184ae59 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py @@ -4,8 +4,8 @@ from typing import Any -from launchdarkly_ai_langchain_agents.handler import create_langchain_agents_handler -from launchdarkly_ai_server import graph, report_usage, within_sdk +from launchdarkly_ai_langchain_agents.handler import _create_langchain_agents_handler +from launchdarkly_ai_server import _graph, report_usage def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: @@ -16,5 +16,4 @@ def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: Use the base ``graph()`` directly for multi-provider graphs. """ report_usage("langchain-agents.langchainGraph") - with within_sdk(): - return graph(key, handlers=[create_langchain_agents_handler(llm)], **options) + return _graph(key, handlers=[_create_langchain_agents_handler(llm)], **options) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index c56637e4..c2d5ea83 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -17,10 +17,9 @@ ProviderHandler, SpanMessage, SpanMessagePart, - call_within_sdk, + _config, + _create_handler, compose_history, - config, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -31,7 +30,6 @@ report_usage, set_input_content_attributes, set_output_content_attributes, - within_sdk, ) from .messages import to_lang_chain_messages @@ -258,14 +256,6 @@ def _run_usage_from_messages(messages: list[Any]) -> Any: def create_langchain_agents_handler( llm: Any = None, *, capture_content: bool = False -) -> ProviderHandler: - report_usage("langchain-agents.createLangChainAgentsHandler") - with within_sdk(): - return _create_langchain_agents_handler(llm, capture_content=capture_content) - - -def _create_langchain_agents_handler( - llm: Any = None, *, capture_content: bool = False ) -> ProviderHandler: """Creates a ``ProviderHandler`` for LangChain via ``create_react_agent``. @@ -276,6 +266,14 @@ def _create_langchain_agents_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("langchain-agents.createLangChainAgentsHandler") + return _create_langchain_agents_handler(llm, capture_content=capture_content) + + +def _create_langchain_agents_handler( + llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_langchain_agents_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -423,7 +421,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("*", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -584,10 +582,8 @@ def langchain_agents( report_usage("langchain-agents.langchainAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_langchain_agents_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_langchain_agents_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/langchain-agents/tests/test_graph.py b/packages/langchain-agents/tests/test_graph.py index ce5e455a..e6ea186f 100644 --- a/packages/langchain-agents/tests/test_graph.py +++ b/packages/langchain-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestLangChainGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated_with_langchain_agents_handler(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated_with_langchain_agents_handler(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", extra=1) kw = mock_graph.call_args[1] @@ -42,7 +42,7 @@ def test_user_cannot_override_handlers(self) -> None: def test_llm_option_forwarded(self) -> None: mock_llm = MagicMock() - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", llm=mock_llm) # llm is consumed by create_langchain_agents_handler(llm), not forwarded to graph diff --git a/packages/langchain-agents/tests/test_handler.py b/packages/langchain-agents/tests/test_handler.py index e43cb3b0..c6d3b2fa 100644 --- a/packages/langchain-agents/tests/test_handler.py +++ b/packages/langchain-agents/tests/test_handler.py @@ -1326,7 +1326,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_agents.handler import langchain_agents ctx = {"kind": "user", "key": "u1"} @@ -1348,7 +1348,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_agents.handler import langchain_agents ctx = {"kind": "user", "key": "u1"} @@ -1836,8 +1836,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_langchain_agents_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_langchain_agents_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.langchain_agents("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/langchain-agents/tests/test_sdk_usage.py b/packages/langchain-agents/tests/test_sdk_usage.py index c5f187dd..4d438fc9 100644 --- a/packages/langchain-agents/tests/test_sdk_usage.py +++ b/packages/langchain-agents/tests/test_sdk_usage.py @@ -1,5 +1,6 @@ """Tests for TESTING.md §3.27 — langchain-agents.""" +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -81,3 +82,41 @@ async def test_helper_sends_one_usage_event( "helper": helper, } assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = langchain_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["langchain-agents.langchainGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_lang_graph(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["langchain-agents.toLangGraph"] diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index 4c7b45f0..bd9f04f9 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -13,10 +13,9 @@ SpanMessage, SpanMessagePart, SpanUsage, - call_within_sdk, + _config, + _create_handler, compose_history, - config, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -32,7 +31,6 @@ set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, - within_sdk, ) from .spans import ( @@ -374,14 +372,6 @@ async def _run_structured_turn( def create_langchain_messages_handler( llm: Any = None, *, capture_content: bool = False -) -> ProviderHandler: - report_usage("langchain-messages.createLangChainHandler") - with within_sdk(): - return _create_langchain_messages_handler(llm, capture_content=capture_content) - - -def _create_langchain_messages_handler( - llm: Any = None, *, capture_content: bool = False ) -> ProviderHandler: """ Creates a ``ProviderHandler`` for LangChain (chat models). @@ -395,6 +385,14 @@ def _create_langchain_messages_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("langchain-messages.createLangChainHandler") + return _create_langchain_messages_handler(llm, capture_content=capture_content) + + +def _create_langchain_messages_handler( + llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_langchain_messages_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -705,7 +703,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("*", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -1027,12 +1025,10 @@ def langchain_messages( report_usage("langchain-messages.langchainMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_langchain_messages_handler( - llm, capture_content=capture_content - ), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_langchain_messages_handler( + llm, capture_content=capture_content + ), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/langchain-messages/tests/test_handler.py b/packages/langchain-messages/tests/test_handler.py index d00e43fe..c305be6d 100644 --- a/packages/langchain-messages/tests/test_handler.py +++ b/packages/langchain-messages/tests/test_handler.py @@ -1109,7 +1109,7 @@ def test_calls_through_to_model_call(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_messages.handler import langchain_messages ctx = {"kind": "user", "key": "u1"} @@ -1132,7 +1132,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_messages.handler import langchain_messages ctx = {"kind": "user", "key": "u1"} @@ -1642,8 +1642,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_langchain_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_langchain_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.langchain_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py index 83a44349..c9b518bd 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py @@ -4,8 +4,8 @@ from typing import Any -from launchdarkly_ai_openai_agents.handler import create_openai_agent_handler -from launchdarkly_ai_server import graph, report_usage, within_sdk +from launchdarkly_ai_openai_agents.handler import _create_openai_agent_handler +from launchdarkly_ai_server import _graph, report_usage def openai_graph(key: str, **options: Any) -> Any: @@ -16,5 +16,4 @@ def openai_graph(key: str, **options: Any) -> Any: Use the base ``graph()`` directly for multi-provider graphs. """ report_usage("openai-agents.openaiGraph") - with within_sdk(): - return graph(key, handlers=[create_openai_agent_handler()], **options) + return _graph(key, handlers=[_create_openai_agent_handler()], **options) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 2729127e..65fccad2 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -30,11 +30,10 @@ ProviderHandler, RunUsage, SpanUsage, - call_within_sdk, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -45,7 +44,6 @@ set_output_content_attributes, set_tool_call_content_attributes, text_message, - within_sdk, ) from .spans import ( @@ -461,18 +459,18 @@ def _usage_from_error(error: BaseException) -> SpanUsage | None: def create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: - report_usage("openai-agents.createOpenAIAgentHandler") - with within_sdk(): - return _create_openai_agent_handler(capture_content=capture_content) - - -def _create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: """Creates a ``ProviderHandler`` for OpenAI via the openai-agents SDK. Set *capture_content* to put prompts, model output, tool arguments and tool results on the emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("openai-agents.createOpenAIAgentHandler") + return _create_openai_agent_handler(capture_content=capture_content) + + +def _create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: + """Non-reporting :func:`create_openai_agent_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -569,7 +567,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("OpenAI", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -724,10 +722,8 @@ def openai_agents( report_usage("openai-agents.openaiAgents") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_openai_agent_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_openai_agent_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/openai-agents/tests/test_graph.py b/packages/openai-agents/tests/test_graph.py index 29c6af40..b7eeab0f 100644 --- a/packages/openai-agents/tests/test_graph.py +++ b/packages/openai-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestOpenAIGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated_with_openai_agent_handler(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated_with_openai_agent_handler(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key", extra=1) kw = mock_graph.call_args[1] diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index 75e20003..fe0d4af2 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -1225,7 +1225,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): ctx = {"kind": "user", "key": "u1"} openai_agents("my-flag", "hello", ctx) @@ -1242,7 +1242,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): ctx = {"kind": "user", "key": "u1"} openai_agents("my-flag", "hello", ctx) @@ -1691,8 +1691,8 @@ def _factory(*, capture_content: bool = False) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_agent_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_agent_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_agents("k", "q", {}, capture_content=True) @@ -1712,8 +1712,8 @@ def _factory(*, capture_content: bool = False) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_agent_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_agent_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_agents("k", "q", {}) diff --git a/packages/openai-agents/tests/test_sdk_usage.py b/packages/openai-agents/tests/test_sdk_usage.py index 5c424a35..5af3a3cc 100644 --- a/packages/openai-agents/tests/test_sdk_usage.py +++ b/packages/openai-agents/tests/test_sdk_usage.py @@ -1,5 +1,6 @@ """Tests for TESTING.md §3.27 — openai-agents.""" +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -75,3 +76,41 @@ async def test_helper_sends_one_usage_event( "helper": helper, } assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = openai_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["openai-agents.openaiGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_openai_agents(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["openai-agents.toOpenAIAgents"] diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index bcf692c8..055a6fed 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -12,11 +12,10 @@ RunUsage, SpanMessage, SpanMessagePart, - call_within_sdk, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -27,7 +26,6 @@ set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, - within_sdk, ) from .spans import ( @@ -214,14 +212,6 @@ async def _run_model_turn( def create_openai_messages_handler(*, capture_content: bool = False) -> ProviderHandler: - report_usage("openai-messages.createOpenAIHandler") - with within_sdk(): - return _create_openai_messages_handler(capture_content=capture_content) - - -def _create_openai_messages_handler( - *, capture_content: bool = False -) -> ProviderHandler: """ Creates a ``ProviderHandler`` for OpenAI (responses API). Requires ``openai`` to be installed as a peer dependency. @@ -230,6 +220,14 @@ def _create_openai_messages_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("openai-messages.createOpenAIHandler") + return _create_openai_messages_handler(capture_content=capture_content) + + +def _create_openai_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_openai_messages_handler`, used by this package's wrappers.""" import importlib openai_mod = importlib.import_module("openai") @@ -418,7 +416,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("OpenAI", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -655,10 +653,8 @@ def openai_messages( report_usage("openai-messages.openaiMessages") variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return call_within_sdk( - lambda: config( - key=config_key, - handler=create_openai_messages_handler(capture_content=capture_content), - **kwargs, - ).invoke(user_input, context, variables=variables) - ) + return _config( + key=config_key, + handler=_create_openai_messages_handler(capture_content=capture_content), + **kwargs, + ).invoke(user_input, context, variables=variables) diff --git a/packages/openai-messages/tests/test_handler.py b/packages/openai-messages/tests/test_handler.py index 4668976f..35da2f9d 100644 --- a/packages/openai-messages/tests/test_handler.py +++ b/packages/openai-messages/tests/test_handler.py @@ -1167,7 +1167,7 @@ def test_calls_through_to_model_call(self, mock_openai: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_openai_messages.handler import openai_messages ctx = {"kind": "user", "key": "u1"} @@ -1190,7 +1190,7 @@ def test_callable_without_extra_kwargs(self, mock_openai: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_openai_messages.handler import openai_messages ctx = {"kind": "user", "key": "u1"} @@ -1866,8 +1866,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs From c42d570e6bfb863df2b47daebeebc87e1aece277 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Tue, 6 Oct 2026 13:24:22 -0700 Subject: [PATCH 3/3] [AIC-3495] feat(AIC-3495): report the helper's own package name and version $ld:ai:sdk:usage now carries helperPackageName and helperPackageVersion next to the core aiSdkName/aiSdkVersion (TESTING.md 3.27, A.14). Handler packages are versioned separately from the core, so aiSdkVersion alone does not say which handler release ran a helper. $ld:ai:sdk:info already lists every package, but it sends them all on one shared context key, so it cannot be tied to the helpers an application calls. The usage event is the only place that link can live. report_usage takes optional package_name and package_version. Each handler package passes the same values it registers with sdk-info, from a new _version.py that holds PACKAGE_NAME and __version__ once, so __init__.py and the handler modules share them without a circular import. release-please now bumps _version.py. client.* helpers pass neither and default to the core identity. The dedupe key is still the helper alone, and held-until-init records keep their package so the flushed event is complete. Co-Authored-By: Claude Opus 5.5 --- packages/claude-agents/agents.md | 1 + .../launchdarkly_ai_claude_agents/__init__.py | 5 +- .../launchdarkly_ai_claude_agents/_version.py | 4 ++ .../launchdarkly_ai_claude_agents/graph.py | 4 +- .../launchdarkly_ai_claude_agents/handler.py | 5 +- .../native_graph.py | 4 +- .../claude-agents/tests/test_sdk_usage.py | 33 +++++++++ packages/claude-messages/agents.md | 1 + .../__init__.py | 5 +- .../_version.py | 4 ++ .../handler.py | 7 +- .../claude-messages/tests/test_sdk_usage.py | 33 +++++++++ .../src/launchdarkly_ai_server/sdk_usage.py | 51 ++++++++++---- packages/client/tests/test_sdk_usage.py | 70 +++++++++++++++++-- packages/langchain-agents/agents.md | 1 + .../__init__.py | 5 +- .../_version.py | 4 ++ .../launchdarkly_ai_langchain_agents/graph.py | 4 +- .../handler.py | 7 +- .../native_graph.py | 3 +- .../langchain-agents/tests/test_sdk_usage.py | 33 +++++++++ packages/langchain-messages/agents.md | 1 + .../__init__.py | 5 +- .../_version.py | 4 ++ .../handler.py | 5 +- .../tests/test_sdk_usage.py | 33 +++++++++ packages/openai-agents/agents.md | 1 + .../launchdarkly_ai_openai_agents/__init__.py | 5 +- .../launchdarkly_ai_openai_agents/_version.py | 4 ++ .../launchdarkly_ai_openai_agents/graph.py | 4 +- .../launchdarkly_ai_openai_agents/handler.py | 5 +- .../native_graph.py | 3 +- .../openai-agents/tests/test_sdk_usage.py | 33 +++++++++ packages/openai-messages/agents.md | 1 + .../__init__.py | 5 +- .../_version.py | 4 ++ .../handler.py | 5 +- .../openai-messages/tests/test_sdk_usage.py | 33 +++++++++ release-please-config.json | 12 ++-- 39 files changed, 387 insertions(+), 60 deletions(-) create mode 100644 packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py create mode 100644 packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py create mode 100644 packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py create mode 100644 packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py create mode 100644 packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py create mode 100644 packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py diff --git a/packages/claude-agents/agents.md b/packages/claude-agents/agents.md index c7202e2e..8ff60679 100644 --- a/packages/claude-agents/agents.md +++ b/packages/claude-agents/agents.md @@ -31,6 +31,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "Anthropi | `src/launchdarkly_ai_claude_agents/native_graph.py` | `to_claude_agents()` native graph adapter | | `src/launchdarkly_ai_claude_agents/builtins.py` | Pre-constructed `NativeTool` sentinels for Claude built-in tools | | `src/launchdarkly_ai_claude_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_claude_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py index 8b2dcab6..d74b486c 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py @@ -3,11 +3,10 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package from . import native_graph # noqa: F401 +from ._version import PACKAGE_NAME, __version__ from .builtins import ( ClaudeBash, ClaudeEdit, @@ -50,4 +49,4 @@ "to_claude_agents", ] -register_ai_sdk_package("launchdarkly-ai-claude-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py new file mode 100644 index 00000000..b6dada91 --- /dev/null +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-claude-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py index 994ffe04..1311a6c2 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py @@ -7,6 +7,8 @@ from launchdarkly_ai_claude_agents.handler import _create_claude_agents_handler from launchdarkly_ai_server import _graph, report_usage +from ._version import PACKAGE_NAME, __version__ + def claude_graph(key: str, **options: Any) -> Any: """ @@ -15,5 +17,5 @@ def claude_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_claude_agents_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - report_usage("claude-agents.claudeGraph") + report_usage("claude-agents.claudeGraph", PACKAGE_NAME, __version__) return _graph(key, handlers=[_create_claude_agents_handler()], **options) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index dbf9468f..eb7665e1 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -49,6 +49,7 @@ set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( MCP_TOOL_PREFIX, TOOL_MCP_NAME, @@ -509,7 +510,7 @@ def create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHa Set *capture_content* to put prompts, model output, tool arguments and tool results on the emitted spans. It defaults to off. See TELEMETRY-CONTRACT.md section 7. """ - report_usage("claude-agents.createClaudeAgentsHandler") + report_usage("claude-agents.createClaudeAgentsHandler", PACKAGE_NAME, __version__) return _create_claude_agents_handler(capture_content=capture_content) @@ -893,7 +894,7 @@ def claude_agents( **kwargs: Any, ) -> Any: """Convenience wrapper: creates a handler and calls config(...).invoke().""" - report_usage("claude-agents.claudeAgents") + report_usage("claude-agents.claudeAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py index 2ad60c37..9f92ab0b 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py @@ -22,6 +22,8 @@ to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ + try: from opentelemetry import trace from opentelemetry.trace import StatusCode as SpanStatusCode @@ -208,7 +210,7 @@ def to_claude_agents( Returns an object with a ``.invoke(input, variables)`` coroutine. """ - report_usage("claude-agents.toClaudeAgents") + report_usage("claude-agents.toClaudeAgents", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/claude-agents/tests/test_sdk_usage.py b/packages/claude-agents/tests/test_sdk_usage.py index 06586199..fd576f0f 100644 --- a/packages/claude-agents/tests/test_sdk_usage.py +++ b/packages/claude-agents/tests/test_sdk_usage.py @@ -13,6 +13,8 @@ create_claude_agents_handler, to_claude_agents, ) +from launchdarkly_ai_claude_agents._version import PACKAGE_NAME +from launchdarkly_ai_claude_agents._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -74,6 +76,8 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 @@ -114,3 +118,32 @@ async def _definition() -> Any: with pytest.raises(ValueError, match="disabled"): await runner.invoke("q") assert _helpers(client) == ["claude-agents.toClaudeAgents"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_claude_agents_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "claude-agents.createClaudeAgentsHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/claude-messages/agents.md b/packages/claude-messages/agents.md index 908d8f26..95705470 100644 --- a/packages/claude-messages/agents.md +++ b/packages/claude-messages/agents.md @@ -28,6 +28,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "Anthropi |---|---| | `src/launchdarkly_ai_claude_messages/handler.py` | All implementation — message building, tool schema conversion, tool-use loop, telemetry | | `src/launchdarkly_ai_claude_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_claude_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py index 86b4b319..768d0930 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import claude_messages, create_claude_messages_handler __all__ = ["claude_messages", "create_claude_messages_handler"] -register_ai_sdk_package("launchdarkly-ai-claude-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py new file mode 100644 index 00000000..7c067e2c --- /dev/null +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-claude-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 9bd00c08..9d01c14f 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -26,6 +26,7 @@ to_semconv_finish_reason, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( RawRunUsage, fail_span, @@ -360,7 +361,9 @@ def create_claude_messages_handler(*, capture_content: bool = False) -> Provider meaning models, token counts, timings and tool names, until a caller asks for more. Turning this on sends the text of every request and response to whatever collector the SDK is pointed at. """ - report_usage("claude-messages.createClaudeMessagesHandler") + report_usage( + "claude-messages.createClaudeMessagesHandler", PACKAGE_NAME, __version__ + ) return _create_claude_messages_handler(capture_content=capture_content) @@ -717,7 +720,7 @@ def claude_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. - report_usage("claude-messages.claudeMessages") + report_usage("claude-messages.claudeMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/claude-messages/tests/test_sdk_usage.py b/packages/claude-messages/tests/test_sdk_usage.py index 5af13e9b..7ecc294b 100644 --- a/packages/claude-messages/tests/test_sdk_usage.py +++ b/packages/claude-messages/tests/test_sdk_usage.py @@ -10,6 +10,8 @@ claude_messages, create_claude_messages_handler, ) +from launchdarkly_ai_claude_messages._version import PACKAGE_NAME +from launchdarkly_ai_claude_messages._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -69,5 +71,36 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_claude_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "claude-messages.createClaudeMessagesHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/client/src/launchdarkly_ai_server/sdk_usage.py b/packages/client/src/launchdarkly_ai_server/sdk_usage.py index 41a1717e..5152709b 100644 --- a/packages/client/src/launchdarkly_ai_server/sdk_usage.py +++ b/packages/client/src/launchdarkly_ai_server/sdk_usage.py @@ -3,6 +3,12 @@ One ``$ld:ai:sdk:usage`` event per helper per client. A call made before a client exists is held and sent on the next init, on the same path as sdk-info. +Each event names the core client (``aiSdkName`` / ``aiSdkVersion``) and the +package the helper belongs to (``helperPackageName`` / ``helperPackageVersion``). +Handler packages are versioned separately from the core, and sdk-info sends +every package on one shared context, so only this event ties a helper call to +the handler release that ran it. + Only a call from outside the SDK reports. Each reporting helper has a non-reporting internal version, and SDK code calls that version instead, so this module never needs to know who the caller is. @@ -19,7 +25,11 @@ _SDK_USAGE_LANGUAGE = "python" _SDK_NAME = "launchdarkly-ai-server" -_pending: set[str] = set() +# A helper's package: (name, version), or None for the core client. +_Package = tuple[str, str] | None + +# Held helpers keep their package so the flushed event is complete. +_pending: dict[str, _Package] = {} _reported: set[str] = set() # Guards _pending and _reported. A helper is marked reported under the lock # before it is delivered, so concurrent first calls send it once. The client is @@ -41,18 +51,22 @@ def _version() -> str: return __version__ -def _deliver(client: Any, helper: str) -> None: +def _deliver(client: Any, helper: str, package: _Package) -> None: from .utils import to_ld_context try: + core_version = _version() + package_name, package_version = package or (_SDK_NAME, core_version) client.track( SDK_USAGE_EVENT, to_ld_context(client, SDK_INFO_CONTEXT), { "aiSdkName": _SDK_NAME, - "aiSdkVersion": _version(), + "aiSdkVersion": core_version, "aiSdkLanguage": _SDK_USAGE_LANGUAGE, "helper": helper, + "helperPackageName": package_name, + "helperPackageVersion": package_version, }, 1, ) @@ -61,25 +75,38 @@ def _deliver(client: Any, helper: str) -> None: return -def report_usage(helper: str) -> None: +def report_usage( + helper: str, + package_name: str | None = None, + package_version: str | None = None, +) -> None: """Record a public helper. + A handler package passes the same *package_name* and *package_version* it + gives :func:`register_ai_sdk_package`. A core client helper passes neither, + and the event names the core client. Pass both or neither; passing only one + reports the core client. + Sends immediately when a client exists, otherwise holds the helper until - :func:`flush_sdk_usage`. No-ops when already reported or held. Never raises. + :func:`flush_sdk_usage`. No-ops when already reported or held; the package + is not part of that key. Never raises. """ try: + package: _Package = None + if package_name is not None and package_version is not None: + package = (package_name, package_version) with _lock: client = _peek_client() if helper in _reported or helper in _pending: return if client is None: - _pending.add(helper) + _pending[helper] = package return _reported.add(helper) - _deliver(client, helper) + _deliver(client, helper, package) except Exception: with _lock: - _pending.discard(helper) + _pending.pop(helper, None) _reported.add(helper) @@ -88,11 +115,11 @@ def flush_sdk_usage(client: Any) -> None: with _lock: if not _pending: return - held = list(_pending) + held = list(_pending.items()) _pending.clear() - _reported.update(held) - for helper in held: - _deliver(client, helper) + _reported.update(helper for helper, _ in held) + for helper, package in held: + _deliver(client, helper, package) def reset_sdk_usage() -> None: diff --git a/packages/client/tests/test_sdk_usage.py b/packages/client/tests/test_sdk_usage.py index 962d9da3..0bf87082 100644 --- a/packages/client/tests/test_sdk_usage.py +++ b/packages/client/tests/test_sdk_usage.py @@ -58,7 +58,14 @@ def _helpers(client: MagicMock) -> list[str]: return [call.args[2].get("helper") for call in _usage(client)] -_USAGE_FIELDS = ("helper", "aiSdkName", "aiSdkVersion", "aiSdkLanguage") +_USAGE_FIELDS = ( + "helper", + "aiSdkName", + "aiSdkVersion", + "aiSdkLanguage", + "helperPackageName", + "helperPackageVersion", +) _ENABLED_CONFIG: dict[str, Any] = { "model": {"name": "gpt-4o"}, @@ -82,7 +89,13 @@ def _expect_usage(client: MagicMock, helper: str) -> None: "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": "launchdarkly-ai-server", + "helperPackageVersion": __version__, } + # A client.* helper belongs to the core package. + payload = calls[0].args[2] + assert payload["helperPackageName"] == payload["aiSdkName"] + assert payload["helperPackageVersion"] == payload["aiSdkVersion"] assert calls[0].args[3] == 1 @@ -433,10 +446,55 @@ async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: assert "$ld:ai:generation:success" in names assert "$ld:ai:tokens:total" in names for call in others: - payload = call.args[2] - assert "helper" not in payload - assert "aiSdkName" not in payload - assert "aiSdkVersion" not in payload - assert "aiSdkLanguage" not in payload + for field in _USAGE_FIELDS: + assert field not in call.args[2] finally: await shutdown() + + +def _package_payload(helper: str) -> dict[str, Any]: + return { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": "launchdarkly-ai-example", + "helperPackageVersion": "9.8.7", + } + + +async def test_package_helper_names_its_package_and_the_core( + client: MagicMock, +) -> None: + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example", "9.8.7") + calls = _usage(client, "example.helper") + assert [call.args[2] for call in calls] == [_package_payload("example.helper")] + + # The package is not part of the dedupe key. + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-other", "1.0.0") + assert len(_usage(client, "example.helper")) == 1 + + +async def test_held_package_helper_flushes_with_its_package() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example", "9.8.7") + await build_judge_tasks(**_judge_kwargs()) + client = _client() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + assert [call.args[2] for call in _usage(client, "example.helper")] == [ + _package_payload("example.helper") + ] + _expect_usage(client, HELPER) + await shutdown() + + +async def test_package_name_without_version_reports_the_core( + client: MagicMock, +) -> None: + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example") + calls = _usage(client, "example.helper") + assert len(calls) == 1 + assert calls[0].args[2]["helperPackageName"] == "launchdarkly-ai-server" + assert calls[0].args[2]["helperPackageVersion"] == __version__ diff --git a/packages/langchain-agents/agents.md b/packages/langchain-agents/agents.md index 37a5bdbf..e771dc13 100644 --- a/packages/langchain-agents/agents.md +++ b/packages/langchain-agents/agents.md @@ -34,6 +34,7 @@ The handler is model-agnostic — it accepts any `BaseChatModel`. The default is | `src/launchdarkly_ai_langchain_agents/graph.py` | `langchain_graph()` convenience wrapper around `graph()` | | `src/launchdarkly_ai_langchain_agents/native_graph.py` | `to_lang_graph()` native graph adapter | | `src/launchdarkly_ai_langchain_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_langchain_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py index bef1b8b6..a38273bc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py @@ -3,10 +3,9 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .graph import langchain_graph from .handler import create_langchain_agents_handler, langchain_agents from .native_graph import to_lang_graph @@ -18,4 +17,4 @@ "to_lang_graph", ] -register_ai_sdk_package("launchdarkly-ai-langchain-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py new file mode 100644 index 00000000..b492a4b0 --- /dev/null +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-langchain-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py index f184ae59..4c83f00d 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py @@ -7,6 +7,8 @@ from launchdarkly_ai_langchain_agents.handler import _create_langchain_agents_handler from launchdarkly_ai_server import _graph, report_usage +from ._version import PACKAGE_NAME, __version__ + def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: """ @@ -15,5 +17,5 @@ def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_langchain_agents_handler(llm)], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - report_usage("langchain-agents.langchainGraph") + report_usage("langchain-agents.langchainGraph", PACKAGE_NAME, __version__) return _graph(key, handlers=[_create_langchain_agents_handler(llm)], **options) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index c2d5ea83..ef600b48 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -32,6 +32,7 @@ set_output_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .messages import to_lang_chain_messages from .spans import ( build_span_callbacks, @@ -266,7 +267,9 @@ def create_langchain_agents_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ - report_usage("langchain-agents.createLangChainAgentsHandler") + report_usage( + "langchain-agents.createLangChainAgentsHandler", PACKAGE_NAME, __version__ + ) return _create_langchain_agents_handler(llm, capture_content=capture_content) @@ -579,7 +582,7 @@ def langchain_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. - report_usage("langchain-agents.langchainAgents") + report_usage("langchain-agents.langchainAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py index 135dd95d..039fe0fc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py @@ -24,6 +24,7 @@ to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ from .messages import to_lang_chain_messages try: @@ -133,7 +134,7 @@ def to_lang_graph( {"context": ctx}, ).invoke("I was double charged") """ - report_usage("langchain-agents.toLangGraph") + report_usage("langchain-agents.toLangGraph", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/langchain-agents/tests/test_sdk_usage.py b/packages/langchain-agents/tests/test_sdk_usage.py index 4d438fc9..27a0ced8 100644 --- a/packages/langchain-agents/tests/test_sdk_usage.py +++ b/packages/langchain-agents/tests/test_sdk_usage.py @@ -13,6 +13,8 @@ langchain_graph, to_lang_graph, ) +from launchdarkly_ai_langchain_agents._version import PACKAGE_NAME +from launchdarkly_ai_langchain_agents._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -80,6 +82,8 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 @@ -120,3 +124,32 @@ async def _definition() -> Any: with pytest.raises(ValueError, match="disabled"): await runner.invoke("q") assert _helpers(client) == ["langchain-agents.toLangGraph"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_langchain_agents_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "langchain-agents.createLangChainAgentsHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/langchain-messages/agents.md b/packages/langchain-messages/agents.md index 78314502..6361a8d3 100644 --- a/packages/langchain-messages/agents.md +++ b/packages/langchain-messages/agents.md @@ -32,6 +32,7 @@ The handler is model-agnostic — it accepts any `BaseChatModel`. The default is |---|---| | `src/launchdarkly_ai_langchain_messages/handler.py` | All implementation — message building, tool binding, invoke loop, telemetry | | `src/launchdarkly_ai_langchain_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_langchain_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py index f6bf8df9..103ca420 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import create_langchain_messages_handler, langchain_messages __all__ = ["create_langchain_messages_handler", "langchain_messages"] -register_ai_sdk_package("launchdarkly-ai-langchain-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py new file mode 100644 index 00000000..bc2964f0 --- /dev/null +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-langchain-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index bd9f04f9..e9089166 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -33,6 +33,7 @@ set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( fail_span, finish_model_span, @@ -385,7 +386,7 @@ def create_langchain_messages_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ - report_usage("langchain-messages.createLangChainHandler") + report_usage("langchain-messages.createLangChainHandler", PACKAGE_NAME, __version__) return _create_langchain_messages_handler(llm, capture_content=capture_content) @@ -1022,7 +1023,7 @@ def langchain_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. - report_usage("langchain-messages.langchainMessages") + report_usage("langchain-messages.langchainMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/langchain-messages/tests/test_sdk_usage.py b/packages/langchain-messages/tests/test_sdk_usage.py index 2d7f9b59..e297e4c6 100644 --- a/packages/langchain-messages/tests/test_sdk_usage.py +++ b/packages/langchain-messages/tests/test_sdk_usage.py @@ -10,6 +10,8 @@ create_langchain_messages_handler, langchain_messages, ) +from launchdarkly_ai_langchain_messages._version import PACKAGE_NAME +from launchdarkly_ai_langchain_messages._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -75,5 +77,36 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_langchain_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "langchain-messages.createLangChainHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/openai-agents/agents.md b/packages/openai-agents/agents.md index f86aee58..414dcc9f 100644 --- a/packages/openai-agents/agents.md +++ b/packages/openai-agents/agents.md @@ -31,6 +31,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "OpenAI"` | `src/launchdarkly_ai_openai_agents/native_graph.py` | `to_openai_agents()` native graph adapter | | `src/launchdarkly_ai_openai_agents/utils.py` | Shared utility helpers (e.g. `await_coroutine_or_run`) | | `src/launchdarkly_ai_openai_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_openai_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py index 04e788e3..0475189c 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py @@ -3,11 +3,10 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package from . import native_graph # noqa: F401 +from ._version import PACKAGE_NAME, __version__ from .graph import openai_graph from .handler import create_openai_agent_handler, openai_agents from .native_graph import to_openai_agents @@ -21,4 +20,4 @@ "to_openai_agents", ] -register_ai_sdk_package("launchdarkly-ai-openai-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py new file mode 100644 index 00000000..ccb6ea43 --- /dev/null +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-openai-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py index c9b518bd..b509f688 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py @@ -7,6 +7,8 @@ from launchdarkly_ai_openai_agents.handler import _create_openai_agent_handler from launchdarkly_ai_server import _graph, report_usage +from ._version import PACKAGE_NAME, __version__ + def openai_graph(key: str, **options: Any) -> Any: """ @@ -15,5 +17,5 @@ def openai_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_openai_agent_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - report_usage("openai-agents.openaiGraph") + report_usage("openai-agents.openaiGraph", PACKAGE_NAME, __version__) return _graph(key, handlers=[_create_openai_agent_handler()], **options) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 65fccad2..355aaab9 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -46,6 +46,7 @@ text_message, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( derive_finish_reason, fail_span, @@ -465,7 +466,7 @@ def create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHan emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ - report_usage("openai-agents.createOpenAIAgentHandler") + report_usage("openai-agents.createOpenAIAgentHandler", PACKAGE_NAME, __version__) return _create_openai_agent_handler(capture_content=capture_content) @@ -719,7 +720,7 @@ def openai_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. - report_usage("openai-agents.openaiAgents") + report_usage("openai-agents.openaiAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py index 96847ddf..eb771ab8 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py @@ -24,6 +24,7 @@ to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ from .handler import _parse_message_content, _to_openai_agent_items try: @@ -103,7 +104,7 @@ def to_openai_agents( {"context": ctx}, ).invoke("I was double charged") """ - report_usage("openai-agents.toOpenAIAgents") + report_usage("openai-agents.toOpenAIAgents", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/openai-agents/tests/test_sdk_usage.py b/packages/openai-agents/tests/test_sdk_usage.py index 5af3a3cc..73621084 100644 --- a/packages/openai-agents/tests/test_sdk_usage.py +++ b/packages/openai-agents/tests/test_sdk_usage.py @@ -13,6 +13,8 @@ openai_graph, to_openai_agents, ) +from launchdarkly_ai_openai_agents._version import PACKAGE_NAME +from launchdarkly_ai_openai_agents._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -74,6 +76,8 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 @@ -114,3 +118,32 @@ async def _definition() -> Any: with pytest.raises(ValueError, match="disabled"): await runner.invoke("q") assert _helpers(client) == ["openai-agents.toOpenAIAgents"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_openai_agent_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "openai-agents.createOpenAIAgentHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/openai-messages/agents.md b/packages/openai-messages/agents.md index 15a858c1..5f29cae4 100644 --- a/packages/openai-messages/agents.md +++ b/packages/openai-messages/agents.md @@ -28,6 +28,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "OpenAI"` |---|---| | `src/launchdarkly_ai_openai_messages/handler.py` | All implementation — message building, tool schema conversion, function-call loop, telemetry | | `src/launchdarkly_ai_openai_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_openai_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py index e9242442..f556a943 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import create_openai_messages_handler, openai_messages __all__ = ["create_openai_messages_handler", "openai_messages"] -register_ai_sdk_package("launchdarkly-ai-openai-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py new file mode 100644 index 00000000..8559e759 --- /dev/null +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-openai-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index 055a6fed..6afc2b49 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -28,6 +28,7 @@ set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( fail_span, finish_model_span, @@ -220,7 +221,7 @@ def create_openai_messages_handler(*, capture_content: bool = False) -> Provider emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ - report_usage("openai-messages.createOpenAIHandler") + report_usage("openai-messages.createOpenAIHandler", PACKAGE_NAME, __version__) return _create_openai_messages_handler(capture_content=capture_content) @@ -650,7 +651,7 @@ def openai_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. - report_usage("openai-messages.openaiMessages") + report_usage("openai-messages.openaiMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) return _config( diff --git a/packages/openai-messages/tests/test_sdk_usage.py b/packages/openai-messages/tests/test_sdk_usage.py index 06697886..ec3ab5fc 100644 --- a/packages/openai-messages/tests/test_sdk_usage.py +++ b/packages/openai-messages/tests/test_sdk_usage.py @@ -10,6 +10,8 @@ create_openai_messages_handler, openai_messages, ) +from launchdarkly_ai_openai_messages._version import PACKAGE_NAME +from launchdarkly_ai_openai_messages._version import __version__ as PACKAGE_VERSION from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -70,5 +72,36 @@ async def test_helper_sends_one_usage_event( "aiSdkVersion": __version__, "aiSdkLanguage": "python", "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, } assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_openai_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "openai-messages.createOpenAIHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/release-please-config.json b/release-please-config.json index 9d2a04e3..d68c9835 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -31,7 +31,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_claude_agents/__init__.py" + "src/launchdarkly_ai_claude_agents/_version.py" ], "component": "launchdarkly-ai-claude-agents" }, @@ -42,7 +42,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_claude_messages/__init__.py" + "src/launchdarkly_ai_claude_messages/_version.py" ], "component": "launchdarkly-ai-claude-messages" }, @@ -53,7 +53,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_openai_agents/__init__.py" + "src/launchdarkly_ai_openai_agents/_version.py" ], "component": "launchdarkly-ai-openai-agents" }, @@ -64,7 +64,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_openai_messages/__init__.py" + "src/launchdarkly_ai_openai_messages/_version.py" ], "component": "launchdarkly-ai-openai-messages" }, @@ -75,7 +75,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_langchain_agents/__init__.py" + "src/launchdarkly_ai_langchain_agents/_version.py" ], "component": "launchdarkly-ai-langchain-agents" }, @@ -86,7 +86,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_langchain_messages/__init__.py" + "src/launchdarkly_ai_langchain_messages/_version.py" ], "component": "launchdarkly-ai-langchain-messages" }